diff --git a/src/Appwrite/Platform/Modules/Compute/Base.php b/src/Appwrite/Platform/Modules/Compute/Base.php index 00d839e472..25f071d253 100644 --- a/src/Appwrite/Platform/Modules/Compute/Base.php +++ b/src/Appwrite/Platform/Modules/Compute/Base.php @@ -4,9 +4,7 @@ namespace Appwrite\Platform\Modules\Compute; use Appwrite\Compute\Job; use Appwrite\Event\Message\Build as BuildMessage; -use Appwrite\Event\Message\Jobs as JobsMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; -use Appwrite\Event\Publisher\Jobs as JobsPublisher; use Appwrite\Extend\Exception; use Appwrite\Filter\BranchDomain as BranchDomainFilter; use Appwrite\Platform\Action; @@ -71,7 +69,7 @@ class Base extends Action return $allowedSpecifications[0]; } - public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null, ?JobsPublisher $publisherForJobs = null): Document + public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null): Document { $deploymentId = ID::unique(); $entrypoint = $function->getAttribute('entrypoint', ''); @@ -122,13 +120,16 @@ class Base extends Action $repositoryUrl = "https://github.com/$owner/$repositoryName"; - // Build a VCS function deployment on the jobs-service when the caller - // opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator. Sites stay on - // the executor. When on jobs the deployment is pre-declared 'waiting' - // with its buildPath so build.sh writes output onto the mounted volume. + // Build a plain (non-template) VCS function deployment on the jobs-service + // when the caller opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator. + // Sites stay on the executor; template-into-repo pushes go through the + // Builds worker (which does the git write, then hands the build to the + // jobs-service itself when on orchestrator). When on jobs the deployment + // is pre-declared 'waiting' with its buildPath so build.sh writes output + // onto the mounted volume. $useJobs = $jobs !== null && $function->getCollection() === 'functions' - && ($template->isEmpty() || $publisherForJobs !== null) + && $template->isEmpty() && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'; $buildFields = $useJobs ? ['status' => 'waiting', 'buildPath' => Job::buildPath($project->getId(), $deploymentId)] @@ -168,22 +169,7 @@ class Base extends Action ...$buildFields, ])); - if ($useJobs && ! $template->isEmpty()) { - // The template must be merged into the repo and pushed as a commit - // before the build — a git *write* the jobs-service artifact system - // has no primitive for. Hand off to the jobs worker, which pushes - // the template and then submits the build for the resulting commit. - $publisherForJobs->enqueue(new JobsMessage( - project: $project, - id: 'template-push-' . $deployment->getId(), - event: 'appwrite.template.push', - data: [ - 'meta' => ['projectId' => $project->getId(), 'deploymentId' => $deployment->getId()], - 'template' => $template->getArrayCopy(), - 'platform' => $platform, - ], - )); - } elseif ($useJobs) { + if ($useJobs) { $ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch'); $source = [ 'url' => $github->getRepositoryPresignedUrl($owner, $repositoryName, $ref), diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php index 363538ba9b..202e1c34d3 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php @@ -6,7 +6,6 @@ use Appwrite\Compute\Job; use Appwrite\Event\Event; use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; -use Appwrite\Event\Publisher\Jobs as JobsPublisher; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; use Appwrite\Platform\Modules\Compute\Base; @@ -84,7 +83,6 @@ class Create extends Base ->inject('queueForEvents') ->inject('project') ->inject('publisherForBuilds') - ->inject('publisherForJobs') ->inject('jobs') ->inject('gitHub') ->inject('authorization') @@ -107,7 +105,6 @@ class Create extends Base Event $queueForEvents, Document $project, BuildPublisher $publisherForBuilds, - JobsPublisher $publisherForJobs, Jobs $jobs, GitHub $github, Authorization $authorization, @@ -132,10 +129,10 @@ class Create extends Base ]); if (!empty($function->getAttribute('providerRepositoryId'))) { - // VCS-connected function: the template is merged into the user's repo - // and pushed as a commit, then that commit is built. The executor does - // the push inside the build job; the jobs backend does it in the jobs - // worker (which then submits the build for the resulting commit). + // VCS-connected function: the Builds worker merges the template into + // the user's repo, pushes it as a commit, then builds that commit — + // on the executor itself, or by submitting a job when + // _APP_BUILDS_BACKEND=orchestrator. $installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId')); $deployment = $this->redeployVcsFunction( @@ -150,9 +147,7 @@ class Create extends Base activate: $activate, platform: $platform, referenceType: $type, - reference: $reference, - jobs: $jobs, - publisherForJobs: $publisherForJobs + reference: $reference ); $queueForEvents diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 7c05c057de..fccafa256d 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Functions\Workers; use Ahc\Jwt\JWT; +use Appwrite\Compute\Job; use Appwrite\Event\Event; use Appwrite\Event\Message\Func as FunctionMessage; use Appwrite\Event\Publisher\Func as FunctionPublisher; @@ -20,6 +21,7 @@ use Exception; use Executor\Exception as ExecutorException; use Executor\Exception\Timeout as ExecutorTimeout; use Executor\Executor; +use OpenRuntimes\Orchestrator\Jobs as JobsClient; use Swoole\Coroutine as Co; use Utopia\Cache\Cache; use Utopia\Config\Config; @@ -98,6 +100,7 @@ class Builds extends Action ->inject('deviceForFiles') ->inject('log') ->inject('executor') + ->inject('jobs') ->inject('plan') ->callback($this->action(...)); } @@ -124,6 +127,7 @@ class Builds extends Action Device $deviceForFiles, Log $log, Executor $executor, + JobsClient $jobs, array $plan ): void { $payload = $message->getPayload(); @@ -168,6 +172,7 @@ class Builds extends Action $getIsResourceBlocked, $log, $executor, + $jobs, $plan, $platform, (int) ($payload['timeout'] ?? System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900)) @@ -204,6 +209,7 @@ class Builds extends Action callable $getIsResourceBlocked, Log $log, Executor $executor, + JobsClient $jobs, array $plan, array $platform, int $timeout @@ -518,6 +524,29 @@ class Builds extends Action ->trigger(); } + // On the jobs backend the only build reaching this worker is the + // template-into-repo push above (a git *write* the jobs-service + // artifact system has no primitive for). With the commit pushed, + // hand the build to the jobs-service like any other VCS commit: + // source from the presigned tarball, output onto the mounted + // builds volume, progress via callbacks to the jobs worker. + if ($resource->getCollection() === 'functions' + && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator') { + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ + 'buildPath' => Job::buildPath($project->getId(), $deploymentId), + ])); + + $ref = $deployment->getAttribute('providerCommitHash') ?: $branchName; + $jobs->create(...Job::build($project, $resource, $deployment, $platform, [ + 'url' => $github->getRepositoryPresignedUrl($cloneOwner, $cloneRepository, $ref), + 'subdir' => $resource->getAttribute('providerRootDirectory', ''), + ])); + + Console::execute('rm -rf ' . \escapeshellarg('/tmp/builds/' . $deploymentId), '', $stdout, $stderr); + + return; + } + $tmpPath = '/tmp/builds/' . $deploymentId; $tmpPathFile = $tmpPath . '/code.tar.gz'; $localDevice = new Local(); diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index 70ffa592d7..9086ffa550 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -13,9 +13,7 @@ use Appwrite\Event\Webhook; use Appwrite\Usage\Build as BuildUsage; use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Response\Model\Deployment; -use OpenRuntimes\Orchestrator\Jobs as JobsClient; use Utopia\Cache\Cache; -use Utopia\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; @@ -66,7 +64,6 @@ class Jobs extends Action ->inject('publisherForUsage') ->inject('usage') ->inject('deviceForBuilds') - ->inject('jobs') ->inject('gitHub') ->inject('cache') ->inject('locks') @@ -85,7 +82,6 @@ class Jobs extends Action UsagePublisher $publisherForUsage, UsageContext $usage, Device $deviceForBuilds, - JobsClient $jobs, GitHub $github, Cache $cache, callable $locks, @@ -97,7 +93,7 @@ class Jobs extends Action return; } - $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $jobs, $github, $cache, $deploymentId): void { + $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $github, $cache, $deploymentId): void { if ($event->id !== '') { $key = 'jobs-event-' . $event->id; if ($cache->load($key, self::DEDUPE_TTL) !== false) { @@ -112,13 +108,10 @@ class Jobs extends Action } // The build writes its output onto the mounted volume before build.sh - // exits, so the exit callback is a truthful completion signal. A - // template push only finalizes when it fails (success just submits - // the build job, whose own exit finalizes later). - $finalizing = \in_array($event->event, ['orchestrator.job.exit', 'appwrite.template.push'], true); + // exits, so the exit callback is a truthful completion signal. + $finalizing = $event->event === 'orchestrator.job.exit'; $deployment = match ($event->event) { - 'appwrite.template.push' => $this->onTemplatePush($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $jobs, $github, $usage, $publisherForUsage), 'orchestrator.job.log' => $this->onLog($dbForProject, $deployment, $event->data), 'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $deployment, $event->data), 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds, $github), @@ -157,7 +150,9 @@ class Jobs extends Action // First build output means the build is running: promote the queued // deployment to 'building' and stamp its start (mirrors the executor). - if ($deployment->getAttribute('status') === 'waiting') { + // 'processing' occurs when the Builds worker prepared the source (the + // template-into-repo push) before handing the build to the jobs-service. + if (\in_array($deployment->getAttribute('status'), ['waiting', 'processing'], true)) { $update['status'] = 'building'; $update['buildStartedAt'] = DateTime::now(); } @@ -165,108 +160,6 @@ class Jobs extends Action return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($update)); } - /** - * Merge a template into a VCS-connected function's repository, push it as a - * commit, and submit the build job for that commit. This is the one VCS - * flow needing a git *write*, which the jobs-service artifact system has no - * primitive for — so it runs here (mirroring the executor Builds worker) - * before the build is handed to the jobs-service. A failed push finalizes - * the deployment as failed; the build's own exit callback finalizes success. - */ - private function onTemplatePush( - Database $dbForProject, - Database $dbForPlatform, - Document $project, - Document $deployment, - array $data, - JobsClient $jobs, - GitHub $github, - UsageContext $usage, - UsagePublisher $publisherForUsage, - ): Document { - $function = $dbForProject->getDocument('functions', $deployment->getAttribute('resourceId')); - - try { - $installation = $dbForPlatform->getDocument('installations', $deployment->getAttribute('installationId', '')); - $providerInstallationId = $installation->getAttribute('providerInstallationId', ''); - if ($function->isEmpty() || $providerInstallationId === '') { - throw new \Exception('Function or installation not found.'); - } - $github->initializeVariables($providerInstallationId, System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY', ''), System::getEnv('_APP_VCS_GITHUB_APP_ID', '')); - - $commitHash = $this->pushTemplate($github, $function, $deployment, new Document($data['template'] ?? [])); - - $owner = $deployment->getAttribute('providerRepositoryOwner', ''); - $repository = $deployment->getAttribute('providerRepositoryName', ''); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'providerCommitHash' => $commitHash, - 'providerCommitAuthorUrl' => APP_VCS_GITHUB_URL, - 'providerCommitAuthor' => APP_VCS_GITHUB_USERNAME, - 'providerCommitMessage' => "Create '" . $function->getAttribute('name', '') . "' function", - 'providerCommitUrl' => "https://github.com/{$owner}/{$repository}/commit/{$commitHash}", - ])); - - $source = [ - 'url' => $github->getRepositoryPresignedUrl($owner, $repository, $commitHash), - 'subdir' => $function->getAttribute('providerRootDirectory', ''), - ]; - $jobs->create(...Job::build($project, $function, $deployment, $data['platform'] ?? [], $source)); - - return $deployment; - } catch (\Throwable $error) { - return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, $error->getMessage(), $usage, $publisherForUsage, $github); - } - } - - /** - * Clone the function's repository, merge the template into it and push the - * result, returning the new commit hash. Replicates the executor Builds - * worker's clone + rsync + commit + push. - */ - private function pushTemplate(GitHub $github, Document $function, Document $deployment, Document $template): string - { - $owner = $deployment->getAttribute('providerRepositoryOwner', ''); - $repository = $deployment->getAttribute('providerRepositoryName', ''); - $branch = $deployment->getAttribute('providerBranch') ?: $function->getAttribute('providerBranch', 'main'); - - $rootDirectory = \ltrim(\ltrim(\rtrim($function->getAttribute('providerRootDirectory', ''), '/'), '.'), '/'); - $templateRootDirectory = \ltrim(\ltrim(\rtrim($template->getAttribute('rootDirectory', ''), '/'), '.'), '/'); - - $tmpDirectory = '/tmp/builds/' . $deployment->getId(); - $repoDirectory = $tmpDirectory . '/code'; - $templateDirectory = $tmpDirectory . '/template'; - $stdout = ''; - $stderr = ''; - - try { - if (Console::execute($github->generateCloneCommand($owner, $repository, $branch, GitHub::CLONE_TYPE_BRANCH, $repoDirectory, $rootDirectory), '', $stdout, $stderr) !== 0) { - throw new \Exception('Unable to clone repository: ' . $stderr); - } - if (Console::execute($github->generateCloneCommand($template->getAttribute('ownerName', ''), $template->getAttribute('repositoryName', ''), $template->getAttribute('referenceValue', ''), $template->getAttribute('referenceType', ''), $templateDirectory, $templateRootDirectory), '', $stdout, $stderr) !== 0) { - throw new \Exception('Unable to clone template repository: ' . $stderr); - } - - Console::execute('mkdir -p ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr); - Console::execute('mkdir -p ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr); - - // Merge the template into the repo, then commit + push the branch. - Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr); - - $message = \escapeshellarg("Create '" . $function->getAttribute('name', '') . "' function"); - if (Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($repoDirectory) . ' && git checkout -b ' . \escapeshellarg($branch) . ' && git add . && git commit -m ' . $message . ' && git push origin ' . \escapeshellarg($branch), '', $stdout, $stderr) !== 0) { - throw new \Exception('Unable to push to repository: ' . $stderr); - } - - if (Console::execute('cd ' . \escapeshellarg($repoDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr) !== 0) { - throw new \Exception('Unable to resolve commit hash: ' . $stderr); - } - - return \trim($stdout); - } finally { - Console::execute('rm -rf ' . \escapeshellarg($tmpDirectory), '', $stdout, $stderr); - } - } - /** * Record a reported artifact size. Remote-source builds (templates / VCS) * fetch the source in the sidecar, so Appwrite can't size it the way the