From e9a9a4162d2dbd071678acf186e61e4fb84d8e52 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:42:51 +0100 Subject: [PATCH] Keep template-into-repo push in the Builds worker; hand off to jobs after Reverts the jobs-worker appwrite.template.push message. The template flow enqueues to the Builds worker unchanged on both backends; the worker does its existing clone + merge + commit + push, and when _APP_BUILDS_BACKEND=orchestrator it then submits the build job for the pushed commit (presigned tarball source, buildPath onto the mounted volume) instead of running the executor build. The jobs worker's onLog promotes 'processing' (set by the Builds worker) to 'building' as well as 'waiting'. Co-Authored-By: Claude Fable 5 --- .../Platform/Modules/Compute/Base.php | 34 ++--- .../Http/Deployments/Template/Create.php | 15 +-- .../Modules/Functions/Workers/Builds.php | 29 +++++ .../Modules/Functions/Workers/Jobs.php | 119 +----------------- 4 files changed, 50 insertions(+), 147 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Compute/Base.php b/src/Appwrite/Platform/Modules/Compute/Base.php index 00d839e472..25f071d253 100644 --- a/src/Appwrite/Platform/Modules/Compute/Base.php +++ b/src/Appwrite/Platform/Modules/Compute/Base.php @@ -4,9 +4,7 @@ namespace Appwrite\Platform\Modules\Compute; use Appwrite\Compute\Job; use Appwrite\Event\Message\Build as BuildMessage; -use Appwrite\Event\Message\Jobs as JobsMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; -use Appwrite\Event\Publisher\Jobs as JobsPublisher; use Appwrite\Extend\Exception; use Appwrite\Filter\BranchDomain as BranchDomainFilter; use Appwrite\Platform\Action; @@ -71,7 +69,7 @@ class Base extends Action return $allowedSpecifications[0]; } - public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null, ?JobsPublisher $publisherForJobs = null): Document + public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null): Document { $deploymentId = ID::unique(); $entrypoint = $function->getAttribute('entrypoint', ''); @@ -122,13 +120,16 @@ class Base extends Action $repositoryUrl = "https://github.com/$owner/$repositoryName"; - // Build a VCS function deployment on the jobs-service when the caller - // opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator. Sites stay on - // the executor. When on jobs the deployment is pre-declared 'waiting' - // with its buildPath so build.sh writes output onto the mounted volume. + // Build a plain (non-template) VCS function deployment on the jobs-service + // when the caller opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator. + // Sites stay on the executor; template-into-repo pushes go through the + // Builds worker (which does the git write, then hands the build to the + // jobs-service itself when on orchestrator). When on jobs the deployment + // is pre-declared 'waiting' with its buildPath so build.sh writes output + // onto the mounted volume. $useJobs = $jobs !== null && $function->getCollection() === 'functions' - && ($template->isEmpty() || $publisherForJobs !== null) + && $template->isEmpty() && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'; $buildFields = $useJobs ? ['status' => 'waiting', 'buildPath' => Job::buildPath($project->getId(), $deploymentId)] @@ -168,22 +169,7 @@ class Base extends Action ...$buildFields, ])); - if ($useJobs && ! $template->isEmpty()) { - // The template must be merged into the repo and pushed as a commit - // before the build — a git *write* the jobs-service artifact system - // has no primitive for. Hand off to the jobs worker, which pushes - // the template and then submits the build for the resulting commit. - $publisherForJobs->enqueue(new JobsMessage( - project: $project, - id: 'template-push-' . $deployment->getId(), - event: 'appwrite.template.push', - data: [ - 'meta' => ['projectId' => $project->getId(), 'deploymentId' => $deployment->getId()], - 'template' => $template->getArrayCopy(), - 'platform' => $platform, - ], - )); - } elseif ($useJobs) { + if ($useJobs) { $ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch'); $source = [ 'url' => $github->getRepositoryPresignedUrl($owner, $repositoryName, $ref), diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php index 363538ba9b..202e1c34d3 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php @@ -6,7 +6,6 @@ use Appwrite\Compute\Job; use Appwrite\Event\Event; use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; -use Appwrite\Event\Publisher\Jobs as JobsPublisher; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; use Appwrite\Platform\Modules\Compute\Base; @@ -84,7 +83,6 @@ class Create extends Base ->inject('queueForEvents') ->inject('project') ->inject('publisherForBuilds') - ->inject('publisherForJobs') ->inject('jobs') ->inject('gitHub') ->inject('authorization') @@ -107,7 +105,6 @@ class Create extends Base Event $queueForEvents, Document $project, BuildPublisher $publisherForBuilds, - JobsPublisher $publisherForJobs, Jobs $jobs, GitHub $github, Authorization $authorization, @@ -132,10 +129,10 @@ class Create extends Base ]); if (!empty($function->getAttribute('providerRepositoryId'))) { - // VCS-connected function: the template is merged into the user's repo - // and pushed as a commit, then that commit is built. The executor does - // the push inside the build job; the jobs backend does it in the jobs - // worker (which then submits the build for the resulting commit). + // VCS-connected function: the Builds worker merges the template into + // the user's repo, pushes it as a commit, then builds that commit — + // on the executor itself, or by submitting a job when + // _APP_BUILDS_BACKEND=orchestrator. $installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId')); $deployment = $this->redeployVcsFunction( @@ -150,9 +147,7 @@ class Create extends Base activate: $activate, platform: $platform, referenceType: $type, - reference: $reference, - jobs: $jobs, - publisherForJobs: $publisherForJobs + reference: $reference ); $queueForEvents diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 7c05c057de..fccafa256d 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Functions\Workers; use Ahc\Jwt\JWT; +use Appwrite\Compute\Job; use Appwrite\Event\Event; use Appwrite\Event\Message\Func as FunctionMessage; use Appwrite\Event\Publisher\Func as FunctionPublisher; @@ -20,6 +21,7 @@ use Exception; use Executor\Exception as ExecutorException; use Executor\Exception\Timeout as ExecutorTimeout; use Executor\Executor; +use OpenRuntimes\Orchestrator\Jobs as JobsClient; use Swoole\Coroutine as Co; use Utopia\Cache\Cache; use Utopia\Config\Config; @@ -98,6 +100,7 @@ class Builds extends Action ->inject('deviceForFiles') ->inject('log') ->inject('executor') + ->inject('jobs') ->inject('plan') ->callback($this->action(...)); } @@ -124,6 +127,7 @@ class Builds extends Action Device $deviceForFiles, Log $log, Executor $executor, + JobsClient $jobs, array $plan ): void { $payload = $message->getPayload(); @@ -168,6 +172,7 @@ class Builds extends Action $getIsResourceBlocked, $log, $executor, + $jobs, $plan, $platform, (int) ($payload['timeout'] ?? System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900)) @@ -204,6 +209,7 @@ class Builds extends Action callable $getIsResourceBlocked, Log $log, Executor $executor, + JobsClient $jobs, array $plan, array $platform, int $timeout @@ -518,6 +524,29 @@ class Builds extends Action ->trigger(); } + // On the jobs backend the only build reaching this worker is the + // template-into-repo push above (a git *write* the jobs-service + // artifact system has no primitive for). With the commit pushed, + // hand the build to the jobs-service like any other VCS commit: + // source from the presigned tarball, output onto the mounted + // builds volume, progress via callbacks to the jobs worker. + if ($resource->getCollection() === 'functions' + && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator') { + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ + 'buildPath' => Job::buildPath($project->getId(), $deploymentId), + ])); + + $ref = $deployment->getAttribute('providerCommitHash') ?: $branchName; + $jobs->create(...Job::build($project, $resource, $deployment, $platform, [ + 'url' => $github->getRepositoryPresignedUrl($cloneOwner, $cloneRepository, $ref), + 'subdir' => $resource->getAttribute('providerRootDirectory', ''), + ])); + + Console::execute('rm -rf ' . \escapeshellarg('/tmp/builds/' . $deploymentId), '', $stdout, $stderr); + + return; + } + $tmpPath = '/tmp/builds/' . $deploymentId; $tmpPathFile = $tmpPath . '/code.tar.gz'; $localDevice = new Local(); diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index 70ffa592d7..9086ffa550 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -13,9 +13,7 @@ use Appwrite\Event\Webhook; use Appwrite\Usage\Build as BuildUsage; use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Response\Model\Deployment; -use OpenRuntimes\Orchestrator\Jobs as JobsClient; use Utopia\Cache\Cache; -use Utopia\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; @@ -66,7 +64,6 @@ class Jobs extends Action ->inject('publisherForUsage') ->inject('usage') ->inject('deviceForBuilds') - ->inject('jobs') ->inject('gitHub') ->inject('cache') ->inject('locks') @@ -85,7 +82,6 @@ class Jobs extends Action UsagePublisher $publisherForUsage, UsageContext $usage, Device $deviceForBuilds, - JobsClient $jobs, GitHub $github, Cache $cache, callable $locks, @@ -97,7 +93,7 @@ class Jobs extends Action return; } - $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $jobs, $github, $cache, $deploymentId): void { + $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $github, $cache, $deploymentId): void { if ($event->id !== '') { $key = 'jobs-event-' . $event->id; if ($cache->load($key, self::DEDUPE_TTL) !== false) { @@ -112,13 +108,10 @@ class Jobs extends Action } // The build writes its output onto the mounted volume before build.sh - // exits, so the exit callback is a truthful completion signal. A - // template push only finalizes when it fails (success just submits - // the build job, whose own exit finalizes later). - $finalizing = \in_array($event->event, ['orchestrator.job.exit', 'appwrite.template.push'], true); + // exits, so the exit callback is a truthful completion signal. + $finalizing = $event->event === 'orchestrator.job.exit'; $deployment = match ($event->event) { - 'appwrite.template.push' => $this->onTemplatePush($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $jobs, $github, $usage, $publisherForUsage), 'orchestrator.job.log' => $this->onLog($dbForProject, $deployment, $event->data), 'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $deployment, $event->data), 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds, $github), @@ -157,7 +150,9 @@ class Jobs extends Action // First build output means the build is running: promote the queued // deployment to 'building' and stamp its start (mirrors the executor). - if ($deployment->getAttribute('status') === 'waiting') { + // 'processing' occurs when the Builds worker prepared the source (the + // template-into-repo push) before handing the build to the jobs-service. + if (\in_array($deployment->getAttribute('status'), ['waiting', 'processing'], true)) { $update['status'] = 'building'; $update['buildStartedAt'] = DateTime::now(); } @@ -165,108 +160,6 @@ class Jobs extends Action return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($update)); } - /** - * Merge a template into a VCS-connected function's repository, push it as a - * commit, and submit the build job for that commit. This is the one VCS - * flow needing a git *write*, which the jobs-service artifact system has no - * primitive for — so it runs here (mirroring the executor Builds worker) - * before the build is handed to the jobs-service. A failed push finalizes - * the deployment as failed; the build's own exit callback finalizes success. - */ - private function onTemplatePush( - Database $dbForProject, - Database $dbForPlatform, - Document $project, - Document $deployment, - array $data, - JobsClient $jobs, - GitHub $github, - UsageContext $usage, - UsagePublisher $publisherForUsage, - ): Document { - $function = $dbForProject->getDocument('functions', $deployment->getAttribute('resourceId')); - - try { - $installation = $dbForPlatform->getDocument('installations', $deployment->getAttribute('installationId', '')); - $providerInstallationId = $installation->getAttribute('providerInstallationId', ''); - if ($function->isEmpty() || $providerInstallationId === '') { - throw new \Exception('Function or installation not found.'); - } - $github->initializeVariables($providerInstallationId, System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY', ''), System::getEnv('_APP_VCS_GITHUB_APP_ID', '')); - - $commitHash = $this->pushTemplate($github, $function, $deployment, new Document($data['template'] ?? [])); - - $owner = $deployment->getAttribute('providerRepositoryOwner', ''); - $repository = $deployment->getAttribute('providerRepositoryName', ''); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'providerCommitHash' => $commitHash, - 'providerCommitAuthorUrl' => APP_VCS_GITHUB_URL, - 'providerCommitAuthor' => APP_VCS_GITHUB_USERNAME, - 'providerCommitMessage' => "Create '" . $function->getAttribute('name', '') . "' function", - 'providerCommitUrl' => "https://github.com/{$owner}/{$repository}/commit/{$commitHash}", - ])); - - $source = [ - 'url' => $github->getRepositoryPresignedUrl($owner, $repository, $commitHash), - 'subdir' => $function->getAttribute('providerRootDirectory', ''), - ]; - $jobs->create(...Job::build($project, $function, $deployment, $data['platform'] ?? [], $source)); - - return $deployment; - } catch (\Throwable $error) { - return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, $error->getMessage(), $usage, $publisherForUsage, $github); - } - } - - /** - * Clone the function's repository, merge the template into it and push the - * result, returning the new commit hash. Replicates the executor Builds - * worker's clone + rsync + commit + push. - */ - private function pushTemplate(GitHub $github, Document $function, Document $deployment, Document $template): string - { - $owner = $deployment->getAttribute('providerRepositoryOwner', ''); - $repository = $deployment->getAttribute('providerRepositoryName', ''); - $branch = $deployment->getAttribute('providerBranch') ?: $function->getAttribute('providerBranch', 'main'); - - $rootDirectory = \ltrim(\ltrim(\rtrim($function->getAttribute('providerRootDirectory', ''), '/'), '.'), '/'); - $templateRootDirectory = \ltrim(\ltrim(\rtrim($template->getAttribute('rootDirectory', ''), '/'), '.'), '/'); - - $tmpDirectory = '/tmp/builds/' . $deployment->getId(); - $repoDirectory = $tmpDirectory . '/code'; - $templateDirectory = $tmpDirectory . '/template'; - $stdout = ''; - $stderr = ''; - - try { - if (Console::execute($github->generateCloneCommand($owner, $repository, $branch, GitHub::CLONE_TYPE_BRANCH, $repoDirectory, $rootDirectory), '', $stdout, $stderr) !== 0) { - throw new \Exception('Unable to clone repository: ' . $stderr); - } - if (Console::execute($github->generateCloneCommand($template->getAttribute('ownerName', ''), $template->getAttribute('repositoryName', ''), $template->getAttribute('referenceValue', ''), $template->getAttribute('referenceType', ''), $templateDirectory, $templateRootDirectory), '', $stdout, $stderr) !== 0) { - throw new \Exception('Unable to clone template repository: ' . $stderr); - } - - Console::execute('mkdir -p ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr); - Console::execute('mkdir -p ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr); - - // Merge the template into the repo, then commit + push the branch. - Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr); - - $message = \escapeshellarg("Create '" . $function->getAttribute('name', '') . "' function"); - if (Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($repoDirectory) . ' && git checkout -b ' . \escapeshellarg($branch) . ' && git add . && git commit -m ' . $message . ' && git push origin ' . \escapeshellarg($branch), '', $stdout, $stderr) !== 0) { - throw new \Exception('Unable to push to repository: ' . $stderr); - } - - if (Console::execute('cd ' . \escapeshellarg($repoDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr) !== 0) { - throw new \Exception('Unable to resolve commit hash: ' . $stderr); - } - - return \trim($stdout); - } finally { - Console::execute('rm -rf ' . \escapeshellarg($tmpDirectory), '', $stdout, $stderr); - } - } - /** * Record a reported artifact size. Remote-source builds (templates / VCS) * fetch the source in the sidecar, so Appwrite can't size it the way the