From 0c075d843e0f174d1c41f9c3d3bccdd118bde579 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:35:59 +0100 Subject: [PATCH] Move template-into-repo git push from the API endpoint to the jobs worker The Template/Create endpoint no longer runs git in the Swoole request flow. On the orchestrator backend, redeployVcsFunction enqueues an appwrite.template.push message to the v1-jobs queue; the jobs worker clones the repo, merges the template, commits + pushes (mirroring the executor Builds worker), records the commit on the deployment, then submits the build job for that commit. A failed push finalizes the deployment as failed with the usual fan-out. The endpoint's orchestrator/executor branch collapses into a single redeployVcsFunction call. Co-Authored-By: Claude Fable 5 --- .../Platform/Modules/Compute/Base.php | 32 +++-- .../Http/Deployments/Template/Create.php | 129 +++--------------- .../Modules/Functions/Workers/Jobs.php | 115 +++++++++++++++- 3 files changed, 158 insertions(+), 118 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Compute/Base.php b/src/Appwrite/Platform/Modules/Compute/Base.php index be88384244..00d839e472 100644 --- a/src/Appwrite/Platform/Modules/Compute/Base.php +++ b/src/Appwrite/Platform/Modules/Compute/Base.php @@ -4,7 +4,9 @@ namespace Appwrite\Platform\Modules\Compute; use Appwrite\Compute\Job; use Appwrite\Event\Message\Build as BuildMessage; +use Appwrite\Event\Message\Jobs as JobsMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; +use Appwrite\Event\Publisher\Jobs as JobsPublisher; use Appwrite\Extend\Exception; use Appwrite\Filter\BranchDomain as BranchDomainFilter; use Appwrite\Platform\Action; @@ -69,7 +71,7 @@ class Base extends Action return $allowedSpecifications[0]; } - public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null): Document + public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null, ?JobsPublisher $publisherForJobs = null): Document { $deploymentId = ID::unique(); $entrypoint = $function->getAttribute('entrypoint', ''); @@ -120,14 +122,13 @@ class Base extends Action $repositoryUrl = "https://github.com/$owner/$repositoryName"; - // Build a plain (non-template) VCS function deployment on the jobs-service - // when the caller opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator. - // Sites and the template-into-repo push stay on the executor. When on - // jobs the deployment is pre-declared 'waiting' with its buildPath so - // build.sh writes output onto the mounted volume. + // Build a VCS function deployment on the jobs-service when the caller + // opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator. Sites stay on + // the executor. When on jobs the deployment is pre-declared 'waiting' + // with its buildPath so build.sh writes output onto the mounted volume. $useJobs = $jobs !== null && $function->getCollection() === 'functions' - && $template->isEmpty() + && ($template->isEmpty() || $publisherForJobs !== null) && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'; $buildFields = $useJobs ? ['status' => 'waiting', 'buildPath' => Job::buildPath($project->getId(), $deploymentId)] @@ -167,7 +168,22 @@ class Base extends Action ...$buildFields, ])); - if ($useJobs) { + if ($useJobs && ! $template->isEmpty()) { + // The template must be merged into the repo and pushed as a commit + // before the build — a git *write* the jobs-service artifact system + // has no primitive for. Hand off to the jobs worker, which pushes + // the template and then submits the build for the resulting commit. + $publisherForJobs->enqueue(new JobsMessage( + project: $project, + id: 'template-push-' . $deployment->getId(), + event: 'appwrite.template.push', + data: [ + 'meta' => ['projectId' => $project->getId(), 'deploymentId' => $deployment->getId()], + 'template' => $template->getArrayCopy(), + 'platform' => $platform, + ], + )); + } elseif ($useJobs) { $ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch'); $source = [ 'url' => $github->getRepositoryPresignedUrl($owner, $repositoryName, $ref), diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php index db8b8aa1ca..363538ba9b 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php @@ -6,6 +6,7 @@ use Appwrite\Compute\Job; use Appwrite\Event\Event; use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; +use Appwrite\Event\Publisher\Jobs as JobsPublisher; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; use Appwrite\Platform\Modules\Compute\Base; @@ -14,7 +15,6 @@ use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response; use OpenRuntimes\Orchestrator\Jobs; -use Utopia\Console; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; @@ -84,6 +84,7 @@ class Create extends Base ->inject('queueForEvents') ->inject('project') ->inject('publisherForBuilds') + ->inject('publisherForJobs') ->inject('jobs') ->inject('gitHub') ->inject('authorization') @@ -106,6 +107,7 @@ class Create extends Base Event $queueForEvents, Document $project, BuildPublisher $publisherForBuilds, + JobsPublisher $publisherForJobs, Jobs $jobs, GitHub $github, Authorization $authorization, @@ -131,49 +133,27 @@ class Create extends Base if (!empty($function->getAttribute('providerRepositoryId'))) { // VCS-connected function: the template is merged into the user's repo - // and pushed as a commit, then that commit is built. + // and pushed as a commit, then that commit is built. The executor does + // the push inside the build job; the jobs backend does it in the jobs + // worker (which then submits the build for the resulting commit). $installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId')); - if (System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator') { - // The jobs-service artifact system only reads source (download / - // unarchive), so it cannot do the git *write*. Perform the merge + - // commit + push here (mirroring the executor Builds worker), then - // build the resulting commit on the jobs-service like any other VCS - // commit — no template merge needed in the build. - $commitHash = $this->pushTemplateToRepository($github, $function, $installation, $owner, $repository, $reference, $type, $rootDirectory); - $deployment = $this->redeployVcsFunction( - request: $request, - function: $function, - project: $project, - installation: $installation, - dbForProject: $dbForProject, - publisherForBuilds: $publisherForBuilds, - template: new Document(), - github: $github, - activate: $activate, - platform: $platform, - referenceType: 'commit', - reference: $commitHash, - jobs: $jobs - ); - } else { - // Executor: the Builds worker clones the repo, merges the template, - // pushes the commit and builds it, all in the build job. - $deployment = $this->redeployVcsFunction( - request: $request, - function: $function, - project: $project, - installation: $installation, - dbForProject: $dbForProject, - publisherForBuilds: $publisherForBuilds, - template: $template, - github: $github, - activate: $activate, - platform: $platform, - referenceType: $type, - reference: $reference - ); - } + $deployment = $this->redeployVcsFunction( + request: $request, + function: $function, + project: $project, + installation: $installation, + dbForProject: $dbForProject, + publisherForBuilds: $publisherForBuilds, + template: $template, + github: $github, + activate: $activate, + platform: $platform, + referenceType: $type, + reference: $reference, + jobs: $jobs, + publisherForJobs: $publisherForJobs + ); $queueForEvents ->setParam('functionId', $function->getId()) @@ -258,69 +238,4 @@ class Create extends Base ->setStatusCode(Response::STATUS_CODE_ACCEPTED) ->dynamic($deployment, Response::MODEL_DEPLOYMENT); } - - /** - * Merge a template into a VCS-connected function's repository and push it as a - * commit, returning the new commit hash. Replicates the executor Builds - * worker's clone + rsync + commit + push for the jobs backend, which has no - * git-write primitive; the resulting commit is then built like any other VCS - * commit. Runs the git binary in the request flow. - */ - private function pushTemplateToRepository( - GitHub $github, - Document $function, - Document $installation, - string $templateOwner, - string $templateRepository, - string $templateReference, - string $templateType, - string $templateRootDirectory, - ): string { - $providerInstallationId = $installation->getAttribute('providerInstallationId', ''); - if (empty($providerInstallationId)) { - throw new Exception(Exception::INSTALLATION_NOT_FOUND); - } - $github->initializeVariables($providerInstallationId, System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY', ''), System::getEnv('_APP_VCS_GITHUB_APP_ID', '')); - - $owner = $github->getOwnerName($providerInstallationId); - $repository = $github->getRepositoryName($function->getAttribute('providerRepositoryId', '')); - $branch = $function->getAttribute('providerBranch', 'main'); - - $rootDirectory = \ltrim(\ltrim(\rtrim($function->getAttribute('providerRootDirectory', ''), '/'), '.'), '/'); - $templateRootDirectory = \ltrim(\ltrim(\rtrim($templateRootDirectory, '/'), '.'), '/'); - - $id = ID::unique(); - $repoDirectory = '/tmp/templates/' . $id . '/code'; - $templateDirectory = '/tmp/templates/' . $id . '/template'; - $stdout = ''; - $stderr = ''; - - try { - if (Console::execute($github->generateCloneCommand($owner, $repository, $branch, GitHub::CLONE_TYPE_BRANCH, $repoDirectory, $rootDirectory), '', $stdout, $stderr) !== 0) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unable to clone repository: ' . $stderr); - } - if (Console::execute($github->generateCloneCommand($templateOwner, $templateRepository, $templateReference, $templateType, $templateDirectory, $templateRootDirectory), '', $stdout, $stderr) !== 0) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unable to clone template repository: ' . $stderr); - } - - Console::execute('mkdir -p ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr); - Console::execute('mkdir -p ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr); - - // Merge the template into the repo, then commit + push the branch. - Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr); - - $message = \escapeshellarg("Create '" . $function->getAttribute('name', '') . "' function"); - if (Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($repoDirectory) . ' && git checkout -b ' . \escapeshellarg($branch) . ' && git add . && git commit -m ' . $message . ' && git push origin ' . \escapeshellarg($branch), '', $stdout, $stderr) !== 0) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unable to push to repository: ' . $stderr); - } - - if (Console::execute('cd ' . \escapeshellarg($repoDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr) !== 0) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unable to resolve commit hash: ' . $stderr); - } - - return \trim($stdout); - } finally { - Console::execute('rm -rf ' . \escapeshellarg('/tmp/templates/' . $id), '', $stdout, $stderr); - } - } } diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index b93f2084a0..70ffa592d7 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -13,7 +13,9 @@ use Appwrite\Event\Webhook; use Appwrite\Usage\Build as BuildUsage; use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Response\Model\Deployment; +use OpenRuntimes\Orchestrator\Jobs as JobsClient; use Utopia\Cache\Cache; +use Utopia\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; @@ -64,6 +66,7 @@ class Jobs extends Action ->inject('publisherForUsage') ->inject('usage') ->inject('deviceForBuilds') + ->inject('jobs') ->inject('gitHub') ->inject('cache') ->inject('locks') @@ -82,6 +85,7 @@ class Jobs extends Action UsagePublisher $publisherForUsage, UsageContext $usage, Device $deviceForBuilds, + JobsClient $jobs, GitHub $github, Cache $cache, callable $locks, @@ -93,7 +97,7 @@ class Jobs extends Action return; } - $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $github, $cache, $deploymentId): void { + $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $jobs, $github, $cache, $deploymentId): void { if ($event->id !== '') { $key = 'jobs-event-' . $event->id; if ($cache->load($key, self::DEDUPE_TTL) !== false) { @@ -108,10 +112,13 @@ class Jobs extends Action } // The build writes its output onto the mounted volume before build.sh - // exits, so the exit callback is a truthful completion signal. - $finalizing = $event->event === 'orchestrator.job.exit'; + // exits, so the exit callback is a truthful completion signal. A + // template push only finalizes when it fails (success just submits + // the build job, whose own exit finalizes later). + $finalizing = \in_array($event->event, ['orchestrator.job.exit', 'appwrite.template.push'], true); $deployment = match ($event->event) { + 'appwrite.template.push' => $this->onTemplatePush($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $jobs, $github, $usage, $publisherForUsage), 'orchestrator.job.log' => $this->onLog($dbForProject, $deployment, $event->data), 'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $deployment, $event->data), 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds, $github), @@ -158,6 +165,108 @@ class Jobs extends Action return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($update)); } + /** + * Merge a template into a VCS-connected function's repository, push it as a + * commit, and submit the build job for that commit. This is the one VCS + * flow needing a git *write*, which the jobs-service artifact system has no + * primitive for — so it runs here (mirroring the executor Builds worker) + * before the build is handed to the jobs-service. A failed push finalizes + * the deployment as failed; the build's own exit callback finalizes success. + */ + private function onTemplatePush( + Database $dbForProject, + Database $dbForPlatform, + Document $project, + Document $deployment, + array $data, + JobsClient $jobs, + GitHub $github, + UsageContext $usage, + UsagePublisher $publisherForUsage, + ): Document { + $function = $dbForProject->getDocument('functions', $deployment->getAttribute('resourceId')); + + try { + $installation = $dbForPlatform->getDocument('installations', $deployment->getAttribute('installationId', '')); + $providerInstallationId = $installation->getAttribute('providerInstallationId', ''); + if ($function->isEmpty() || $providerInstallationId === '') { + throw new \Exception('Function or installation not found.'); + } + $github->initializeVariables($providerInstallationId, System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY', ''), System::getEnv('_APP_VCS_GITHUB_APP_ID', '')); + + $commitHash = $this->pushTemplate($github, $function, $deployment, new Document($data['template'] ?? [])); + + $owner = $deployment->getAttribute('providerRepositoryOwner', ''); + $repository = $deployment->getAttribute('providerRepositoryName', ''); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ + 'providerCommitHash' => $commitHash, + 'providerCommitAuthorUrl' => APP_VCS_GITHUB_URL, + 'providerCommitAuthor' => APP_VCS_GITHUB_USERNAME, + 'providerCommitMessage' => "Create '" . $function->getAttribute('name', '') . "' function", + 'providerCommitUrl' => "https://github.com/{$owner}/{$repository}/commit/{$commitHash}", + ])); + + $source = [ + 'url' => $github->getRepositoryPresignedUrl($owner, $repository, $commitHash), + 'subdir' => $function->getAttribute('providerRootDirectory', ''), + ]; + $jobs->create(...Job::build($project, $function, $deployment, $data['platform'] ?? [], $source)); + + return $deployment; + } catch (\Throwable $error) { + return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, $error->getMessage(), $usage, $publisherForUsage, $github); + } + } + + /** + * Clone the function's repository, merge the template into it and push the + * result, returning the new commit hash. Replicates the executor Builds + * worker's clone + rsync + commit + push. + */ + private function pushTemplate(GitHub $github, Document $function, Document $deployment, Document $template): string + { + $owner = $deployment->getAttribute('providerRepositoryOwner', ''); + $repository = $deployment->getAttribute('providerRepositoryName', ''); + $branch = $deployment->getAttribute('providerBranch') ?: $function->getAttribute('providerBranch', 'main'); + + $rootDirectory = \ltrim(\ltrim(\rtrim($function->getAttribute('providerRootDirectory', ''), '/'), '.'), '/'); + $templateRootDirectory = \ltrim(\ltrim(\rtrim($template->getAttribute('rootDirectory', ''), '/'), '.'), '/'); + + $tmpDirectory = '/tmp/builds/' . $deployment->getId(); + $repoDirectory = $tmpDirectory . '/code'; + $templateDirectory = $tmpDirectory . '/template'; + $stdout = ''; + $stderr = ''; + + try { + if (Console::execute($github->generateCloneCommand($owner, $repository, $branch, GitHub::CLONE_TYPE_BRANCH, $repoDirectory, $rootDirectory), '', $stdout, $stderr) !== 0) { + throw new \Exception('Unable to clone repository: ' . $stderr); + } + if (Console::execute($github->generateCloneCommand($template->getAttribute('ownerName', ''), $template->getAttribute('repositoryName', ''), $template->getAttribute('referenceValue', ''), $template->getAttribute('referenceType', ''), $templateDirectory, $templateRootDirectory), '', $stdout, $stderr) !== 0) { + throw new \Exception('Unable to clone template repository: ' . $stderr); + } + + Console::execute('mkdir -p ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr); + Console::execute('mkdir -p ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr); + + // Merge the template into the repo, then commit + push the branch. + Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr); + + $message = \escapeshellarg("Create '" . $function->getAttribute('name', '') . "' function"); + if (Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($repoDirectory) . ' && git checkout -b ' . \escapeshellarg($branch) . ' && git add . && git commit -m ' . $message . ' && git push origin ' . \escapeshellarg($branch), '', $stdout, $stderr) !== 0) { + throw new \Exception('Unable to push to repository: ' . $stderr); + } + + if (Console::execute('cd ' . \escapeshellarg($repoDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr) !== 0) { + throw new \Exception('Unable to resolve commit hash: ' . $stderr); + } + + return \trim($stdout); + } finally { + Console::execute('rm -rf ' . \escapeshellarg($tmpDirectory), '', $stdout, $stderr); + } + } + /** * Record a reported artifact size. Remote-source builds (templates / VCS) * fetch the source in the sidecar, so Appwrite can't size it the way the