Move template-into-repo git push from the API endpoint to the jobs worker

The Template/Create endpoint no longer runs git in the Swoole request flow.
On the orchestrator backend, redeployVcsFunction enqueues an
appwrite.template.push message to the v1-jobs queue; the jobs worker clones
the repo, merges the template, commits + pushes (mirroring the executor
Builds worker), records the commit on the deployment, then submits the build
job for that commit. A failed push finalizes the deployment as failed with
the usual fan-out. The endpoint's orchestrator/executor branch collapses
into a single redeployVcsFunction call.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
loks0nandClaude Fable 5 committed 2026-07-10 16:46:56 +01:00
1 parent f8f157414c
commit 0c075d843e
3 files changed
+158 -118

No files matched your search

+24 -8
View File
@@ -4,7 +4,9 @@ namespace Appwrite\Platform\Modules\Compute;
use Appwrite\Compute\Job;
use Appwrite\Event\Message\Build as BuildMessage;
use Appwrite\Event\Message\Jobs as JobsMessage;
use Appwrite\Event\Publisher\Build as BuildPublisher;
use Appwrite\Event\Publisher\Jobs as JobsPublisher;
use Appwrite\Extend\Exception;
use Appwrite\Filter\BranchDomain as BranchDomainFilter;
use Appwrite\Platform\Action;
@@ -69,7 +71,7 @@ class Base extends Action
return $allowedSpecifications[0];
}
public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null): Document
public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null, ?JobsPublisher $publisherForJobs = null): Document
{
$deploymentId = ID::unique();
$entrypoint = $function->getAttribute('entrypoint', '');
@@ -120,14 +122,13 @@ class Base extends Action
$repositoryUrl = "https://github.com/$owner/$repositoryName";
// Build a plain (non-template) VCS function deployment on the jobs-service
// when the caller opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator.
// Sites and the template-into-repo push stay on the executor. When on
// jobs the deployment is pre-declared 'waiting' with its buildPath so
// build.sh writes output onto the mounted volume.
// Build a VCS function deployment on the jobs-service when the caller
// opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator. Sites stay on
// the executor. When on jobs the deployment is pre-declared 'waiting'
// with its buildPath so build.sh writes output onto the mounted volume.
$useJobs = $jobs !== null
&& $function->getCollection() === 'functions'
&& $template->isEmpty()
&& ($template->isEmpty() || $publisherForJobs !== null)
&& System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
$buildFields = $useJobs
? ['status' => 'waiting', 'buildPath' => Job::buildPath($project->getId(), $deploymentId)]
@@ -167,7 +168,22 @@ class Base extends Action
...$buildFields,
]));
if ($useJobs) {
if ($useJobs && ! $template->isEmpty()) {
// The template must be merged into the repo and pushed as a commit
// before the build — a git *write* the jobs-service artifact system
// has no primitive for. Hand off to the jobs worker, which pushes
// the template and then submits the build for the resulting commit.
$publisherForJobs->enqueue(new JobsMessage(
project: $project,
id: 'template-push-' . $deployment->getId(),
event: 'appwrite.template.push',
data: [
'meta' => ['projectId' => $project->getId(), 'deploymentId' => $deployment->getId()],
'template' => $template->getArrayCopy(),
'platform' => $platform,
],
));
} elseif ($useJobs) {
$ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch');
$source = [
'url' => $github->getRepositoryPresignedUrl($owner, $repositoryName, $ref),
@@ -6,6 +6,7 @@ use Appwrite\Compute\Job;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Build as BuildMessage;
use Appwrite\Event\Publisher\Build as BuildPublisher;
use Appwrite\Event\Publisher\Jobs as JobsPublisher;
use Appwrite\Extend\Exception;
use Appwrite\Platform\Action;
use Appwrite\Platform\Modules\Compute\Base;
@@ -14,7 +15,6 @@ use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Response;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
@@ -84,6 +84,7 @@ class Create extends Base
->inject('queueForEvents')
->inject('project')
->inject('publisherForBuilds')
->inject('publisherForJobs')
->inject('jobs')
->inject('gitHub')
->inject('authorization')
@@ -106,6 +107,7 @@ class Create extends Base
Event $queueForEvents,
Document $project,
BuildPublisher $publisherForBuilds,
JobsPublisher $publisherForJobs,
Jobs $jobs,
GitHub $github,
Authorization $authorization,
@@ -131,49 +133,27 @@ class Create extends Base
if (!empty($function->getAttribute('providerRepositoryId'))) {
// VCS-connected function: the template is merged into the user's repo
// and pushed as a commit, then that commit is built.
// and pushed as a commit, then that commit is built. The executor does
// the push inside the build job; the jobs backend does it in the jobs
// worker (which then submits the build for the resulting commit).
$installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId'));
if (System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator') {
// The jobs-service artifact system only reads source (download /
// unarchive), so it cannot do the git *write*. Perform the merge +
// commit + push here (mirroring the executor Builds worker), then
// build the resulting commit on the jobs-service like any other VCS
// commit — no template merge needed in the build.
$commitHash = $this->pushTemplateToRepository($github, $function, $installation, $owner, $repository, $reference, $type, $rootDirectory);
$deployment = $this->redeployVcsFunction(
request: $request,
function: $function,
project: $project,
installation: $installation,
dbForProject: $dbForProject,
publisherForBuilds: $publisherForBuilds,
template: new Document(),
github: $github,
activate: $activate,
platform: $platform,
referenceType: 'commit',
reference: $commitHash,
jobs: $jobs
);
} else {
// Executor: the Builds worker clones the repo, merges the template,
// pushes the commit and builds it, all in the build job.
$deployment = $this->redeployVcsFunction(
request: $request,
function: $function,
project: $project,
installation: $installation,
dbForProject: $dbForProject,
publisherForBuilds: $publisherForBuilds,
template: $template,
github: $github,
activate: $activate,
platform: $platform,
referenceType: $type,
reference: $reference
);
}
$deployment = $this->redeployVcsFunction(
request: $request,
function: $function,
project: $project,
installation: $installation,
dbForProject: $dbForProject,
publisherForBuilds: $publisherForBuilds,
template: $template,
github: $github,
activate: $activate,
platform: $platform,
referenceType: $type,
reference: $reference,
jobs: $jobs,
publisherForJobs: $publisherForJobs
);
$queueForEvents
->setParam('functionId', $function->getId())
@@ -258,69 +238,4 @@ class Create extends Base
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
->dynamic($deployment, Response::MODEL_DEPLOYMENT);
}
/**
* Merge a template into a VCS-connected function's repository and push it as a
* commit, returning the new commit hash. Replicates the executor Builds
* worker's clone + rsync + commit + push for the jobs backend, which has no
* git-write primitive; the resulting commit is then built like any other VCS
* commit. Runs the git binary in the request flow.
*/
private function pushTemplateToRepository(
GitHub $github,
Document $function,
Document $installation,
string $templateOwner,
string $templateRepository,
string $templateReference,
string $templateType,
string $templateRootDirectory,
): string {
$providerInstallationId = $installation->getAttribute('providerInstallationId', '');
if (empty($providerInstallationId)) {
throw new Exception(Exception::INSTALLATION_NOT_FOUND);
}
$github->initializeVariables($providerInstallationId, System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY', ''), System::getEnv('_APP_VCS_GITHUB_APP_ID', ''));
$owner = $github->getOwnerName($providerInstallationId);
$repository = $github->getRepositoryName($function->getAttribute('providerRepositoryId', ''));
$branch = $function->getAttribute('providerBranch', 'main');
$rootDirectory = \ltrim(\ltrim(\rtrim($function->getAttribute('providerRootDirectory', ''), '/'), '.'), '/');
$templateRootDirectory = \ltrim(\ltrim(\rtrim($templateRootDirectory, '/'), '.'), '/');
$id = ID::unique();
$repoDirectory = '/tmp/templates/' . $id . '/code';
$templateDirectory = '/tmp/templates/' . $id . '/template';
$stdout = '';
$stderr = '';
try {
if (Console::execute($github->generateCloneCommand($owner, $repository, $branch, GitHub::CLONE_TYPE_BRANCH, $repoDirectory, $rootDirectory), '', $stdout, $stderr) !== 0) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unable to clone repository: ' . $stderr);
}
if (Console::execute($github->generateCloneCommand($templateOwner, $templateRepository, $templateReference, $templateType, $templateDirectory, $templateRootDirectory), '', $stdout, $stderr) !== 0) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unable to clone template repository: ' . $stderr);
}
Console::execute('mkdir -p ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr);
Console::execute('mkdir -p ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr);
// Merge the template into the repo, then commit + push the branch.
Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr);
$message = \escapeshellarg("Create '" . $function->getAttribute('name', '') . "' function");
if (Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($repoDirectory) . ' && git checkout -b ' . \escapeshellarg($branch) . ' && git add . && git commit -m ' . $message . ' && git push origin ' . \escapeshellarg($branch), '', $stdout, $stderr) !== 0) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unable to push to repository: ' . $stderr);
}
if (Console::execute('cd ' . \escapeshellarg($repoDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr) !== 0) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unable to resolve commit hash: ' . $stderr);
}
return \trim($stdout);
} finally {
Console::execute('rm -rf ' . \escapeshellarg('/tmp/templates/' . $id), '', $stdout, $stderr);
}
}
}
@@ -13,7 +13,9 @@ use Appwrite\Event\Webhook;
use Appwrite\Usage\Build as BuildUsage;
use Appwrite\Usage\Context as UsageContext;
use Appwrite\Utopia\Response\Model\Deployment;
use OpenRuntimes\Orchestrator\Jobs as JobsClient;
use Utopia\Cache\Cache;
use Utopia\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
@@ -64,6 +66,7 @@ class Jobs extends Action
->inject('publisherForUsage')
->inject('usage')
->inject('deviceForBuilds')
->inject('jobs')
->inject('gitHub')
->inject('cache')
->inject('locks')
@@ -82,6 +85,7 @@ class Jobs extends Action
UsagePublisher $publisherForUsage,
UsageContext $usage,
Device $deviceForBuilds,
JobsClient $jobs,
GitHub $github,
Cache $cache,
callable $locks,
@@ -93,7 +97,7 @@ class Jobs extends Action
return;
}
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $github, $cache, $deploymentId): void {
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $jobs, $github, $cache, $deploymentId): void {
if ($event->id !== '') {
$key = 'jobs-event-' . $event->id;
if ($cache->load($key, self::DEDUPE_TTL) !== false) {
@@ -108,10 +112,13 @@ class Jobs extends Action
}
// The build writes its output onto the mounted volume before build.sh
// exits, so the exit callback is a truthful completion signal.
$finalizing = $event->event === 'orchestrator.job.exit';
// exits, so the exit callback is a truthful completion signal. A
// template push only finalizes when it fails (success just submits
// the build job, whose own exit finalizes later).
$finalizing = \in_array($event->event, ['orchestrator.job.exit', 'appwrite.template.push'], true);
$deployment = match ($event->event) {
'appwrite.template.push' => $this->onTemplatePush($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $jobs, $github, $usage, $publisherForUsage),
'orchestrator.job.log' => $this->onLog($dbForProject, $deployment, $event->data),
'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $deployment, $event->data),
'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds, $github),
@@ -158,6 +165,108 @@ class Jobs extends Action
return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($update));
}
/**
* Merge a template into a VCS-connected function's repository, push it as a
* commit, and submit the build job for that commit. This is the one VCS
* flow needing a git *write*, which the jobs-service artifact system has no
* primitive for — so it runs here (mirroring the executor Builds worker)
* before the build is handed to the jobs-service. A failed push finalizes
* the deployment as failed; the build's own exit callback finalizes success.
*/
private function onTemplatePush(
Database $dbForProject,
Database $dbForPlatform,
Document $project,
Document $deployment,
array $data,
JobsClient $jobs,
GitHub $github,
UsageContext $usage,
UsagePublisher $publisherForUsage,
): Document {
$function = $dbForProject->getDocument('functions', $deployment->getAttribute('resourceId'));
try {
$installation = $dbForPlatform->getDocument('installations', $deployment->getAttribute('installationId', ''));
$providerInstallationId = $installation->getAttribute('providerInstallationId', '');
if ($function->isEmpty() || $providerInstallationId === '') {
throw new \Exception('Function or installation not found.');
}
$github->initializeVariables($providerInstallationId, System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY', ''), System::getEnv('_APP_VCS_GITHUB_APP_ID', ''));
$commitHash = $this->pushTemplate($github, $function, $deployment, new Document($data['template'] ?? []));
$owner = $deployment->getAttribute('providerRepositoryOwner', '');
$repository = $deployment->getAttribute('providerRepositoryName', '');
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
'providerCommitHash' => $commitHash,
'providerCommitAuthorUrl' => APP_VCS_GITHUB_URL,
'providerCommitAuthor' => APP_VCS_GITHUB_USERNAME,
'providerCommitMessage' => "Create '" . $function->getAttribute('name', '') . "' function",
'providerCommitUrl' => "https://github.com/{$owner}/{$repository}/commit/{$commitHash}",
]));
$source = [
'url' => $github->getRepositoryPresignedUrl($owner, $repository, $commitHash),
'subdir' => $function->getAttribute('providerRootDirectory', ''),
];
$jobs->create(...Job::build($project, $function, $deployment, $data['platform'] ?? [], $source));
return $deployment;
} catch (\Throwable $error) {
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, $error->getMessage(), $usage, $publisherForUsage, $github);
}
}
/**
* Clone the function's repository, merge the template into it and push the
* result, returning the new commit hash. Replicates the executor Builds
* worker's clone + rsync + commit + push.
*/
private function pushTemplate(GitHub $github, Document $function, Document $deployment, Document $template): string
{
$owner = $deployment->getAttribute('providerRepositoryOwner', '');
$repository = $deployment->getAttribute('providerRepositoryName', '');
$branch = $deployment->getAttribute('providerBranch') ?: $function->getAttribute('providerBranch', 'main');
$rootDirectory = \ltrim(\ltrim(\rtrim($function->getAttribute('providerRootDirectory', ''), '/'), '.'), '/');
$templateRootDirectory = \ltrim(\ltrim(\rtrim($template->getAttribute('rootDirectory', ''), '/'), '.'), '/');
$tmpDirectory = '/tmp/builds/' . $deployment->getId();
$repoDirectory = $tmpDirectory . '/code';
$templateDirectory = $tmpDirectory . '/template';
$stdout = '';
$stderr = '';
try {
if (Console::execute($github->generateCloneCommand($owner, $repository, $branch, GitHub::CLONE_TYPE_BRANCH, $repoDirectory, $rootDirectory), '', $stdout, $stderr) !== 0) {
throw new \Exception('Unable to clone repository: ' . $stderr);
}
if (Console::execute($github->generateCloneCommand($template->getAttribute('ownerName', ''), $template->getAttribute('repositoryName', ''), $template->getAttribute('referenceValue', ''), $template->getAttribute('referenceType', ''), $templateDirectory, $templateRootDirectory), '', $stdout, $stderr) !== 0) {
throw new \Exception('Unable to clone template repository: ' . $stderr);
}
Console::execute('mkdir -p ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr);
Console::execute('mkdir -p ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr);
// Merge the template into the repo, then commit + push the branch.
Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr);
$message = \escapeshellarg("Create '" . $function->getAttribute('name', '') . "' function");
if (Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($repoDirectory) . ' && git checkout -b ' . \escapeshellarg($branch) . ' && git add . && git commit -m ' . $message . ' && git push origin ' . \escapeshellarg($branch), '', $stdout, $stderr) !== 0) {
throw new \Exception('Unable to push to repository: ' . $stderr);
}
if (Console::execute('cd ' . \escapeshellarg($repoDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr) !== 0) {
throw new \Exception('Unable to resolve commit hash: ' . $stderr);
}
return \trim($stdout);
} finally {
Console::execute('rm -rf ' . \escapeshellarg($tmpDirectory), '', $stdout, $stderr);
}
}
/**
* Record a reported artifact size. Remote-source builds (templates / VCS)
* fetch the source in the sidecar, so Appwrite can't size it the way the