Do the template-into-repo git write in the API endpoint for jobs

The VCS-connected template flow (deploy a template onto a repo-linked function)
is a git write — clone + merge + commit + push — which the jobs-service artifact
system can't do. On _APP_BUILDS_BACKEND=orchestrator, replicate the executor
Builds worker's clone/rsync/commit/push directly in Template/Create (git binary,
in the request flow), then build the resulting commit via the jobs-capable VCS
path (redeployVcsFunction with an empty template + the commit ref). The executor
path is unchanged (its worker still clones/merges/pushes/builds in the job).

This is the last function build path pinned to the executor; with it, every
function trigger (manual, duplicate, template, VCS manual/webhook/connect-time,
and template-into-repo) builds on jobs when orchestrator is selected.

Note: runs blocking git clone/push inside the Swoole HTTP request (the executor
does it off-request); acceptable for this low-frequency path but flagged.
Static-checked only — VCS isn't verifiable locally.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
loks0nandClaude Opus 4.8 committed 2026-07-10 16:46:56 +01:00
1 parent 5a4b2fa19c
commit f8f157414c
1 file changed
+107 -30
@@ -14,6 +14,7 @@ use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Response;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
@@ -130,38 +131,49 @@ class Create extends Base
if (!empty($function->getAttribute('providerRepositoryId'))) {
// VCS-connected function: the template is merged into the user's repo
// and pushed as a commit, then that commit is built. This stays on the
// executor for now because it is a git *write*, which the jobs-service
// artifact system (download/unarchive) does not cover.
//
// Plan to move it to the jobs backend (3a — Git Data API push, no git
// binary), when utopia-php/vcs grows the bulk-push methods:
// 1. download the template tarball (codeload) + read the tree of the
// target repo/branch (github->getRepositoryTree).
// 2. for each template file: create a blob (POST .../git/blobs).
// 3. create a tree from the blobs on top of the branch's base tree
// (POST .../git/trees), then a commit (POST .../git/commits), then
// fast-forward the branch ref (PATCH .../git/refs/heads/{branch}).
// 4. take the resulting commit hash and submit the build via
// Job::build (same authenticated-tarball path as createVcsDeployment).
// This replaces clone + rsync + `git commit && git push` with ~4 API
// calls and no working tree. Until then, executor handles this branch.
// and pushed as a commit, then that commit is built.
$installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId'));
$deployment = $this->redeployVcsFunction(
request: $request,
function: $function,
project: $project,
installation: $installation,
dbForProject: $dbForProject,
publisherForBuilds: $publisherForBuilds,
template: $template,
github: $github,
activate: $activate,
platform: $platform,
referenceType: $type,
reference: $reference
);
if (System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator') {
// The jobs-service artifact system only reads source (download /
// unarchive), so it cannot do the git *write*. Perform the merge +
// commit + push here (mirroring the executor Builds worker), then
// build the resulting commit on the jobs-service like any other VCS
// commit — no template merge needed in the build.
$commitHash = $this->pushTemplateToRepository($github, $function, $installation, $owner, $repository, $reference, $type, $rootDirectory);
$deployment = $this->redeployVcsFunction(
request: $request,
function: $function,
project: $project,
installation: $installation,
dbForProject: $dbForProject,
publisherForBuilds: $publisherForBuilds,
template: new Document(),
github: $github,
activate: $activate,
platform: $platform,
referenceType: 'commit',
reference: $commitHash,
jobs: $jobs
);
} else {
// Executor: the Builds worker clones the repo, merges the template,
// pushes the commit and builds it, all in the build job.
$deployment = $this->redeployVcsFunction(
request: $request,
function: $function,
project: $project,
installation: $installation,
dbForProject: $dbForProject,
publisherForBuilds: $publisherForBuilds,
template: $template,
github: $github,
activate: $activate,
platform: $platform,
referenceType: $type,
reference: $reference
);
}
$queueForEvents
->setParam('functionId', $function->getId())
@@ -246,4 +258,69 @@ class Create extends Base
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
->dynamic($deployment, Response::MODEL_DEPLOYMENT);
}
/**
* Merge a template into a VCS-connected function's repository and push it as a
* commit, returning the new commit hash. Replicates the executor Builds
* worker's clone + rsync + commit + push for the jobs backend, which has no
* git-write primitive; the resulting commit is then built like any other VCS
* commit. Runs the git binary in the request flow.
*/
private function pushTemplateToRepository(
GitHub $github,
Document $function,
Document $installation,
string $templateOwner,
string $templateRepository,
string $templateReference,
string $templateType,
string $templateRootDirectory,
): string {
$providerInstallationId = $installation->getAttribute('providerInstallationId', '');
if (empty($providerInstallationId)) {
throw new Exception(Exception::INSTALLATION_NOT_FOUND);
}
$github->initializeVariables($providerInstallationId, System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY', ''), System::getEnv('_APP_VCS_GITHUB_APP_ID', ''));
$owner = $github->getOwnerName($providerInstallationId);
$repository = $github->getRepositoryName($function->getAttribute('providerRepositoryId', ''));
$branch = $function->getAttribute('providerBranch', 'main');
$rootDirectory = \ltrim(\ltrim(\rtrim($function->getAttribute('providerRootDirectory', ''), '/'), '.'), '/');
$templateRootDirectory = \ltrim(\ltrim(\rtrim($templateRootDirectory, '/'), '.'), '/');
$id = ID::unique();
$repoDirectory = '/tmp/templates/' . $id . '/code';
$templateDirectory = '/tmp/templates/' . $id . '/template';
$stdout = '';
$stderr = '';
try {
if (Console::execute($github->generateCloneCommand($owner, $repository, $branch, GitHub::CLONE_TYPE_BRANCH, $repoDirectory, $rootDirectory), '', $stdout, $stderr) !== 0) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unable to clone repository: ' . $stderr);
}
if (Console::execute($github->generateCloneCommand($templateOwner, $templateRepository, $templateReference, $templateType, $templateDirectory, $templateRootDirectory), '', $stdout, $stderr) !== 0) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unable to clone template repository: ' . $stderr);
}
Console::execute('mkdir -p ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr);
Console::execute('mkdir -p ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr);
// Merge the template into the repo, then commit + push the branch.
Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($templateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($repoDirectory . '/' . $rootDirectory), '', $stdout, $stderr);
$message = \escapeshellarg("Create '" . $function->getAttribute('name', '') . "' function");
if (Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($repoDirectory) . ' && git checkout -b ' . \escapeshellarg($branch) . ' && git add . && git commit -m ' . $message . ' && git push origin ' . \escapeshellarg($branch), '', $stdout, $stderr) !== 0) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unable to push to repository: ' . $stderr);
}
if (Console::execute('cd ' . \escapeshellarg($repoDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr) !== 0) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unable to resolve commit hash: ' . $stderr);
}
return \trim($stdout);
} finally {
Console::execute('rm -rf ' . \escapeshellarg('/tmp/templates/' . $id), '', $stdout, $stderr);
}
}
}