Wire VCS function builds onto the jobs-service backend

Generalize Job::build's source to an optional remote tarball ({url, subdir})
so a VCS deployment can be built from a presigned GitHub tarball, alongside the
existing manual-upload/duplicate (presigned Appwrite) and template (public
codeload) paths.

- Base::redeployVcsFunction gains an optional Jobs client. When passed and
  _APP_BUILDS_BACKEND=orchestrator (function collection, non-template), the
  deployment is pre-declared 'waiting' with its buildPath and submitted to the
  jobs-service via getRepositoryPresignedUrl; otherwise it enqueues to the
  executor unchanged. Sites, Functions create/update, template-into-repo push
  and the VCS webhook stay on the executor (they pass no Jobs client).
- Vcs/Create injects and passes the Jobs client; Template/Create already builds
  its own source for the public path.
- Jobs worker reports the terminal outcome as a GitHub commit status
  (best-effort, skips non-VCS and silent-mode). "pending" at build start and
  PR comments remain follow-ups.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
loks0nandClaude Opus 4.8 committed 2026-07-10 16:46:02 +01:00
1 parent d1b1ab9fb0
commit 9cacea05d5
5 files changed
+121 -29

No files matched your search

+18 -13
View File
@@ -32,12 +32,18 @@ final class Job
/**
* @return array<string, mixed> Named arguments for OpenRuntimes\Orchestrator\Jobs::create().
*/
/**
* @param ?array{url: string, subdir?: string} $source Remote tarball source
* (template codeload URL, or a VCS presigned URL). When null, the source is
* the deployment's own uploaded tarball, fetched from Appwrite via a
* presigned token (manual upload / duplicate).
*/
public static function build(
Document $project,
Document $function,
Document $deployment,
array $platform,
?Document $template = null,
?array $source = null,
): array {
$projectId = $project->getId();
$deploymentId = $deployment->getId();
@@ -55,19 +61,18 @@ final class Job
$protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https';
$endpoint = System::getEnv('_APP_JOBS_ENDPOINT', "$protocol://{$platform['apiHostname']}");
// Source artifacts: manual/duplicate deployments download a presigned
// tarball from Appwrite; template deployments pull the public GitHub
// tarball directly (the unarchive auto-strips the "{repo}-{ref}/" wrapper
// and, via subdir, extracts just the template's rootDirectory). Either
// way the build input lands in /mnt/code/source.
if ($template !== null) {
$owner = $template->getAttribute('ownerName', '');
$repository = $template->getAttribute('repositoryName', '');
$reference = $template->getAttribute('referenceValue', '');
$rootDirectory = \trim($template->getAttribute('rootDirectory', ''), '/');
// Source artifacts, both ending in /mnt/code/source:
// - remote tarball ($source): templates (public codeload URL) and VCS
// (a short-lived presigned URL). The unarchive auto-strips the
// "{repo}-{ref}/" wrapper and, via subdir, extracts just the
// rootDirectory.
// - otherwise: the deployment's uploaded tarball, fetched from Appwrite
// over a presigned GET (manual upload / duplicate).
if ($source !== null) {
$subdir = \trim($source['subdir'] ?? '', '/');
$sourceArtifacts = [
new DownloadArtifact(id: 'source', in: "https://codeload.github.com/{$owner}/{$repository}/tar.gz/{$reference}", out: 'source.tar.gz'),
new UnarchiveArtifact(id: 'extract', in: 'source.tar.gz', out: 'source', subdir: $rootDirectory !== '' ? $rootDirectory : null),
new DownloadArtifact(id: 'source', in: $source['url'], out: 'source.tar.gz'),
new UnarchiveArtifact(id: 'extract', in: 'source.tar.gz', out: 'source', subdir: $subdir !== '' ? $subdir : null),
];
} else {
// Presigned source-download URL (GET, no request-body cap), fetched by
+34 -9
View File
@@ -2,6 +2,7 @@
namespace Appwrite\Platform\Modules\Compute;
use Appwrite\Compute\Job;
use Appwrite\Event\Message\Build as BuildMessage;
use Appwrite\Event\Publisher\Build as BuildPublisher;
use Appwrite\Extend\Exception;
@@ -9,6 +10,7 @@ use Appwrite\Filter\BranchDomain as BranchDomainFilter;
use Appwrite\Platform\Action;
use Appwrite\Platform\Modules\Compute\Validator\Specification as SpecificationValidator;
use Appwrite\Platform\Permission as AppwritePermission;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
@@ -67,7 +69,7 @@ class Base extends Action
return $allowedSpecifications[0];
}
public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = ''): Document
public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null): Document
{
$deploymentId = ID::unique();
$entrypoint = $function->getAttribute('entrypoint', '');
@@ -118,6 +120,19 @@ class Base extends Action
$repositoryUrl = "https://github.com/$owner/$repositoryName";
// Build a plain (non-template) VCS function deployment on the jobs-service
// when the caller opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator.
// Sites and the template-into-repo push stay on the executor. When on
// jobs the deployment is pre-declared 'waiting' with its buildPath so
// build.sh writes output onto the mounted volume.
$useJobs = $jobs !== null
&& $function->getCollection() === 'functions'
&& $template->isEmpty()
&& System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
$buildFields = $useJobs
? ['status' => 'waiting', 'buildPath' => Job::buildPath($project->getId(), $deploymentId)]
: [];
$deployment = $dbForProject->createDocument('deployments', new Document([
'$id' => $deploymentId,
'$permissions' => [
@@ -149,16 +164,26 @@ class Base extends Action
'providerBranch' => $providerBranch,
'providerRootDirectory' => $function->getAttribute('providerRootDirectory', ''),
'activate' => $activate,
...$buildFields,
]));
$publisherForBuilds->enqueue(new BuildMessage(
project: $project,
resource: $function,
deployment: $deployment,
type: BUILD_TYPE_DEPLOYMENT,
template: $template,
platform: $platform,
));
if ($useJobs) {
$ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch');
$source = [
'url' => $github->getRepositoryPresignedUrl($owner, $repositoryName, $ref),
'subdir' => $function->getAttribute('providerRootDirectory', ''),
];
$jobs->create(...Job::build($project, $function, $deployment, $platform, $source));
} else {
$publisherForBuilds->enqueue(new BuildMessage(
project: $project,
resource: $function,
deployment: $deployment,
type: BUILD_TYPE_DEPLOYMENT,
template: $template,
platform: $platform,
));
}
return $deployment;
}
@@ -207,7 +207,13 @@ class Create extends Base
$this->updateEmptyManualRule($project, $function, $deployment, $dbForPlatform, $authorization);
if ($useJobs) {
$jobs->create(...Job::build($project, $function, $deployment, $platform, $template));
// Public template: pull the source straight from GitHub's codeload
// tarball; unarchive strips the "{repo}-{ref}/" wrapper + the rootDirectory.
$source = [
'url' => "https://codeload.github.com/{$owner}/{$repository}/tar.gz/{$reference}",
'subdir' => $rootDirectory,
];
$jobs->create(...Job::build($project, $function, $deployment, $platform, $source));
} else {
$publisherForBuilds->enqueue(new BuildMessage(
project: $project,
@@ -11,6 +11,7 @@ use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Response;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Validator\UID;
@@ -73,6 +74,7 @@ class Create extends Base
->inject('project')
->inject('queueForEvents')
->inject('publisherForBuilds')
->inject('jobs')
->inject('gitHub')
->inject('platform')
->callback($this->action(...));
@@ -90,6 +92,7 @@ class Create extends Base
Document $project,
Event $queueForEvents,
BuildPublisher $publisherForBuilds,
Jobs $jobs,
GitHub $github,
array $platform,
) {
@@ -119,7 +122,8 @@ class Create extends Base
activate: $activate,
platform: $platform,
reference: $reference,
referenceType: $type
referenceType: $type,
jobs: $jobs
);
$queueForEvents
@@ -22,6 +22,7 @@ use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\Storage\Device;
use Utopia\System\System;
use Utopia\VCS\Adapter\Git\GitHub;
/**
* Applies open-runtimes jobs-service callbacks to a deployment. The API
@@ -63,6 +64,7 @@ class Jobs extends Action
->inject('publisherForUsage')
->inject('usage')
->inject('deviceForBuilds')
->inject('gitHub')
->inject('cache')
->inject('locks')
->callback($this->action(...));
@@ -80,6 +82,7 @@ class Jobs extends Action
UsagePublisher $publisherForUsage,
UsageContext $usage,
Device $deviceForBuilds,
GitHub $github,
Cache $cache,
callable $locks,
): void {
@@ -90,7 +93,7 @@ class Jobs extends Action
return;
}
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $cache, $deploymentId): void {
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $github, $cache, $deploymentId): void {
if ($event->id !== '') {
$key = 'jobs-event-' . $event->id;
if ($cache->load($key, self::DEDUPE_TTL) !== false) {
@@ -110,7 +113,7 @@ class Jobs extends Action
$deployment = match ($event->event) {
'orchestrator.job.log' => $this->onLog($dbForProject, $deployment, $event->data),
'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds),
'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds, $github),
default => $deployment,
};
@@ -168,9 +171,10 @@ class Jobs extends Action
UsageContext $usage,
UsagePublisher $publisherForUsage,
Device $deviceForBuilds,
GitHub $github,
): Document {
if ($exitCode !== 0) {
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage);
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $github);
}
$path = Job::buildPath($project->getId(), $deployment->getId());
@@ -179,10 +183,10 @@ class Jobs extends Action
$limit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000');
if ($limit !== 0 && $size > $limit) {
$deviceForBuilds->delete($path);
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build size should be less than ' . \number_format($limit / (1000 * 1000), 2) . ' MBs.', $usage, $publisherForUsage);
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build size should be less than ' . \number_format($limit / (1000 * 1000), 2) . ' MBs.', $usage, $publisherForUsage, $github);
}
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, true, '', $usage, $publisherForUsage, $size);
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, true, '', $usage, $publisherForUsage, $github, $size);
}
/**
@@ -199,6 +203,7 @@ class Jobs extends Action
string $message,
UsageContext $usage,
UsagePublisher $publisherForUsage,
GitHub $github,
int $buildSize = 0,
): Document {
$function = $dbForProject->getDocument('functions', $deployment->getAttribute('resourceId'));
@@ -244,9 +249,56 @@ class Jobs extends Action
$this->updateLatestDeployment($dbForProject, $function);
}
// Report the terminal outcome as a VCS commit status (jobs-built VCS
// deployments only; best-effort). "pending" at build start is a follow-up.
$status = $deployment->getAttribute('status');
if (\in_array($status, ['ready', 'failed'], true) && ! $function->isEmpty()) {
$this->gitStatus($github, $dbForPlatform, $project, $function, $deployment, $status === 'ready' ? 'success' : 'failure');
}
return $deployment;
}
/**
* Post a GitHub commit status for a VCS deployment (no-op for non-VCS builds
* or silent mode). Best-effort — a failed status update never fails the build.
*/
private function gitStatus(GitHub $github, Database $dbForPlatform, Document $project, Document $function, Document $deployment, string $state): void
{
$commitHash = $deployment->getAttribute('providerCommitHash', '');
if ($commitHash === '' || $function->getAttribute('providerSilentMode', false) === true) {
return;
}
try {
$installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId', ''));
$providerInstallationId = $installation->getAttribute('providerInstallationId', '');
if ($providerInstallationId === '') {
return;
}
$github->initializeVariables($providerInstallationId, System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY', ''), System::getEnv('_APP_VCS_GITHUB_APP_ID', ''));
$protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https';
$hostname = System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN', ''));
$region = $project->getAttribute('region', 'default');
$targetUrl = "{$protocol}://{$hostname}/console/project-{$region}-{$project->getId()}/functions/function-{$function->getId()}";
$message = $state === 'success' ? 'Build succeeded.' : 'Build failed.';
$name = $function->getAttribute('name', '') . ' (' . $project->getAttribute('name', '') . ')';
$github->updateCommitStatus(
$deployment->getAttribute('providerRepositoryName', ''),
$commitHash,
$deployment->getAttribute('providerRepositoryOwner', ''),
$state,
$message,
$targetUrl,
$name,
);
} catch (\Throwable) {
// Best-effort; the build outcome stands regardless of the status update.
}
}
/**
* Point the function at this deployment (auto-activate). Mirrors the
* essential function activation from the Builds worker.