mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Wire VCS function builds onto the jobs-service backend
Generalize Job::build's source to an optional remote tarball ({url, subdir})
so a VCS deployment can be built from a presigned GitHub tarball, alongside the
existing manual-upload/duplicate (presigned Appwrite) and template (public
codeload) paths.
- Base::redeployVcsFunction gains an optional Jobs client. When passed and
_APP_BUILDS_BACKEND=orchestrator (function collection, non-template), the
deployment is pre-declared 'waiting' with its buildPath and submitted to the
jobs-service via getRepositoryPresignedUrl; otherwise it enqueues to the
executor unchanged. Sites, Functions create/update, template-into-repo push
and the VCS webhook stay on the executor (they pass no Jobs client).
- Vcs/Create injects and passes the Jobs client; Template/Create already builds
its own source for the public path.
- Jobs worker reports the terminal outcome as a GitHub commit status
(best-effort, skips non-VCS and silent-mode). "pending" at build start and
PR comments remain follow-ups.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d1b1ab9fb0
commit
9cacea05d5
5 files changed
+121
-29
No files matched your search
@@ -32,12 +32,18 @@ final class Job
|
||||
/**
|
||||
* @return array<string, mixed> Named arguments for OpenRuntimes\Orchestrator\Jobs::create().
|
||||
*/
|
||||
/**
|
||||
* @param ?array{url: string, subdir?: string} $source Remote tarball source
|
||||
* (template codeload URL, or a VCS presigned URL). When null, the source is
|
||||
* the deployment's own uploaded tarball, fetched from Appwrite via a
|
||||
* presigned token (manual upload / duplicate).
|
||||
*/
|
||||
public static function build(
|
||||
Document $project,
|
||||
Document $function,
|
||||
Document $deployment,
|
||||
array $platform,
|
||||
?Document $template = null,
|
||||
?array $source = null,
|
||||
): array {
|
||||
$projectId = $project->getId();
|
||||
$deploymentId = $deployment->getId();
|
||||
@@ -55,19 +61,18 @@ final class Job
|
||||
$protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https';
|
||||
$endpoint = System::getEnv('_APP_JOBS_ENDPOINT', "$protocol://{$platform['apiHostname']}");
|
||||
|
||||
// Source artifacts: manual/duplicate deployments download a presigned
|
||||
// tarball from Appwrite; template deployments pull the public GitHub
|
||||
// tarball directly (the unarchive auto-strips the "{repo}-{ref}/" wrapper
|
||||
// and, via subdir, extracts just the template's rootDirectory). Either
|
||||
// way the build input lands in /mnt/code/source.
|
||||
if ($template !== null) {
|
||||
$owner = $template->getAttribute('ownerName', '');
|
||||
$repository = $template->getAttribute('repositoryName', '');
|
||||
$reference = $template->getAttribute('referenceValue', '');
|
||||
$rootDirectory = \trim($template->getAttribute('rootDirectory', ''), '/');
|
||||
// Source artifacts, both ending in /mnt/code/source:
|
||||
// - remote tarball ($source): templates (public codeload URL) and VCS
|
||||
// (a short-lived presigned URL). The unarchive auto-strips the
|
||||
// "{repo}-{ref}/" wrapper and, via subdir, extracts just the
|
||||
// rootDirectory.
|
||||
// - otherwise: the deployment's uploaded tarball, fetched from Appwrite
|
||||
// over a presigned GET (manual upload / duplicate).
|
||||
if ($source !== null) {
|
||||
$subdir = \trim($source['subdir'] ?? '', '/');
|
||||
$sourceArtifacts = [
|
||||
new DownloadArtifact(id: 'source', in: "https://codeload.github.com/{$owner}/{$repository}/tar.gz/{$reference}", out: 'source.tar.gz'),
|
||||
new UnarchiveArtifact(id: 'extract', in: 'source.tar.gz', out: 'source', subdir: $rootDirectory !== '' ? $rootDirectory : null),
|
||||
new DownloadArtifact(id: 'source', in: $source['url'], out: 'source.tar.gz'),
|
||||
new UnarchiveArtifact(id: 'extract', in: 'source.tar.gz', out: 'source', subdir: $subdir !== '' ? $subdir : null),
|
||||
];
|
||||
} else {
|
||||
// Presigned source-download URL (GET, no request-body cap), fetched by
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Compute;
|
||||
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Message\Build as BuildMessage;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
use Appwrite\Extend\Exception;
|
||||
@@ -9,6 +10,7 @@ use Appwrite\Filter\BranchDomain as BranchDomainFilter;
|
||||
use Appwrite\Platform\Action;
|
||||
use Appwrite\Platform\Modules\Compute\Validator\Specification as SpecificationValidator;
|
||||
use Appwrite\Platform\Permission as AppwritePermission;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
@@ -67,7 +69,7 @@ class Base extends Action
|
||||
return $allowedSpecifications[0];
|
||||
}
|
||||
|
||||
public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = ''): Document
|
||||
public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null): Document
|
||||
{
|
||||
$deploymentId = ID::unique();
|
||||
$entrypoint = $function->getAttribute('entrypoint', '');
|
||||
@@ -118,6 +120,19 @@ class Base extends Action
|
||||
|
||||
$repositoryUrl = "https://github.com/$owner/$repositoryName";
|
||||
|
||||
// Build a plain (non-template) VCS function deployment on the jobs-service
|
||||
// when the caller opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator.
|
||||
// Sites and the template-into-repo push stay on the executor. When on
|
||||
// jobs the deployment is pre-declared 'waiting' with its buildPath so
|
||||
// build.sh writes output onto the mounted volume.
|
||||
$useJobs = $jobs !== null
|
||||
&& $function->getCollection() === 'functions'
|
||||
&& $template->isEmpty()
|
||||
&& System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
|
||||
$buildFields = $useJobs
|
||||
? ['status' => 'waiting', 'buildPath' => Job::buildPath($project->getId(), $deploymentId)]
|
||||
: [];
|
||||
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
@@ -149,16 +164,26 @@ class Base extends Action
|
||||
'providerBranch' => $providerBranch,
|
||||
'providerRootDirectory' => $function->getAttribute('providerRootDirectory', ''),
|
||||
'activate' => $activate,
|
||||
...$buildFields,
|
||||
]));
|
||||
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
template: $template,
|
||||
platform: $platform,
|
||||
));
|
||||
if ($useJobs) {
|
||||
$ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch');
|
||||
$source = [
|
||||
'url' => $github->getRepositoryPresignedUrl($owner, $repositoryName, $ref),
|
||||
'subdir' => $function->getAttribute('providerRootDirectory', ''),
|
||||
];
|
||||
$jobs->create(...Job::build($project, $function, $deployment, $platform, $source));
|
||||
} else {
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
template: $template,
|
||||
platform: $platform,
|
||||
));
|
||||
}
|
||||
|
||||
return $deployment;
|
||||
}
|
||||
|
||||
@@ -207,7 +207,13 @@ class Create extends Base
|
||||
$this->updateEmptyManualRule($project, $function, $deployment, $dbForPlatform, $authorization);
|
||||
|
||||
if ($useJobs) {
|
||||
$jobs->create(...Job::build($project, $function, $deployment, $platform, $template));
|
||||
// Public template: pull the source straight from GitHub's codeload
|
||||
// tarball; unarchive strips the "{repo}-{ref}/" wrapper + the rootDirectory.
|
||||
$source = [
|
||||
'url' => "https://codeload.github.com/{$owner}/{$repository}/tar.gz/{$reference}",
|
||||
'subdir' => $rootDirectory,
|
||||
];
|
||||
$jobs->create(...Job::build($project, $function, $deployment, $platform, $source));
|
||||
} else {
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
|
||||
@@ -11,6 +11,7 @@ use Appwrite\SDK\AuthType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Response;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Validator\UID;
|
||||
@@ -73,6 +74,7 @@ class Create extends Base
|
||||
->inject('project')
|
||||
->inject('queueForEvents')
|
||||
->inject('publisherForBuilds')
|
||||
->inject('jobs')
|
||||
->inject('gitHub')
|
||||
->inject('platform')
|
||||
->callback($this->action(...));
|
||||
@@ -90,6 +92,7 @@ class Create extends Base
|
||||
Document $project,
|
||||
Event $queueForEvents,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
Jobs $jobs,
|
||||
GitHub $github,
|
||||
array $platform,
|
||||
) {
|
||||
@@ -119,7 +122,8 @@ class Create extends Base
|
||||
activate: $activate,
|
||||
platform: $platform,
|
||||
reference: $reference,
|
||||
referenceType: $type
|
||||
referenceType: $type,
|
||||
jobs: $jobs
|
||||
);
|
||||
|
||||
$queueForEvents
|
||||
|
||||
@@ -22,6 +22,7 @@ use Utopia\Platform\Action;
|
||||
use Utopia\Queue\Message;
|
||||
use Utopia\Storage\Device;
|
||||
use Utopia\System\System;
|
||||
use Utopia\VCS\Adapter\Git\GitHub;
|
||||
|
||||
/**
|
||||
* Applies open-runtimes jobs-service callbacks to a deployment. The API
|
||||
@@ -63,6 +64,7 @@ class Jobs extends Action
|
||||
->inject('publisherForUsage')
|
||||
->inject('usage')
|
||||
->inject('deviceForBuilds')
|
||||
->inject('gitHub')
|
||||
->inject('cache')
|
||||
->inject('locks')
|
||||
->callback($this->action(...));
|
||||
@@ -80,6 +82,7 @@ class Jobs extends Action
|
||||
UsagePublisher $publisherForUsage,
|
||||
UsageContext $usage,
|
||||
Device $deviceForBuilds,
|
||||
GitHub $github,
|
||||
Cache $cache,
|
||||
callable $locks,
|
||||
): void {
|
||||
@@ -90,7 +93,7 @@ class Jobs extends Action
|
||||
return;
|
||||
}
|
||||
|
||||
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $cache, $deploymentId): void {
|
||||
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $github, $cache, $deploymentId): void {
|
||||
if ($event->id !== '') {
|
||||
$key = 'jobs-event-' . $event->id;
|
||||
if ($cache->load($key, self::DEDUPE_TTL) !== false) {
|
||||
@@ -110,7 +113,7 @@ class Jobs extends Action
|
||||
|
||||
$deployment = match ($event->event) {
|
||||
'orchestrator.job.log' => $this->onLog($dbForProject, $deployment, $event->data),
|
||||
'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds),
|
||||
'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds, $github),
|
||||
default => $deployment,
|
||||
};
|
||||
|
||||
@@ -168,9 +171,10 @@ class Jobs extends Action
|
||||
UsageContext $usage,
|
||||
UsagePublisher $publisherForUsage,
|
||||
Device $deviceForBuilds,
|
||||
GitHub $github,
|
||||
): Document {
|
||||
if ($exitCode !== 0) {
|
||||
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage);
|
||||
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $github);
|
||||
}
|
||||
|
||||
$path = Job::buildPath($project->getId(), $deployment->getId());
|
||||
@@ -179,10 +183,10 @@ class Jobs extends Action
|
||||
$limit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000');
|
||||
if ($limit !== 0 && $size > $limit) {
|
||||
$deviceForBuilds->delete($path);
|
||||
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build size should be less than ' . \number_format($limit / (1000 * 1000), 2) . ' MBs.', $usage, $publisherForUsage);
|
||||
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build size should be less than ' . \number_format($limit / (1000 * 1000), 2) . ' MBs.', $usage, $publisherForUsage, $github);
|
||||
}
|
||||
|
||||
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, true, '', $usage, $publisherForUsage, $size);
|
||||
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, true, '', $usage, $publisherForUsage, $github, $size);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -199,6 +203,7 @@ class Jobs extends Action
|
||||
string $message,
|
||||
UsageContext $usage,
|
||||
UsagePublisher $publisherForUsage,
|
||||
GitHub $github,
|
||||
int $buildSize = 0,
|
||||
): Document {
|
||||
$function = $dbForProject->getDocument('functions', $deployment->getAttribute('resourceId'));
|
||||
@@ -244,9 +249,56 @@ class Jobs extends Action
|
||||
$this->updateLatestDeployment($dbForProject, $function);
|
||||
}
|
||||
|
||||
// Report the terminal outcome as a VCS commit status (jobs-built VCS
|
||||
// deployments only; best-effort). "pending" at build start is a follow-up.
|
||||
$status = $deployment->getAttribute('status');
|
||||
if (\in_array($status, ['ready', 'failed'], true) && ! $function->isEmpty()) {
|
||||
$this->gitStatus($github, $dbForPlatform, $project, $function, $deployment, $status === 'ready' ? 'success' : 'failure');
|
||||
}
|
||||
|
||||
return $deployment;
|
||||
}
|
||||
|
||||
/**
|
||||
* Post a GitHub commit status for a VCS deployment (no-op for non-VCS builds
|
||||
* or silent mode). Best-effort — a failed status update never fails the build.
|
||||
*/
|
||||
private function gitStatus(GitHub $github, Database $dbForPlatform, Document $project, Document $function, Document $deployment, string $state): void
|
||||
{
|
||||
$commitHash = $deployment->getAttribute('providerCommitHash', '');
|
||||
if ($commitHash === '' || $function->getAttribute('providerSilentMode', false) === true) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
$installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId', ''));
|
||||
$providerInstallationId = $installation->getAttribute('providerInstallationId', '');
|
||||
if ($providerInstallationId === '') {
|
||||
return;
|
||||
}
|
||||
$github->initializeVariables($providerInstallationId, System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY', ''), System::getEnv('_APP_VCS_GITHUB_APP_ID', ''));
|
||||
|
||||
$protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https';
|
||||
$hostname = System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN', ''));
|
||||
$region = $project->getAttribute('region', 'default');
|
||||
$targetUrl = "{$protocol}://{$hostname}/console/project-{$region}-{$project->getId()}/functions/function-{$function->getId()}";
|
||||
$message = $state === 'success' ? 'Build succeeded.' : 'Build failed.';
|
||||
$name = $function->getAttribute('name', '') . ' (' . $project->getAttribute('name', '') . ')';
|
||||
|
||||
$github->updateCommitStatus(
|
||||
$deployment->getAttribute('providerRepositoryName', ''),
|
||||
$commitHash,
|
||||
$deployment->getAttribute('providerRepositoryOwner', ''),
|
||||
$state,
|
||||
$message,
|
||||
$targetUrl,
|
||||
$name,
|
||||
);
|
||||
} catch (\Throwable) {
|
||||
// Best-effort; the build outcome stands regardless of the status update.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Point the function at this deployment (auto-activate). Mirrors the
|
||||
* essential function activation from the Builds worker.
|
||||
|
||||
Reference in new issue
Block a user