diff --git a/src/Appwrite/Compute/Job.php b/src/Appwrite/Compute/Job.php index 83e19cc393..1fcc4e5a12 100644 --- a/src/Appwrite/Compute/Job.php +++ b/src/Appwrite/Compute/Job.php @@ -32,12 +32,18 @@ final class Job /** * @return array Named arguments for OpenRuntimes\Orchestrator\Jobs::create(). */ + /** + * @param ?array{url: string, subdir?: string} $source Remote tarball source + * (template codeload URL, or a VCS presigned URL). When null, the source is + * the deployment's own uploaded tarball, fetched from Appwrite via a + * presigned token (manual upload / duplicate). + */ public static function build( Document $project, Document $function, Document $deployment, array $platform, - ?Document $template = null, + ?array $source = null, ): array { $projectId = $project->getId(); $deploymentId = $deployment->getId(); @@ -55,19 +61,18 @@ final class Job $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; $endpoint = System::getEnv('_APP_JOBS_ENDPOINT', "$protocol://{$platform['apiHostname']}"); - // Source artifacts: manual/duplicate deployments download a presigned - // tarball from Appwrite; template deployments pull the public GitHub - // tarball directly (the unarchive auto-strips the "{repo}-{ref}/" wrapper - // and, via subdir, extracts just the template's rootDirectory). Either - // way the build input lands in /mnt/code/source. - if ($template !== null) { - $owner = $template->getAttribute('ownerName', ''); - $repository = $template->getAttribute('repositoryName', ''); - $reference = $template->getAttribute('referenceValue', ''); - $rootDirectory = \trim($template->getAttribute('rootDirectory', ''), '/'); + // Source artifacts, both ending in /mnt/code/source: + // - remote tarball ($source): templates (public codeload URL) and VCS + // (a short-lived presigned URL). The unarchive auto-strips the + // "{repo}-{ref}/" wrapper and, via subdir, extracts just the + // rootDirectory. + // - otherwise: the deployment's uploaded tarball, fetched from Appwrite + // over a presigned GET (manual upload / duplicate). + if ($source !== null) { + $subdir = \trim($source['subdir'] ?? '', '/'); $sourceArtifacts = [ - new DownloadArtifact(id: 'source', in: "https://codeload.github.com/{$owner}/{$repository}/tar.gz/{$reference}", out: 'source.tar.gz'), - new UnarchiveArtifact(id: 'extract', in: 'source.tar.gz', out: 'source', subdir: $rootDirectory !== '' ? $rootDirectory : null), + new DownloadArtifact(id: 'source', in: $source['url'], out: 'source.tar.gz'), + new UnarchiveArtifact(id: 'extract', in: 'source.tar.gz', out: 'source', subdir: $subdir !== '' ? $subdir : null), ]; } else { // Presigned source-download URL (GET, no request-body cap), fetched by diff --git a/src/Appwrite/Platform/Modules/Compute/Base.php b/src/Appwrite/Platform/Modules/Compute/Base.php index 41463859f3..be88384244 100644 --- a/src/Appwrite/Platform/Modules/Compute/Base.php +++ b/src/Appwrite/Platform/Modules/Compute/Base.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Modules\Compute; +use Appwrite\Compute\Job; use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -9,6 +10,7 @@ use Appwrite\Filter\BranchDomain as BranchDomainFilter; use Appwrite\Platform\Action; use Appwrite\Platform\Modules\Compute\Validator\Specification as SpecificationValidator; use Appwrite\Platform\Permission as AppwritePermission; +use OpenRuntimes\Orchestrator\Jobs; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; @@ -67,7 +69,7 @@ class Base extends Action return $allowedSpecifications[0]; } - public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = ''): Document + public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null): Document { $deploymentId = ID::unique(); $entrypoint = $function->getAttribute('entrypoint', ''); @@ -118,6 +120,19 @@ class Base extends Action $repositoryUrl = "https://github.com/$owner/$repositoryName"; + // Build a plain (non-template) VCS function deployment on the jobs-service + // when the caller opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator. + // Sites and the template-into-repo push stay on the executor. When on + // jobs the deployment is pre-declared 'waiting' with its buildPath so + // build.sh writes output onto the mounted volume. + $useJobs = $jobs !== null + && $function->getCollection() === 'functions' + && $template->isEmpty() + && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'; + $buildFields = $useJobs + ? ['status' => 'waiting', 'buildPath' => Job::buildPath($project->getId(), $deploymentId)] + : []; + $deployment = $dbForProject->createDocument('deployments', new Document([ '$id' => $deploymentId, '$permissions' => [ @@ -149,16 +164,26 @@ class Base extends Action 'providerBranch' => $providerBranch, 'providerRootDirectory' => $function->getAttribute('providerRootDirectory', ''), 'activate' => $activate, + ...$buildFields, ])); - $publisherForBuilds->enqueue(new BuildMessage( - project: $project, - resource: $function, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - template: $template, - platform: $platform, - )); + if ($useJobs) { + $ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch'); + $source = [ + 'url' => $github->getRepositoryPresignedUrl($owner, $repositoryName, $ref), + 'subdir' => $function->getAttribute('providerRootDirectory', ''), + ]; + $jobs->create(...Job::build($project, $function, $deployment, $platform, $source)); + } else { + $publisherForBuilds->enqueue(new BuildMessage( + project: $project, + resource: $function, + deployment: $deployment, + type: BUILD_TYPE_DEPLOYMENT, + template: $template, + platform: $platform, + )); + } return $deployment; } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php index e53feae5fc..74e6fbee0f 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php @@ -207,7 +207,13 @@ class Create extends Base $this->updateEmptyManualRule($project, $function, $deployment, $dbForPlatform, $authorization); if ($useJobs) { - $jobs->create(...Job::build($project, $function, $deployment, $platform, $template)); + // Public template: pull the source straight from GitHub's codeload + // tarball; unarchive strips the "{repo}-{ref}/" wrapper + the rootDirectory. + $source = [ + 'url' => "https://codeload.github.com/{$owner}/{$repository}/tar.gz/{$reference}", + 'subdir' => $rootDirectory, + ]; + $jobs->create(...Job::build($project, $function, $deployment, $platform, $source)); } else { $publisherForBuilds->enqueue(new BuildMessage( project: $project, diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php index 65919f328c..1f372adda0 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php @@ -11,6 +11,7 @@ use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response; +use OpenRuntimes\Orchestrator\Jobs; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Validator\UID; @@ -73,6 +74,7 @@ class Create extends Base ->inject('project') ->inject('queueForEvents') ->inject('publisherForBuilds') + ->inject('jobs') ->inject('gitHub') ->inject('platform') ->callback($this->action(...)); @@ -90,6 +92,7 @@ class Create extends Base Document $project, Event $queueForEvents, BuildPublisher $publisherForBuilds, + Jobs $jobs, GitHub $github, array $platform, ) { @@ -119,7 +122,8 @@ class Create extends Base activate: $activate, platform: $platform, reference: $reference, - referenceType: $type + referenceType: $type, + jobs: $jobs ); $queueForEvents diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index 1feffc54f9..7615935e1f 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -22,6 +22,7 @@ use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\Storage\Device; use Utopia\System\System; +use Utopia\VCS\Adapter\Git\GitHub; /** * Applies open-runtimes jobs-service callbacks to a deployment. The API @@ -63,6 +64,7 @@ class Jobs extends Action ->inject('publisherForUsage') ->inject('usage') ->inject('deviceForBuilds') + ->inject('gitHub') ->inject('cache') ->inject('locks') ->callback($this->action(...)); @@ -80,6 +82,7 @@ class Jobs extends Action UsagePublisher $publisherForUsage, UsageContext $usage, Device $deviceForBuilds, + GitHub $github, Cache $cache, callable $locks, ): void { @@ -90,7 +93,7 @@ class Jobs extends Action return; } - $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $cache, $deploymentId): void { + $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $github, $cache, $deploymentId): void { if ($event->id !== '') { $key = 'jobs-event-' . $event->id; if ($cache->load($key, self::DEDUPE_TTL) !== false) { @@ -110,7 +113,7 @@ class Jobs extends Action $deployment = match ($event->event) { 'orchestrator.job.log' => $this->onLog($dbForProject, $deployment, $event->data), - 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds), + 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds, $github), default => $deployment, }; @@ -168,9 +171,10 @@ class Jobs extends Action UsageContext $usage, UsagePublisher $publisherForUsage, Device $deviceForBuilds, + GitHub $github, ): Document { if ($exitCode !== 0) { - return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage); + return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $github); } $path = Job::buildPath($project->getId(), $deployment->getId()); @@ -179,10 +183,10 @@ class Jobs extends Action $limit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000'); if ($limit !== 0 && $size > $limit) { $deviceForBuilds->delete($path); - return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build size should be less than ' . \number_format($limit / (1000 * 1000), 2) . ' MBs.', $usage, $publisherForUsage); + return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build size should be less than ' . \number_format($limit / (1000 * 1000), 2) . ' MBs.', $usage, $publisherForUsage, $github); } - return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, true, '', $usage, $publisherForUsage, $size); + return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, true, '', $usage, $publisherForUsage, $github, $size); } /** @@ -199,6 +203,7 @@ class Jobs extends Action string $message, UsageContext $usage, UsagePublisher $publisherForUsage, + GitHub $github, int $buildSize = 0, ): Document { $function = $dbForProject->getDocument('functions', $deployment->getAttribute('resourceId')); @@ -244,9 +249,56 @@ class Jobs extends Action $this->updateLatestDeployment($dbForProject, $function); } + // Report the terminal outcome as a VCS commit status (jobs-built VCS + // deployments only; best-effort). "pending" at build start is a follow-up. + $status = $deployment->getAttribute('status'); + if (\in_array($status, ['ready', 'failed'], true) && ! $function->isEmpty()) { + $this->gitStatus($github, $dbForPlatform, $project, $function, $deployment, $status === 'ready' ? 'success' : 'failure'); + } + return $deployment; } + /** + * Post a GitHub commit status for a VCS deployment (no-op for non-VCS builds + * or silent mode). Best-effort — a failed status update never fails the build. + */ + private function gitStatus(GitHub $github, Database $dbForPlatform, Document $project, Document $function, Document $deployment, string $state): void + { + $commitHash = $deployment->getAttribute('providerCommitHash', ''); + if ($commitHash === '' || $function->getAttribute('providerSilentMode', false) === true) { + return; + } + + try { + $installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId', '')); + $providerInstallationId = $installation->getAttribute('providerInstallationId', ''); + if ($providerInstallationId === '') { + return; + } + $github->initializeVariables($providerInstallationId, System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY', ''), System::getEnv('_APP_VCS_GITHUB_APP_ID', '')); + + $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; + $hostname = System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN', '')); + $region = $project->getAttribute('region', 'default'); + $targetUrl = "{$protocol}://{$hostname}/console/project-{$region}-{$project->getId()}/functions/function-{$function->getId()}"; + $message = $state === 'success' ? 'Build succeeded.' : 'Build failed.'; + $name = $function->getAttribute('name', '') . ' (' . $project->getAttribute('name', '') . ')'; + + $github->updateCommitStatus( + $deployment->getAttribute('providerRepositoryName', ''), + $commitHash, + $deployment->getAttribute('providerRepositoryOwner', ''), + $state, + $message, + $targetUrl, + $name, + ); + } catch (\Throwable) { + // Best-effort; the build outcome stands regardless of the status update. + } + } + /** * Point the function at this deployment (auto-activate). Mirrors the * essential function activation from the Builds worker.