From 11550cf07f0c8b069b25af7f094eccb7228a2a31 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:13:55 +0100 Subject: [PATCH] Extract a Deployments service to centralize build dispatch across functions and sites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consolidates the scattered deployment-creation and build-dispatch logic across HTTP actions (manual upload, duplicate, template, VCS webhook, VCS reconnect) into a single Appwrite\Service\Deployments abstraction, with Executor and Jobs implementations selected per-request by _APP_BUILDS_BACKEND. Sites are now forced onto the executor implementation regardless of that env var, since the jobs-service payload builder is function-shaped. - upload() for chunked-upload bookkeeping (no build), createFromUpload()/ createFromRef()/createFromUrl() for the three real build-source shapes, delete() for best-effort cancel — replaces duplicated permissions/ resourceId-triplet construction and deactivate-others logic at every call site. - Appwrite\Compute\Job merged into Deployments\Jobs; its output/cache handling is now an overridable storage() extension point (late-static-bound) so a fork can swap the shared-volume strategy for upload/download artifacts without touching payload construction. Co-Authored-By: Claude Sonnet 5 --- app/init/resources/request.php | 15 ++ .../Platform/Modules/Compute/Base.php | 59 +++---- .../Functions/Http/Deployments/Create.php | 124 +++----------- .../Http/Deployments/Duplicate/Create.php | 45 ++--- .../Http/Deployments/Status/Update.php | 22 +-- .../Http/Deployments/Template/Create.php | 110 +++++------- .../Functions/Http/Deployments/Vcs/Create.php | 8 +- .../Functions/Http/Functions/Create.php | 8 +- .../Functions/Http/Functions/Update.php | 8 +- .../Modules/Functions/Workers/Builds.php | 6 +- .../Modules/Functions/Workers/Jobs.php | 4 +- .../Modules/Sites/Http/Deployments/Create.php | 100 +++-------- .../Http/GitHub/Authorize/External/Update.php | 8 +- .../Modules/VCS/Http/GitHub/Deployment.php | 83 ++++----- .../Modules/VCS/Http/GitHub/Events/Create.php | 18 +- src/Appwrite/Service/Deployments.php | 143 ++++++++++++++++ src/Appwrite/Service/Deployments/Executor.php | 86 ++++++++++ .../Job.php => Service/Deployments/Jobs.php} | 158 ++++++++++++++---- 18 files changed, 567 insertions(+), 438 deletions(-) create mode 100644 src/Appwrite/Service/Deployments.php create mode 100644 src/Appwrite/Service/Deployments/Executor.php rename src/Appwrite/{Compute/Job.php => Service/Deployments/Jobs.php} (62%) diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 54309c2430..5242d9fce9 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -8,6 +8,7 @@ use Appwrite\Databases\TransactionState; use Appwrite\Event\Context\Audit as AuditContext; use Appwrite\Event\Event; use Appwrite\Event\Message\Func as FunctionMessage; +use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Event\Publisher\Func as FunctionPublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Webhook; @@ -20,10 +21,14 @@ use Appwrite\Network\Cors; use Appwrite\Network\Platform; use Appwrite\Network\Validator\Origin; use Appwrite\Network\Validator\Redirect; +use Appwrite\Service\Deployments\Executor as DeploymentsExecutor; +use Appwrite\Service\Deployments\Jobs as DeploymentsJobs; use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; +use Executor\Executor; +use OpenRuntimes\Orchestrator\Jobs; use Utopia\Agents\Adapters\Appwrite as AppwriteAdapter; use Utopia\Agents\Agent; use Utopia\Audit\Adapter\Database as AdapterDatabase; @@ -191,6 +196,16 @@ return function (Container $context): void { $publisher, new Queue(System::getEnv('_APP_FUNCTIONS_QUEUE_NAME', Event::FUNCTIONS_QUEUE_NAME), 'utopia-queue', Event::FUNCTIONS_QUEUE_TTL) ), ['publisher']); + $context->set('deployments', function (BuildPublisher $publisherForBuilds, Jobs $jobs, Database $dbForProject, Document $project, Executor $executor, array $platform, Request $request) { + // The jobs-service orchestrator backend only understands functions + // (its payload builder reads function-shaped attributes like + // `runtime`); sites always build on the executor. + $isSite = \str_starts_with($request->getURI(), '/v1/sites'); + + return !$isSite && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator' + ? new DeploymentsJobs($jobs, $dbForProject, $project, $platform) + : new DeploymentsExecutor($publisherForBuilds, $dbForProject, $project, $executor, $platform); + }, ['publisherForBuilds', 'jobs', 'dbForProject', 'project', 'executor', 'platform', 'request']); $context->set('eventProcessor', fn () => new EventProcessor(), []); $context->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory( $pools, diff --git a/src/Appwrite/Platform/Modules/Compute/Base.php b/src/Appwrite/Platform/Modules/Compute/Base.php index 25f071d253..a86a412b94 100644 --- a/src/Appwrite/Platform/Modules/Compute/Base.php +++ b/src/Appwrite/Platform/Modules/Compute/Base.php @@ -2,7 +2,6 @@ namespace Appwrite\Platform\Modules\Compute; -use Appwrite\Compute\Job; use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -10,7 +9,7 @@ use Appwrite\Filter\BranchDomain as BranchDomainFilter; use Appwrite\Platform\Action; use Appwrite\Platform\Modules\Compute\Validator\Specification as SpecificationValidator; use Appwrite\Platform\Permission as AppwritePermission; -use OpenRuntimes\Orchestrator\Jobs; +use Appwrite\Service\Deployments; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; @@ -69,7 +68,7 @@ class Base extends Action return $allowedSpecifications[0]; } - public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null): Document + public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Deployments $deployments = null): Document { $deploymentId = ID::unique(); $entrypoint = $function->getAttribute('entrypoint', ''); @@ -120,28 +119,8 @@ class Base extends Action $repositoryUrl = "https://github.com/$owner/$repositoryName"; - // Build a plain (non-template) VCS function deployment on the jobs-service - // when the caller opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator. - // Sites stay on the executor; template-into-repo pushes go through the - // Builds worker (which does the git write, then hands the build to the - // jobs-service itself when on orchestrator). When on jobs the deployment - // is pre-declared 'waiting' with its buildPath so build.sh writes output - // onto the mounted volume. - $useJobs = $jobs !== null - && $function->getCollection() === 'functions' - && $template->isEmpty() - && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'; - $buildFields = $useJobs - ? ['status' => 'waiting', 'buildPath' => Job::buildPath($project->getId(), $deploymentId)] - : []; - - $deployment = $dbForProject->createDocument('deployments', new Document([ + $deployment = new Document([ '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], 'resourceId' => $function->getId(), 'resourceInternalId' => $function->getSequence(), 'resourceType' => 'functions', @@ -166,17 +145,33 @@ class Base extends Action 'providerBranch' => $providerBranch, 'providerRootDirectory' => $function->getAttribute('providerRootDirectory', ''), 'activate' => $activate, - ...$buildFields, - ])); + ]); - if ($useJobs) { + // Build a plain (non-template) VCS function deployment through + // $deployments (executor or jobs-service, decided by + // _APP_BUILDS_BACKEND) when the caller opts in. Sites stay on the + // executor; template-into-repo pushes go through the Builds worker + // (which does the git write, then hands the build to the + // jobs-service itself when on orchestrator). + if ($deployments !== null && $function->getCollection() === 'functions' && $template->isEmpty()) { $ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch'); - $source = [ - 'url' => $github->getRepositoryPresignedUrl($owner, $repositoryName, $ref), - 'subdir' => $function->getAttribute('providerRootDirectory', ''), - ]; - $jobs->create(...Job::build($project, $function, $deployment, $platform, $source)); + $deployment = $deployments->createFromUrl( + $function, + $deployment, + $github->getRepositoryPresignedUrl($owner, $repositoryName, $ref), + $function->getAttribute('providerRootDirectory', ''), + ); } else { + $deployment = $dbForProject->createDocument('deployments', new Document([ + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ...$deployment->getArrayCopy(), + 'status' => 'waiting', + ])); + $publisherForBuilds->enqueue(new BuildMessage( project: $project, resource: $function, diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php index f6e4c59e44..146cb8afed 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php @@ -2,24 +2,18 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments; -use Appwrite\Compute\Job; use Appwrite\Event\Event; -use Appwrite\Event\Message\Build as BuildMessage; -use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\MethodType; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Service\Deployments; use Appwrite\Utopia\Response; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; -use Utopia\Database\Helpers\Permission; -use Utopia\Database\Helpers\Role; -use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Http\Adapter\Swoole\Request; @@ -92,11 +86,9 @@ class Create extends Action ->inject('project') ->inject('deviceForFunctions') ->inject('deviceForLocal') - ->inject('publisherForBuilds') - ->inject('jobs') + ->inject('deployments') ->inject('plan') ->inject('authorization') - ->inject('platform') ->inject('locks') ->callback($this->action(...)); } @@ -114,11 +106,9 @@ class Create extends Action Document $project, Device $deviceForFunctions, Device $deviceForLocal, - BuildPublisher $publisherForBuilds, - Jobs $jobs, + Deployments $deployments, array $plan, Authorization $authorization, - array $platform, callable $locks ) { $activate = \strval($activate) === 'true' || \strval($activate) === '1'; @@ -227,7 +217,7 @@ class Create extends Action $type = $request->getHeaderLine('x-sdk-language') === 'cli' ? 'cli' : 'manual'; try { - $locks($lockKey, 600, function () use ($activate, &$chunks, $commands, $contentRange, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, &$metadata, $path, $type, &$completed, $response): void { + $locks($lockKey, 600, function () use ($activate, &$chunks, $commands, $contentRange, $dbForProject, $deploymentId, $deployments, $deviceForFunctions, $entrypoint, $fileSize, &$function, &$metadata, $path, $type, &$completed, $response): void { $deployment = $dbForProject->getDocument('deployments', $deploymentId); if (!$deployment->isEmpty()) { @@ -249,16 +239,8 @@ class Create extends Action $deviceForFunctions->prepareUpload($path, $metadata['content_type'] ?? '', $chunks, $metadata); if (!empty($contentRange)) { - $deployment = $dbForProject->createDocument('deployments', new Document([ + $deployment = $deployments->upload($function, $deployment->setAttributes([ '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceInternalId' => $function->getSequence(), - 'resourceId' => $function->getId(), - 'resourceType' => 'functions', 'entrypoint' => $entrypoint, 'buildCommands' => $commands, 'startCommand' => $function->getAttribute('startCommand', ''), @@ -269,7 +251,7 @@ class Create extends Action 'sourceChunksUploaded' => 0, 'activate' => $activate, 'sourceMetadata' => $metadata, - 'type' => $type + 'type' => $type, ])); } } @@ -291,7 +273,7 @@ class Create extends Action } try { - $locks($lockKey, 600, function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $functionId, $path, &$metadata, $mergeUploadMetadata, $platform, $project, $publisherForBuilds, $jobs, $queueForEvents, $response, $type): void { + $locks($lockKey, 600, function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $path, &$metadata, $mergeUploadMetadata, $deployments, $queueForEvents, $response, $type): void { $deployment = $dbForProject->getDocument('deployments', $deploymentId); $uploaded = 0; @@ -315,86 +297,24 @@ class Create extends Action if ($chunksUploaded === $chunks && $uploaded < $chunks) { $deviceForFunctions->finalizeUpload($path, $chunks, $metadata); - if ($activate) { - // Remove deploy for all other deployments. - $activeDeployments = $dbForProject->find('deployments', [ - Query::equal('activate', [true]), - Query::equal('resourceId', [$functionId]), - Query::equal('resourceType', ['functions']) - ]); - - foreach ($activeDeployments as $activeDeployment) { - $dbForProject->updateDocument('deployments', $activeDeployment->getId(), new Document([ - 'activate' => false, - ])); - } - } - $fileSize = $deviceForFunctions->getFileSize($path); - // Build backend for manual-upload function deployments: - // 'orchestrator' (open-runtimes jobs-service, submitted below - // in the request flow) or 'executor' (default; enqueued to the - // Builds worker). Selected by _APP_BUILDS_BACKEND. - $useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'; - - // Fields marking the deployment as queued. The build worker - // promotes 'waiting' → 'building' on the first callback. The - // jobs path also pre-declares buildPath so build.sh writes the - // output straight onto the mounted builds volume. - $buildFields = ['status' => 'waiting']; - if ($useJobs) { - $buildFields['buildPath'] = Job::buildPath($project->getId(), $deploymentId); - } - - if ($deployment->isEmpty()) { - $deployment = $dbForProject->createDocument('deployments', new Document([ - '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceInternalId' => $function->getSequence(), - 'resourceId' => $function->getId(), - 'resourceType' => 'functions', - 'entrypoint' => $entrypoint, - 'buildCommands' => $commands, - 'startCommand' => $function->getAttribute('startCommand', ''), - 'sourcePath' => $path, - 'sourceSize' => $fileSize, - 'totalSize' => $fileSize, - 'sourceChunksTotal' => $chunks, - 'sourceChunksUploaded' => $chunksUploaded, - 'activate' => $activate, - 'sourceMetadata' => $metadata, - 'type' => $type, - ...$buildFields, - ])); - - } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ - 'sourceSize' => $fileSize, - 'sourceChunksUploaded' => $chunksUploaded, - 'sourceMetadata' => $metadata, - ...$buildFields, - ])); - } - - if ($useJobs) { - $jobs->create(...Job::build($project, $function, $deployment, $platform)); - } else { - // Default: build on the executor via the Builds worker. - $publisherForBuilds->enqueue(new BuildMessage( - project: $project, - resource: $function, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - platform: $platform, - )); - } + $deployment = $deployments->createFromUpload($function, $deployment->setAttributes([ + '$id' => $deploymentId, + 'entrypoint' => $entrypoint, + 'buildCommands' => $commands, + 'startCommand' => $function->getAttribute('startCommand', ''), + 'sourcePath' => $path, + 'sourceSize' => $fileSize, + 'totalSize' => $fileSize, + 'sourceChunksTotal' => $chunks, + 'sourceChunksUploaded' => $chunksUploaded, + 'activate' => $activate, + 'sourceMetadata' => $metadata, + 'type' => $type, + ])); } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ + $deployment = $deployments->upload($function, $deployment->setAttributes([ 'sourceChunksUploaded' => $chunksUploaded, 'sourceMetadata' => $metadata, ])); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php index 1e50fcd214..f959a66da7 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php @@ -2,24 +2,19 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Duplicate; -use Appwrite\Compute\Job; use Appwrite\Event\Event; -use Appwrite\Event\Message\Build as BuildMessage; -use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Service\Deployments; use Appwrite\Utopia\Response; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Database\Database; -use Utopia\Database\Document; use Utopia\Database\Helpers\ID; use Utopia\Database\Validator\UID; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; use Utopia\Storage\Device; -use Utopia\System\System; class Create extends Action { @@ -66,11 +61,8 @@ class Create extends Action ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('publisherForBuilds') - ->inject('jobs') + ->inject('deployments') ->inject('deviceForFunctions') - ->inject('project') - ->inject('platform') ->callback($this->action(...)); } @@ -81,11 +73,8 @@ class Create extends Action Response $response, Database $dbForProject, Event $queueForEvents, - BuildPublisher $publisherForBuilds, - Jobs $jobs, + Deployments $deployments, Device $deviceForFunctions, - Document $project, - array $platform ) { $function = $dbForProject->getDocument('functions', $functionId); @@ -105,17 +94,14 @@ class Create extends Action $deploymentId = ID::unique(); - // Build backend for the rebuild: 'orchestrator' (jobs-service, submitted - // here) or 'executor' (default; Builds worker). The jobs path pre-declares - // buildPath so build.sh writes output straight onto the mounted volume. - $useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'; - $destination = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); $deviceForFunctions->transfer($path, $destination, $deviceForFunctions); + // Cloning the source deployment's attributes onto the new one, with + // its own $id and no $sequence, tells the service to create it fresh + // rather than update the deployment being duplicated. $deployment->removeAttribute('$sequence'); - - $deployment = $dbForProject->createDocument('deployments', $deployment->setAttributes([ + $deployment->setAttributes([ '$id' => $deploymentId, 'sourcePath' => $destination, 'totalSize' => $deployment->getAttribute('sourceSize', 0), @@ -126,22 +112,11 @@ class Create extends Action 'buildEndedAt' => null, 'buildDuration' => null, 'buildSize' => null, - 'status' => 'waiting', - 'buildPath' => $useJobs ? Job::buildPath($project->getId(), $deploymentId) : '', + 'buildPath' => '', 'buildLogs' => '', - ])); + ]); - if ($useJobs) { - $jobs->create(...Job::build($project, $function, $deployment, $platform)); - } else { - $publisherForBuilds->enqueue(new BuildMessage( - project: $project, - resource: $function, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - platform: $platform, - )); - } + $deployment = $deployments->createFromUpload($function, $deployment); $queueForEvents ->setParam('functionId', $function->getId()) diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php index fb354f799d..c59c108e60 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php @@ -2,15 +2,13 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Status; -use Appwrite\Compute\Job; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Service\Deployments; use Appwrite\Utopia\Response; -use Executor\Executor; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; @@ -60,10 +58,8 @@ class Update extends Action ->param('deploymentId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Deployment ID.', false, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('project') ->inject('queueForEvents') - ->inject('executor') - ->inject('jobs') + ->inject('deployments') ->callback($this->action(...)); } @@ -72,10 +68,8 @@ class Update extends Action string $deploymentId, Response $response, Database $dbForProject, - Document $project, Event $queueForEvents, - Executor $executor, - Jobs $jobs, + Deployments $deployments, ) { $function = $dbForProject->getDocument('functions', $functionId); @@ -115,15 +109,9 @@ class Update extends Action } } - // Best-effort cleanup on both backends — the deployment is already - // marked 'canceled', and only one backend actually holds the build. + // Best-effort cleanup — the deployment is already marked 'canceled'. try { - $executor->deleteRuntime($project->getId(), $deploymentId . "-build"); - } catch (\Throwable) { - } - - try { - $jobs->delete(Job::id($project->getId(), $deploymentId)); + $deployments->delete($deploymentId); } catch (\Throwable) { } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php index 202e1c34d3..2a851421b2 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php @@ -2,9 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Template; -use Appwrite\Compute\Job; use Appwrite\Event\Event; -use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; @@ -12,19 +10,16 @@ use Appwrite\Platform\Modules\Compute\Base; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Service\Deployments; use Appwrite\Utopia\Response; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; -use Utopia\Database\Helpers\Permission; -use Utopia\Database\Helpers\Role; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Http\Adapter\Swoole\Request; use Utopia\Platform\Enum; use Utopia\Platform\Scope\HTTP; -use Utopia\System\System; use Utopia\Validator\Boolean; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; @@ -83,7 +78,7 @@ class Create extends Base ->inject('queueForEvents') ->inject('project') ->inject('publisherForBuilds') - ->inject('jobs') + ->inject('deployments') ->inject('gitHub') ->inject('authorization') ->inject('platform') @@ -105,7 +100,7 @@ class Create extends Base Event $queueForEvents, Document $project, BuildPublisher $publisherForBuilds, - Jobs $jobs, + Deployments $deployments, GitHub $github, Authorization $authorization, array $platform @@ -145,6 +140,7 @@ class Create extends Base template: $template, github: $github, activate: $activate, + deployments: $deployments, platform: $platform, referenceType: $type, reference: $reference @@ -161,70 +157,48 @@ class Create extends Base return; } - // Backend for the template build: 'orchestrator' (jobs-service, which - // pulls the public GitHub tarball via artifacts) or 'executor' (default; - // the Builds worker clones the repo). The jobs path pre-declares buildPath. - $useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'; - $deploymentId = ID::unique(); - $deployment = $dbForProject->createDocument('deployments', new Document([ - '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceId' => $function->getId(), - 'resourceInternalId' => $function->getSequence(), - 'resourceType' => 'functions', - 'entrypoint' => $function->getAttribute('entrypoint', ''), - 'buildCommands' => $function->getAttribute('commands', ''), - 'startCommand' => $function->getAttribute('startCommand', ''), - 'providerRepositoryName' => $repository, - 'providerRepositoryOwner' => $owner, - 'providerRepositoryUrl' => $repositoryUrl, - 'providerBranchUrl' => $branchUrl, - 'providerBranch' => $type == GitHub::CLONE_TYPE_BRANCH ? $reference : '', - 'type' => 'vcs', - 'activate' => $activate, - 'status' => 'waiting', - 'buildPath' => $useJobs ? Job::buildPath($project->getId(), $deploymentId) : '', - ])); + + // Templates can pin a version range (e.g. "0.3.*"); codeload only + // takes a concrete ref, so resolve the range to the highest matching + // tag (mirrors the executor's `git ls-remote --tags | tail -1`). + $ref = $reference; + if ($type === GitHub::CLONE_TYPE_TAG && \str_contains($reference, '*')) { + try { + $tags = $github->listTags($owner, $repository, $reference); + $ref = \end($tags) ?: $reference; + } catch (\Throwable) { + // Fall back to the raw reference; the build surfaces a bad ref. + } + } + + // Public template: pull the source straight from GitHub's public repo + // (codeload tarball on the jobs backend, a plain git clone on the + // executor); unarchive/checkout strips down to the rootDirectory. + $deployment = $deployments->createFromRef( + $function, + new Document([ + '$id' => $deploymentId, + 'entrypoint' => $function->getAttribute('entrypoint', ''), + 'buildCommands' => $function->getAttribute('commands', ''), + 'startCommand' => $function->getAttribute('startCommand', ''), + 'providerRepositoryName' => $repository, + 'providerRepositoryOwner' => $owner, + 'providerRepositoryUrl' => $repositoryUrl, + 'providerBranchUrl' => $branchUrl, + 'providerBranch' => $type == GitHub::CLONE_TYPE_BRANCH ? $reference : '', + 'type' => 'vcs', + 'activate' => $activate, + ]), + $owner, + $repository, + $type, + $ref, + $rootDirectory, + ); $this->updateEmptyManualRule($project, $function, $deployment, $dbForPlatform, $authorization); - if ($useJobs) { - // Templates can pin a version range (e.g. "0.3.*"); codeload only - // takes a concrete ref, so resolve the range to the highest matching - // tag (mirrors the executor's `git ls-remote --tags | tail -1`). - $ref = $reference; - if ($type === GitHub::CLONE_TYPE_TAG && \str_contains($reference, '*')) { - try { - $tags = $github->listTags($owner, $repository, $reference); - $ref = \end($tags) ?: $reference; - } catch (\Throwable) { - // Fall back to the raw reference; the build surfaces a bad ref. - } - } - - // Public template: pull the source straight from GitHub's codeload - // tarball; unarchive strips the "{repo}-{ref}/" wrapper + the rootDirectory. - $source = [ - 'url' => "https://codeload.github.com/{$owner}/{$repository}/tar.gz/{$ref}", - 'subdir' => $rootDirectory, - ]; - $jobs->create(...Job::build($project, $function, $deployment, $platform, $source)); - } else { - $publisherForBuilds->enqueue(new BuildMessage( - project: $project, - resource: $function, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - template: $template, - platform: $platform, - )); - } - $queueForEvents ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php index 1f372adda0..86b4a907ea 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php @@ -10,8 +10,8 @@ use Appwrite\Platform\Modules\Compute\Base; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Service\Deployments; use Appwrite\Utopia\Response; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Validator\UID; @@ -74,7 +74,7 @@ class Create extends Base ->inject('project') ->inject('queueForEvents') ->inject('publisherForBuilds') - ->inject('jobs') + ->inject('deployments') ->inject('gitHub') ->inject('platform') ->callback($this->action(...)); @@ -92,7 +92,7 @@ class Create extends Base Document $project, Event $queueForEvents, BuildPublisher $publisherForBuilds, - Jobs $jobs, + Deployments $deployments, GitHub $github, array $platform, ) { @@ -123,7 +123,7 @@ class Create extends Base platform: $platform, reference: $reference, referenceType: $type, - jobs: $jobs + deployments: $deployments ); $queueForEvents diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php index ada8596e06..3d3710b3bc 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php @@ -16,11 +16,11 @@ use Appwrite\Platform\Modules\Compute\Validator\Specification; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Service\Deployments; use Appwrite\Task\Validator\Cron; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model\Rule; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Abuse\Abuse; use Utopia\Config\Config; use Utopia\Database\Database; @@ -124,7 +124,7 @@ class Create extends Base ->inject('project') ->inject('queueForEvents') ->inject('publisherForBuilds') - ->inject('jobs') + ->inject('deployments') ->inject('queueForRealtime') ->inject('queueForWebhooks') ->inject('publisherForFunctions') @@ -169,7 +169,7 @@ class Create extends Base Document $project, Event $queueForEvents, BuildPublisher $publisherForBuilds, - Jobs $jobs, + Deployments $deployments, Realtime $queueForRealtime, Webhook $queueForWebhooks, FunctionPublisher $publisherForFunctions, @@ -346,7 +346,7 @@ class Create extends Base installation: $installation, dbForProject: $dbForProject, publisherForBuilds: $publisherForBuilds, - jobs: $jobs, + deployments: $deployments, template: $template, github: $github, activate: true, diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php index ceb8ec7919..9666021f09 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php @@ -11,10 +11,10 @@ use Appwrite\Platform\Modules\Compute\Validator\Specification; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Service\Deployments; use Appwrite\Task\Validator\Cron; use Appwrite\Utopia\Response; use Executor\Executor; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -112,7 +112,7 @@ class Update extends Base ->inject('project') ->inject('queueForEvents') ->inject('publisherForBuilds') - ->inject('jobs') + ->inject('deployments') ->inject('dbForPlatform') ->inject('gitHub') ->inject('executor') @@ -150,7 +150,7 @@ class Update extends Base Document $project, Event $queueForEvents, BuildPublisher $publisherForBuilds, - Jobs $jobs, + Deployments $deployments, Database $dbForPlatform, GitHub $github, Executor $executor, @@ -312,7 +312,7 @@ class Update extends Base // Redeploy logic if (!$isConnected && !empty($providerRepositoryId)) { - $this->redeployVcsFunction($request, $function, $project, $installation, $dbForProject, $publisherForBuilds, new Document(), $github, true, $platform, jobs: $jobs); + $this->redeployVcsFunction($request, $function, $project, $installation, $dbForProject, $publisherForBuilds, new Document(), $github, true, $platform, deployments: $deployments); } // Inform scheduler if function is still active diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index fccafa256d..97f5ecef3b 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Functions\Workers; use Ahc\Jwt\JWT; -use Appwrite\Compute\Job; use Appwrite\Event\Event; use Appwrite\Event\Message\Func as FunctionMessage; use Appwrite\Event\Publisher\Func as FunctionPublisher; @@ -13,6 +12,7 @@ use Appwrite\Event\Realtime; use Appwrite\Event\Webhook; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Filter\BranchDomain as BranchDomainFilter; +use Appwrite\Service\Deployments\Jobs as DeploymentsJobs; use Appwrite\Usage\Build as BuildUsage; use Appwrite\Usage\Context; use Appwrite\Utopia\Response\Model\Deployment; @@ -533,11 +533,11 @@ class Builds extends Action if ($resource->getCollection() === 'functions' && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator') { $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'buildPath' => Job::buildPath($project->getId(), $deploymentId), + 'buildPath' => DeploymentsJobs::buildPath($project->getId(), $deploymentId), ])); $ref = $deployment->getAttribute('providerCommitHash') ?: $branchName; - $jobs->create(...Job::build($project, $resource, $deployment, $platform, [ + $jobs->create(...DeploymentsJobs::payload($project, $resource, $deployment, $platform, [ 'url' => $github->getRepositoryPresignedUrl($cloneOwner, $cloneRepository, $ref), 'subdir' => $resource->getAttribute('providerRootDirectory', ''), ])); diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index 9086ffa550..3a1e135b45 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -2,7 +2,6 @@ namespace Appwrite\Platform\Modules\Functions\Workers; -use Appwrite\Compute\Job; use Appwrite\Event\Event; use Appwrite\Event\Message\Func as FunctionMessage; use Appwrite\Event\Message\Jobs as JobsMessage; @@ -10,6 +9,7 @@ use Appwrite\Event\Publisher\Func as FunctionPublisher; use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Webhook; +use Appwrite\Service\Deployments\Jobs as DeploymentsJobs; use Appwrite\Usage\Build as BuildUsage; use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Response\Model\Deployment; @@ -209,7 +209,7 @@ class Jobs extends Action return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $github); } - $path = Job::buildPath($project->getId(), $deployment->getId()); + $path = DeploymentsJobs::buildPath($project->getId(), $deployment->getId()); $size = $deviceForBuilds->exists($path) ? $deviceForBuilds->getFileSize($path) : 0; $limit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000'); diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php index 79164c4999..3eea1215ca 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php @@ -3,21 +3,17 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments; use Appwrite\Event\Event; -use Appwrite\Event\Message\Build as BuildMessage; -use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\MethodType; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Service\Deployments; use Appwrite\Utopia\Response; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; -use Utopia\Database\Helpers\Permission; -use Utopia\Database\Helpers\Role; -use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Http\Adapter\Swoole\Request; @@ -88,7 +84,7 @@ class Create extends Action ->inject('queueForEvents') ->inject('deviceForSites') ->inject('deviceForLocal') - ->inject('publisherForBuilds') + ->inject('deployments') ->inject('plan') ->inject('authorization') ->inject('platform') @@ -111,7 +107,7 @@ class Create extends Action Event $queueForEvents, Device $deviceForSites, Device $deviceForLocal, - BuildPublisher $publisherForBuilds, + Deployments $deployments, array $plan, Authorization $authorization, array $platform, @@ -231,7 +227,7 @@ class Create extends Action } try { - $locks($lockKey, 600, function () use ($activate, $authorization, &$chunks, $commands, $contentRange, $dbForPlatform, $dbForProject, $deploymentId, $deviceForSites, $fileSize, &$metadata, $outputDirectory, $path, $platform, $project, &$site, $type, &$completed, $response): void { + $locks($lockKey, 600, function () use ($activate, $authorization, &$chunks, $commands, $contentRange, $dbForPlatform, $dbForProject, $deploymentId, $deployments, $deviceForSites, $fileSize, &$metadata, $outputDirectory, $path, $platform, $project, &$site, $type, &$completed, $response): void { $deployment = $dbForProject->getDocument('deployments', $deploymentId); if (!$deployment->isEmpty()) { @@ -253,16 +249,8 @@ class Create extends Action $deviceForSites->prepareUpload($path, $metadata['content_type'] ?? '', $chunks, $metadata); if (!empty($contentRange)) { - $deployment = $dbForProject->createDocument('deployments', new Document([ + $deployment = $deployments->upload($site, $deployment->setAttributes([ '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceInternalId' => $site->getSequence(), - 'resourceId' => $site->getId(), - 'resourceType' => 'sites', 'buildCommands' => \implode(' && ', $commands), 'startCommand' => $site->getAttribute('startCommand', ''), 'buildOutput' => $outputDirectory, @@ -325,7 +313,7 @@ class Create extends Action } try { - $locks($lockKey, 600, function () use ($activate, $authorization, $commands, &$chunks, $chunksUploaded, $dbForPlatform, $dbForProject, $deploymentId, $deviceForSites, $fileSize, &$metadata, $mergeUploadMetadata, $outputDirectory, $path, $platform, $project, $publisherForBuilds, $queueForEvents, $response, &$site, $siteId, $type): void { + $locks($lockKey, 600, function () use ($activate, $authorization, $commands, &$chunks, $chunksUploaded, $dbForPlatform, $dbForProject, $deploymentId, $deployments, $deviceForSites, $fileSize, &$metadata, $mergeUploadMetadata, $outputDirectory, $path, $platform, $project, $queueForEvents, $response, &$site, $type): void { $deployment = $dbForProject->getDocument('deployments', $deploymentId); $uploaded = 0; @@ -349,47 +337,28 @@ class Create extends Action if ($chunksUploaded === $chunks && $uploaded < $chunks) { $deviceForSites->finalizeUpload($path, $chunks, $metadata); - if ($activate) { - // Remove deploy for all other deployments. - $activeDeployments = $dbForProject->find('deployments', [ - Query::equal('activate', [true]), - Query::equal('resourceId', [$siteId]), - Query::equal('resourceType', ['sites']) - ]); - - foreach ($activeDeployments as $activeDeployment) { - $dbForProject->updateDocument('deployments', $activeDeployment->getId(), new Document(['activate' => false])); - } - } - $fileSize = $deviceForSites->getFileSize($path); + $isNewDeployment = $deployment->isEmpty(); - if ($deployment->isEmpty()) { - $deployment = $dbForProject->createDocument('deployments', new Document([ - '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceInternalId' => $site->getSequence(), - 'resourceId' => $site->getId(), - 'resourceType' => 'sites', - 'buildCommands' => \implode(' && ', $commands), - 'startCommand' => $site->getAttribute('startCommand', ''), - 'buildOutput' => $outputDirectory, - 'adapter' => $site->getAttribute('adapter', ''), - 'fallbackFile' => $site->getAttribute('fallbackFile', ''), - 'sourcePath' => $path, - 'sourceSize' => $fileSize, - 'totalSize' => $fileSize, - 'sourceChunksTotal' => $chunks, - 'sourceChunksUploaded' => $chunksUploaded, - 'activate' => $activate, - 'sourceMetadata' => $metadata, - 'type' => $type, - ])); + // Start the build + $deployment = $deployments->createFromUpload($site, $deployment->setAttributes([ + '$id' => $deploymentId, + 'buildCommands' => \implode(' && ', $commands), + 'startCommand' => $site->getAttribute('startCommand', ''), + 'buildOutput' => $outputDirectory, + 'adapter' => $site->getAttribute('adapter', ''), + 'fallbackFile' => $site->getAttribute('fallbackFile', ''), + 'sourcePath' => $path, + 'sourceSize' => $fileSize, + 'totalSize' => $fileSize, + 'sourceChunksTotal' => $chunks, + 'sourceChunksUploaded' => $chunksUploaded, + 'activate' => $activate, + 'sourceMetadata' => $metadata, + 'type' => $type, + ])); + if ($isNewDeployment) { $sitesDomain = $platform['sitesDomain']; $domain = ID::unique() . "." . $sitesDomain; @@ -405,8 +374,8 @@ class Create extends Action 'domain' => $domain, 'type' => 'deployment', 'trigger' => 'deployment', - 'deploymentId' => $deployment->isEmpty() ? '' : $deployment->getId(), - 'deploymentInternalId' => $deployment->isEmpty() ? '' : $deployment->getSequence(), + 'deploymentId' => $deployment->getId(), + 'deploymentInternalId' => $deployment->getSequence(), 'deploymentResourceType' => 'site', 'deploymentResourceId' => $site->getId(), 'deploymentResourceInternalId' => $site->getSequence(), @@ -417,22 +386,7 @@ class Create extends Action 'region' => $project->getAttribute('region') ])) ); - } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ - 'sourceSize' => $fileSize, - 'sourceChunksUploaded' => $chunksUploaded, - 'sourceMetadata' => $metadata, - ])); } - - // Start the build - $publisherForBuilds->enqueue(new BuildMessage( - project: $project, - resource: $site, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - platform: $platform, - )); } else { $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ 'sourceChunksUploaded' => $chunksUploaded, diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Authorize/External/Update.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Authorize/External/Update.php index c5bdd93b26..5e8503aade 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Authorize/External/Update.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Authorize/External/Update.php @@ -9,8 +9,8 @@ use Appwrite\Platform\Modules\VCS\Http\GitHub\Deployment; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Service\Deployments; use Appwrite\Utopia\Response; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; @@ -62,7 +62,7 @@ class Update extends Action ->inject('authorization') ->inject('getProjectDB') ->inject('publisherForBuilds') - ->inject('jobs') + ->inject('deployments') ->inject('platform') ->callback($this->action(...)); } @@ -78,7 +78,7 @@ class Update extends Action Authorization $authorization, callable $getProjectDB, BuildPublisher $publisherForBuilds, - Jobs $jobs, + Deployments $deployments, array $platform ) { $installation = $dbForPlatform->getDocument('installations', $installationId); @@ -140,7 +140,7 @@ class Update extends Action ...array_filter(array_column($prFiles, 'previous_filename')) ]; - $this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, true, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs); + $this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, true, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments); $response->noContent(); } diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php index a16ad62593..3526072883 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php @@ -2,14 +2,12 @@ namespace Appwrite\Platform\Modules\VCS\Http\GitHub; -use Appwrite\Compute\Job; -use Appwrite\Event\Event; use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\Filter\BranchDomain as BranchDomainFilter; +use Appwrite\Service\Deployments; use Appwrite\Vcs\Comment; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Config\Config; use Utopia\Console; use Utopia\Database\Database; @@ -52,7 +50,7 @@ trait Deployment BuildPublisher $publisherForBuilds, callable $getProjectDB, array $platform, - ?Jobs $jobs = null, + ?Deployments $deployments = null, ) { $errors = []; foreach ($repositories as $repository) { @@ -363,25 +361,8 @@ trait Deployment $commands[] = $resource->getAttribute('commands', ''); } - // Build function deployments on the jobs-service when the caller - // opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator. Sites stay - // on the executor. On jobs the deployment is pre-declared 'waiting' - // with its buildPath so build.sh writes onto the mounted volume. - $useJobs = $jobs !== null - && $resourceCollection === 'functions' - && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'; - $buildFields = $useJobs - ? ['status' => 'waiting', 'buildPath' => Job::buildPath($projectId, $deploymentId)] - : []; - - $deployment = $authorization->skip(fn () => $dbForProject->createDocument('deployments', new Document([ + $deployment = new Document([ '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - ...$buildFields, 'resourceId' => $resourceId, 'resourceInternalId' => $resourceInternalId, 'resourceType' => $resourceCollection, @@ -409,7 +390,37 @@ trait Deployment 'providerCommentId' => \strval($latestCommentId), 'providerBranch' => $providerBranch, 'activate' => $activate, - ]))); + ]); + + // Build function deployments through $deployments (executor + // or jobs-service, decided by _APP_BUILDS_BACKEND) when the + // caller opts in. Sites always stay on the executor. + if ($deployments !== null && $resourceCollection === 'functions') { + $deployment = $authorization->skip(fn () => $deployments->createFromUrl( + $resource, + $deployment, + $github->getRepositoryPresignedUrl($providerRepositoryOwner, $providerRepositoryName, $providerCommitHash), + $resource->getAttribute('providerRootDirectory', ''), + )); + } else { + $deployment = $authorization->skip(fn () => $dbForProject->createDocument('deployments', new Document([ + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ...$deployment->getArrayCopy(), + 'status' => 'waiting', + ]))); + + $publisherForBuilds->enqueue(new BuildMessage( + project: $project, + resource: $resource, + deployment: $deployment, + type: BUILD_TYPE_DEPLOYMENT, + platform: $platform, + )); + } if ($resource->getCollection() === 'sites') { $projectId = $project->getId(); @@ -572,27 +583,6 @@ trait Deployment $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'pending', $message, $providerTargetUrl, $name); } - if ($useJobs) { - $source = [ - 'url' => $github->getRepositoryPresignedUrl($providerRepositoryOwner, $providerRepositoryName, $providerCommitHash), - 'subdir' => $resource->getAttribute('providerRootDirectory', ''), - ]; - $jobs->create(...Job::build($project, $resource, $deployment, $platform, $source)); - } else { - $queueName = $this->getBuildQueueName($project, $dbForPlatform, $authorization); - - $publisherForBuilds->enqueue( - new BuildMessage( - project: $project, - resource: $resource, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - platform: $platform, - ), - new \Utopia\Queue\Queue($queueName) - ); - } - Span::add("{$logBase}.build.triggered", 'true'); //TODO: Add event? } catch (Exception $e) { @@ -618,9 +608,4 @@ trait Deployment { } - protected function getBuildQueueName(Document $project, Database $dbForPlatform, Authorization $authorization): string - { - return System::getEnv('_APP_BUILDS_QUEUE_NAME', Event::BUILDS_QUEUE_NAME); - } - } diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Events/Create.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Events/Create.php index 11e967e058..cbda326da0 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Events/Create.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Events/Create.php @@ -6,9 +6,9 @@ use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; use Appwrite\Platform\Modules\VCS\Http\GitHub\Deployment; +use Appwrite\Service\Deployments; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Console; use Utopia\Database\Database; use Utopia\Database\Document; @@ -44,7 +44,7 @@ class Create extends Action ->inject('authorization') ->inject('getProjectDB') ->inject('publisherForBuilds') - ->inject('jobs') + ->inject('deployments') ->inject('platform') ->callback($this->action(...)); } @@ -57,7 +57,7 @@ class Create extends Action Authorization $authorization, callable $getProjectDB, BuildPublisher $publisherForBuilds, - Jobs $jobs, + Deployments $deployments, array $platform ) { $this->preprocessEvent($request); @@ -82,8 +82,8 @@ class Create extends Action match ($event) { $github::EVENT_INSTALLATION => $this->handleInstallationEvent($parsedPayload, $dbForPlatform, $authorization, $getProjectDB), - $github::EVENT_PUSH => $this->handlePushEvent($parsedPayload, $githubAppId, $privateKey, $github, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs), - $github::EVENT_PULL_REQUEST => $this->handlePullRequestEvent($parsedPayload, $privateKey, $githubAppId, $github, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs), + $github::EVENT_PUSH => $this->handlePushEvent($parsedPayload, $githubAppId, $privateKey, $github, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments), + $github::EVENT_PULL_REQUEST => $this->handlePullRequestEvent($parsedPayload, $privateKey, $githubAppId, $github, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments), default => null, }; @@ -190,7 +190,7 @@ class Create extends Action BuildPublisher $publisherForBuilds, callable $getProjectDB, array $platform, - ?Jobs $jobs = null, + ?Deployments $deployments = null, ) { $providerBranchDeleted = $parsedPayload["branchDeleted"] ?? false; $providerBranch = $parsedPayload["branch"] ?? ''; @@ -223,7 +223,7 @@ class Create extends Action // Create new deployment only on push (not committed by us) and not when branch is deleted if ($providerCommitAuthorEmail !== APP_VCS_GITHUB_EMAIL && !$providerBranchDeleted) { $providerAffectedFiles = $parsedPayload['affectedFiles'] ?? []; - $this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthorName, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', $providerAffectedFiles, false, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs); + $this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthorName, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', $providerAffectedFiles, false, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments); } } @@ -237,7 +237,7 @@ class Create extends Action BuildPublisher $publisherForBuilds, callable $getProjectDB, array $platform, - ?Jobs $jobs = null, + ?Deployments $deployments = null, ) { $action = $parsedPayload["action"] ?? ''; @@ -288,7 +288,7 @@ class Create extends Action Query::orderDesc('$createdAt') ])); - $this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, $external, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs); + $this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, $external, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments); } elseif ($action == "closed") { // Allowed external contributions cleanup diff --git a/src/Appwrite/Service/Deployments.php b/src/Appwrite/Service/Deployments.php new file mode 100644 index 0000000000..f36208a6d0 --- /dev/null +++ b/src/Appwrite/Service/Deployments.php @@ -0,0 +1,143 @@ +getSequence() === null) { + return $this->dbForProject->createDocument('deployments', new Document([ + '$permissions' => self::permissions(), + ...self::resourceFields($resource), + ...$deployment->getArrayCopy(), + ])); + } + + return $this->dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + } + + /** + * Finalizes the deployment document (see upload() for what `$deployment` + * should carry) and dispatches it for building from its own uploaded + * source. Deactivates any other active deployment for $resource, marks + * it queued (and writes whatever else the backend needs, e.g. + * buildPath). Returns the persisted, updated deployment. + */ + abstract public function createFromUpload(Document $resource, Document $deployment): Document; + + /** + * Same as createFromUpload(), but builds from a public git reference + * (a template's repository) instead of the deployment's own uploaded + * source. $reference is already resolved to a concrete commit/branch/tag + * — resolving a version range (e.g. "0.3.*") is the caller's job, since + * only it holds the GitHub client that can do so. + */ + abstract public function createFromRef( + Document $resource, + Document $deployment, + string $owner, + string $repository, + string $type, + string $reference, + string $rootDirectory = '', + ): Document; + + /** + * Same as createFromUpload(), but builds from a remote tarball at $url + * (a VCS presigned URL) instead of the deployment's own uploaded source. + */ + abstract public function createFromUrl( + Document $resource, + Document $deployment, + string $url, + string $rootDirectory = '', + ): Document; + + /** + * Best-effort cancel of an in-flight build. The deployment is already + * marked canceled by the caller; this only needs to stop the backend from + * still writing to it. + */ + abstract public function delete(string $deploymentId): void; + + /** + * Deactivates any other active deployment for $resource before this one + * goes live. Shared by both backends, called right before they mark the + * deployment queued. + */ + protected function deactivateOthers(Document $resource, Document $deployment): void + { + if (!$deployment->getAttribute('activate', false)) { + return; + } + + $others = $this->dbForProject->find('deployments', [ + Query::equal('activate', [true]), + Query::equal('resourceId', [$resource->getId()]), + Query::equal('resourceType', [$resource->getCollection()]), + Query::notEqual('$id', $deployment->getId()), + ]); + + foreach ($others as $other) { + $this->dbForProject->updateDocument('deployments', $other->getId(), new Document([ + 'activate' => false, + ])); + } + } + + /** + * @return array + */ + private static function permissions(): array + { + return [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ]; + } + + /** + * @return array + */ + private static function resourceFields(Document $resource): array + { + return [ + 'resourceInternalId' => $resource->getSequence(), + 'resourceId' => $resource->getId(), + 'resourceType' => $resource->getCollection(), + ]; + } +} diff --git a/src/Appwrite/Service/Deployments/Executor.php b/src/Appwrite/Service/Deployments/Executor.php new file mode 100644 index 0000000000..ddd7929b87 --- /dev/null +++ b/src/Appwrite/Service/Deployments/Executor.php @@ -0,0 +1,86 @@ +dispatch($resource, $deployment, null); + } + + public function createFromRef( + Document $resource, + Document $deployment, + string $owner, + string $repository, + string $type, + string $reference, + string $rootDirectory = '', + ): Document { + // No presigned URL to fetch here — the Builds worker clones the + // public repo itself via these raw git coordinates. + return $this->dispatch($resource, $deployment, new Document([ + 'ownerName' => $owner, + 'repositoryName' => $repository, + 'referenceType' => $type, + 'referenceValue' => $reference, + 'rootDirectory' => $rootDirectory, + ])); + } + + public function createFromUrl( + Document $resource, + Document $deployment, + string $url, + string $rootDirectory = '', + ): Document { + // The Builds worker clones the repo via the deployment's own + // installationId/providerRepositoryId instead of $url. + return $this->dispatch($resource, $deployment, null); + } + + private function dispatch(Document $resource, Document $deployment, ?Document $template): Document + { + $deployment = $this->upload($resource, $deployment); + $this->deactivateOthers($resource, $deployment); + + // The Builds worker promotes 'waiting' → 'building' on pickup. + $deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ + 'status' => 'waiting', + ])); + + $this->publisherForBuilds->enqueue(new BuildMessage( + project: $this->project, + resource: $resource, + deployment: $deployment, + type: BUILD_TYPE_DEPLOYMENT, + template: $template, + platform: $this->platform, + )); + + return $deployment; + } + + public function delete(string $deploymentId): void + { + $this->executor->deleteRuntime($this->project->getId(), $deploymentId . '-build'); + } +} diff --git a/src/Appwrite/Compute/Job.php b/src/Appwrite/Service/Deployments/Jobs.php similarity index 62% rename from src/Appwrite/Compute/Job.php rename to src/Appwrite/Service/Deployments/Jobs.php index ac8d2ce84e..854db9ec31 100644 --- a/src/Appwrite/Compute/Job.php +++ b/src/Appwrite/Service/Deployments/Jobs.php @@ -1,45 +1,108 @@ submit($resource, $deployment, null); + } + + public function createFromRef( + Document $resource, + Document $deployment, + string $owner, + string $repository, + string $type, + string $reference, + string $rootDirectory = '', + ): Document { + // The jobs-service has no GitHub client of its own — it only fetches + // tarballs — so $reference must already be a concrete commit/branch/ + // tag; codeload only understands one ref per tarball, not a range. + $url = "https://codeload.github.com/{$owner}/{$repository}/tar.gz/{$reference}"; + + return $this->submit($resource, $deployment, ['url' => $url, 'subdir' => $rootDirectory]); + } + + public function createFromUrl( + Document $resource, + Document $deployment, + string $url, + string $rootDirectory = '', + ): Document { + return $this->submit($resource, $deployment, ['url' => $url, 'subdir' => $rootDirectory]); + } + + private function submit(Document $resource, Document $deployment, ?array $source): Document + { + $deployment = $this->upload($resource, $deployment); + $this->deactivateOthers($resource, $deployment); + + // Pre-declare buildPath so build.sh writes output straight onto the + // mounted builds volume, at the path this deployment expects. + $deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ + 'status' => 'waiting', + 'buildPath' => static::buildPath($this->project->getId(), $deployment->getId()), + ])); + + $this->jobs->create(...static::payload($this->project, $resource, $deployment, $this->platform, $source)); + + return $deployment; + } + + public function delete(string $deploymentId): void + { + $this->jobs->delete(static::id($this->project->getId(), $deploymentId)); + } + /** * @return array Named arguments for OpenRuntimes\Orchestrator\Jobs::create(). */ - /** - * @param ?array{url: string, subdir?: string} $source Remote tarball source - * (template codeload URL, or a VCS presigned URL). When null, the source is - * the deployment's own uploaded tarball, fetched from Appwrite via a - * presigned token (manual upload / duplicate). - */ - public static function build( + public static function payload( Document $project, Document $function, Document $deployment, @@ -97,20 +160,18 @@ final class Job ]; } - // Output + cache live on the mounted builds volume (see class doc): the - // build writes code.tar.gz to the deployment's build dir and the cache - // squashfs to its per-function keyed path — both directly on the volume - // Appwrite reads, so nothing large crosses the HTTP boundary. - $cacheKey = self::cacheKey($projectId, $function->getId(), $runtime['image'] ?? ''); + // Where output + cache land is a swappable strategy (see storage()) — + // the default mounts the shared builds volume; nothing else here cares + // which strategy is active. + $output = static::storage($project, $function, $deployment); + $command = $deployment->getAttribute('buildCommands', ''); $env = self::variables($project, $function, $deployment, $runtime, $cpus, $memory, $endpoint, $timeout) + [ 'OPEN_RUNTIMES_BUILD_INPUT_DIR' => '/mnt/code/source', - 'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => \dirname(self::buildPath($projectId, $deploymentId)), - 'OPEN_RUNTIMES_BUILD_CACHE_ARTIFACT' => self::cachePath($projectId, $cacheKey), - ]; + ] + $output['environment']; return [ - 'id' => self::id($projectId, $deploymentId), + 'id' => static::id($projectId, $deploymentId), 'image' => $runtime['image'], 'command' => '/usr/local/server/helpers/build.sh ' . \escapeshellarg($command), 'cpu' => $cpus, @@ -125,13 +186,8 @@ final class Job ], // The orchestrator expects environment as a string->string map. 'environment' => \array_map('strval', $env), - 'artifacts' => $sourceArtifacts, - // Attach the builds storage volume (Docker volume / K8s PVC named by - // _APP_BUILDS_VOLUME) to the worker at its Appwrite path so build.sh - // writes output + cache straight onto it. - 'volumes' => [ - new Volume(source: System::getEnv('_APP_BUILDS_VOLUME', 'appwrite-builds'), path: APP_STORAGE_BUILDS), - ], + 'artifacts' => [...$sourceArtifacts, ...$output['artifacts']], + 'volumes' => $output['volumes'], 'callback' => new Callback( url: "{$endpoint}/v1/jobs/event?" . \http_build_query(['project' => $projectId]), // Artifact callbacks only carry the source-size stat, which exists @@ -176,6 +232,44 @@ final class Job return APP_STORAGE_BUILDS . "/app-{$projectId}/cache/{$cacheKey}.sqfs"; } + /** + * Where build.sh's output (code.tar.gz) and package-manager cache + * (a squashfs) land, and what the job needs to get them there. The + * default mounts the shared builds volume at buildPath()/cachePath(); + * build.sh only cares that OPEN_RUNTIMES_BUILD_OUTPUT_DIR/_CACHE_ARTIFACT + * point somewhere on its local filesystem, volume-backed or not — so a + * strategy without a shared volume (e.g. S3) instead points them at a + * local tmp path and moves things in/out via 'artifacts': + * - cache pull, before the build: a plain DownloadArtifact (no + * `depends`, so it runs before the command) into the local cache path. + * - cache push and output upload, after the build: an UploadArtifact + * with `depends: 'job'` — 'job' is the orchestrator's sentinel id for + * "after the build command finishes", not an id of another artifact. + * + * @return array{volumes: array, artifacts: array, environment: array} + */ + protected static function storage(Document $project, Document $resource, Document $deployment): array + { + $projectId = $project->getId(); + $deploymentId = $deployment->getId(); + $runtime = self::runtime($resource, $resource->getAttribute('version', 'v2')); + $cacheKey = static::cacheKey($projectId, $resource->getId(), $runtime['image'] ?? ''); + + return [ + // Docker volume / K8s PVC named by _APP_BUILDS_VOLUME, attached + // to the worker at its Appwrite path so build.sh writes output + + // cache straight onto it. + 'volumes' => [ + new Volume(source: System::getEnv('_APP_BUILDS_VOLUME', 'appwrite-builds'), path: APP_STORAGE_BUILDS), + ], + 'artifacts' => [], + 'environment' => [ + 'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => \dirname(static::buildPath($projectId, $deploymentId)), + 'OPEN_RUNTIMES_BUILD_CACHE_ARTIFACT' => static::cachePath($projectId, $cacheKey), + ], + ]; + } + private static function runtime(Document $function, string $version): array { $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []);