From a82dc6e386a6ac7b88e853a1e6fdb59e0b79ddca Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:09:12 +0100 Subject: [PATCH 1/3] chore: remove the executor build backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds have run on the open-runtimes jobs-service for a while (_APP_BUILDS_BACKEND=orchestrator); this removes the switch and the executor build path behind it. - Merge `Deployment\Backend` and `Backend\Orchestrator` into a single `Deployment\Deployments` service and delete `Backend\Executor`. - The Builds worker's backend check becomes an unconditional hand-off to the jobs-service, which leaves everything after it unreachable: the executor create/log coroutines, tarball packing, adapter detection, activation and fan-out are all deleted. The worker keeps only the template-into-repo git push (a git *write* the artifact system has no primitive for), after which the Jobs worker owns the deployment's lifecycle. - Route createFunction's deprecated `template*` params through `createFromRef()` — the worker's non-VCS branch was their only build path. Share the template ref-range resolution the three template callers need as `Compute\Base::resolveTemplateRef()`. - Drop the now always-true `instanceof Orchestrator` branch from the deployment status endpoints, and `Executor::createRuntime()`/`getLogs()`. Behaviour note: template-into-repo builds no longer enforce a source size limit, matching plain VCS builds since they moved to the jobs-service. Verified with composer lint, composer analyze, tests/unit, and the full Functions and Sites e2e suites. Co-Authored-By: Claude Opus 5 (1M context) --- .env | 1 - app/config/variables.php | 11 +- app/init/resources.php | 5 +- app/init/resources/request.php | 16 +- app/init/worker/message.php | 6 +- docker-compose.yml | 2 - src/Appwrite/Deployment/Backend.php | 164 --- src/Appwrite/Deployment/Backend/Executor.php | 87 -- .../Orchestrator.php => Deployments.php} | 139 ++- .../Platform/Modules/Compute/Base.php | 37 +- .../Functions/Http/Deployments/Create.php | 4 +- .../Http/Deployments/Duplicate/Create.php | 4 +- .../Http/Deployments/Status/Update.php | 30 +- .../Http/Deployments/Template/Create.php | 27 +- .../Functions/Http/Deployments/Vcs/Create.php | 4 +- .../Functions/Http/Functions/Create.php | 59 +- .../Functions/Http/Functions/Update.php | 4 +- .../Functions/Workers/BuildException.php | 2 +- .../Modules/Functions/Workers/Builds.php | 1069 ++--------------- .../Modules/Functions/Workers/Jobs.php | 3 +- .../Modules/Sites/Http/Deployments/Create.php | 4 +- .../Http/Deployments/Duplicate/Create.php | 4 +- .../Sites/Http/Deployments/Status/Update.php | 30 +- .../Http/Deployments/Template/Create.php | 20 +- .../Sites/Http/Deployments/Vcs/Create.php | 4 +- .../Modules/Sites/Http/Sites/Update.php | 4 +- .../Modules/VCS/Http/GitHub/Deployment.php | 2 +- src/Appwrite/Usage/Build.php | 6 +- src/Executor/Executor.php | 94 -- .../{BackendTest.php => DeploymentsTest.php} | 8 +- 30 files changed, 373 insertions(+), 1477 deletions(-) delete mode 100644 src/Appwrite/Deployment/Backend.php delete mode 100644 src/Appwrite/Deployment/Backend/Executor.php rename src/Appwrite/Deployment/{Backend/Orchestrator.php => Deployments.php} (78%) rename tests/unit/Deployment/{BackendTest.php => DeploymentsTest.php} (81%) diff --git a/.env b/.env index b48a023df1..4a4b7245a1 100644 --- a/.env +++ b/.env @@ -117,7 +117,6 @@ _APP_SITES_TIMEOUT=30 _APP_EXECUTOR_HOST=http://exc1/v1 _APP_EXECUTOR_SECRET=your-secret-key _APP_EXECUTOR_IMAGES=openruntimes/node:v5-22,openruntimes/static:v5-1 -_APP_BUILDS_BACKEND=orchestrator _APP_BUILDS_VOLUME=appwrite-builds _APP_JOBS_HOST=http://orchestrator-jobs:8080 _APP_JOBS_SECRET=your-secret-key diff --git a/app/config/variables.php b/app/config/variables.php index e25a0f8fd2..789890ae55 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -988,18 +988,9 @@ return [ 'question' => '', 'filter' => '' ], - [ - 'name' => '_APP_BUILDS_BACKEND', - 'description' => 'Backend that builds manual-upload function deployments: "executor" (default; the open-runtimes executor, via the Builds worker) or "orchestrator" (the open-runtimes jobs-service, submitted in the request flow). Other build flows always use the executor.', - 'introduction' => '1.9.0', - 'default' => 'executor', - 'required' => false, - 'question' => '', - 'filter' => '' - ], [ 'name' => '_APP_BUILDS_VOLUME', - 'description' => 'The Docker volume (or Kubernetes PersistentVolumeClaim) holding build storage, attached to jobs-service build workers so they write output directly onto it. Must match the storage the "builds" device is backed by. Only used when _APP_BUILDS_BACKEND is "orchestrator".', + 'description' => 'The Docker volume (or Kubernetes PersistentVolumeClaim) holding build storage, attached to jobs-service build workers so they write output directly onto it. Must match the storage the "builds" device is backed by.', 'introduction' => '1.9.0', 'default' => 'appwrite-builds', 'required' => false, diff --git a/app/init/resources.php b/app/init/resources.php index cc01f3df40..f11e96c0ce 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -77,9 +77,8 @@ $container->set('jobs', function () { ->withBearerAuth(System::getEnv('_APP_JOBS_SECRET', '')) ->withTimeout(30); - // No host on executor-only installs: keep the injection resolvable and - // fail at call time instead (the client is only used when - // _APP_BUILDS_BACKEND=orchestrator, which requires _APP_JOBS_HOST). + // Keep the injection resolvable without _APP_JOBS_HOST and fail at call + // time instead, so installs that never build stay bootable. $host = System::getEnv('_APP_JOBS_HOST', ''); if ($host !== '') { $client = $client->withBaseUri($host); diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 3208c0e096..f64c8dcd23 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -5,13 +5,10 @@ use Ahc\Jwt\JWTException; use Appwrite\Auth\Key; use Appwrite\Database\Factory as DatabaseFactory; use Appwrite\Databases\TransactionState; -use Appwrite\Deployment\Backend; -use Appwrite\Deployment\Backend\Executor as ExecutorBackend; -use Appwrite\Deployment\Backend\Orchestrator; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Context\Audit as AuditContext; use Appwrite\Event\Event; use Appwrite\Event\Message\Func as FunctionMessage; -use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Event\Publisher\Func as FunctionPublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Webhook; @@ -28,7 +25,6 @@ use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; -use Executor\Executor; use OpenRuntimes\Orchestrator\Jobs; use Utopia\Agents\Adapters\Appwrite as AppwriteAdapter; use Utopia\Agents\Agent; @@ -197,13 +193,11 @@ return function (Container $context): void { $publisher, new Queue(System::getEnv('_APP_FUNCTIONS_QUEUE_NAME', Event::FUNCTIONS_QUEUE_NAME), 'utopia-queue', Event::FUNCTIONS_QUEUE_TTL) ), ['publisher']); - // Builds a Backend bound to a given project — webhook handlers resolve + // Builds a Deployments bound to a given project — webhook handlers resolve // their tenant projects mid-request, after this container is initialized. - $context->set('deploymentsFactory', function (BuildPublisher $publisherForBuilds, Jobs $jobs, Executor $executor, array $platform) { - return fn (Database $dbForProject, Document $project): Backend => System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator' - ? new Orchestrator($jobs, $dbForProject, $project, $platform) - : new ExecutorBackend($publisherForBuilds, $dbForProject, $project, $executor, $platform); - }, ['publisherForBuilds', 'jobs', 'executor', 'platform']); + $context->set('deploymentsFactory', function (Jobs $jobs, array $platform) { + return fn (Database $dbForProject, Document $project): Deployments => new Deployments($jobs, $dbForProject, $project, $platform); + }, ['jobs', 'platform']); $context->set('deployments', fn (callable $deploymentsFactory, Database $dbForProject, Document $project) => $deploymentsFactory($dbForProject, $project), ['deploymentsFactory', 'dbForProject', 'project']); $context->set('eventProcessor', fn () => new EventProcessor(), []); $context->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory( diff --git a/app/init/worker/message.php b/app/init/worker/message.php index d581820549..6f6fc39eef 100644 --- a/app/init/worker/message.php +++ b/app/init/worker/message.php @@ -1,7 +1,7 @@ set('deployments', function (Jobs $jobs, Database $dbForProject, Document $project, array $platform) { - return new Orchestrator($jobs, $dbForProject, $project, $platform); + return new Deployments($jobs, $dbForProject, $project, $platform); }, ['jobs', 'dbForProject', 'project', 'platform']); $container->set('logError', function (Registry $register, Document $project) { diff --git a/docker-compose.yml b/docker-compose.yml index b415610233..55dc9b0c0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -171,7 +171,6 @@ services: - _APP_DOMAIN_SITES - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - - _APP_BUILDS_BACKEND - _APP_BUILDS_VOLUME - _APP_JOBS_HOST - _APP_JOBS_SECRET @@ -538,7 +537,6 @@ services: - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES - _APP_DOMAIN_SITES - - _APP_BUILDS_BACKEND - _APP_BUILDS_VOLUME - _APP_JOBS_HOST - _APP_JOBS_SECRET diff --git a/src/Appwrite/Deployment/Backend.php b/src/Appwrite/Deployment/Backend.php deleted file mode 100644 index a8f29ae8f0..0000000000 --- a/src/Appwrite/Deployment/Backend.php +++ /dev/null @@ -1,164 +0,0 @@ -getSequence() === null) { - return $this->dbForProject->createDocument('deployments', new Document([ - '$permissions' => self::permissions(), - ...self::resourceFields($resource), - ...$deployment->getArrayCopy(), - ])); - } - - return $this->dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); - } - - /** - * Finalizes the deployment document (see upload() for what `$deployment` - * should carry) and dispatches it for building from its own uploaded - * source. Deactivates any other active deployment for $resource, marks - * it queued (and writes whatever else the backend needs, e.g. - * buildPath). Returns the persisted, updated deployment. - */ - abstract public function createFromUpload(Document $resource, Document $deployment): Document; - - /** - * Same as createFromUpload(), but builds from a public git reference - * (a template's repository) instead of the deployment's own uploaded - * source. $reference is already resolved to a concrete commit/branch/tag - * — resolving a version range (e.g. "0.3.*") is the caller's job, since - * only it holds the GitHub client that can do so. - */ - abstract public function createFromRef( - Document $resource, - Document $deployment, - string $owner, - string $repository, - string $type, - string $reference, - string $rootDirectory = '', - ): Document; - - /** - * Same as createFromUpload(), but builds from a remote tarball at $url - * (a VCS presigned URL) instead of the deployment's own uploaded source. - */ - abstract public function createFromUrl( - Document $resource, - Document $deployment, - string $url, - string $rootDirectory = '', - ): Document; - - /** - * Best-effort cancel of an in-flight build. The deployment is already - * marked canceled by the caller; this only needs to stop the backend from - * still writing to it. - */ - abstract public function cancel(string $deploymentId): void; - - /** - * Deactivates any other active deployment for $resource before this one - * goes live. Shared by both backends, called right before they mark the - * deployment queued. - */ - protected function deactivateOthers(Document $resource, Document $deployment): void - { - if (!$deployment->getAttribute('activate', false)) { - return; - } - - $others = $this->dbForProject->find('deployments', [ - Query::equal('activate', [true]), - Query::equal('resourceId', [$resource->getId()]), - Query::equal('resourceType', [$resource->getCollection()]), - Query::notEqual('$id', $deployment->getId()), - ]); - - foreach ($others as $other) { - $this->dbForProject->updateDocument('deployments', $other->getId(), new Document([ - 'activate' => false, - ])); - } - } - - /** - * The build command for a deployment: its buildCommands, wrapped for - * sites with the framework's env and bundle commands. - */ - public static function command(Document $resource, Document $deployment): string - { - $command = $deployment->getAttribute('buildCommands', ''); - if ($resource->getCollection() !== 'sites') { - return $command; - } - - $framework = Config::getParam('frameworks', [])[$resource->getAttribute('framework', '')] ?? []; - - return \implode(' && ', \array_filter([ - $framework['envCommand'] ?? '', - $command, - $framework['bundleCommand'] ?? '', - ], fn ($command) => !empty($command))); - } - - /** - * @return array - */ - private static function permissions(): array - { - return [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ]; - } - - /** - * @return array - */ - private static function resourceFields(Document $resource): array - { - return [ - 'resourceInternalId' => $resource->getSequence(), - 'resourceId' => $resource->getId(), - 'resourceType' => $resource->getCollection(), - ]; - } -} diff --git a/src/Appwrite/Deployment/Backend/Executor.php b/src/Appwrite/Deployment/Backend/Executor.php deleted file mode 100644 index f665d9e979..0000000000 --- a/src/Appwrite/Deployment/Backend/Executor.php +++ /dev/null @@ -1,87 +0,0 @@ -dispatch($resource, $deployment, null); - } - - public function createFromRef( - Document $resource, - Document $deployment, - string $owner, - string $repository, - string $type, - string $reference, - string $rootDirectory = '', - ): Document { - // No presigned URL to fetch here — the Builds worker clones the - // public repo itself via these raw git coordinates. - return $this->dispatch($resource, $deployment, new Document([ - 'ownerName' => $owner, - 'repositoryName' => $repository, - 'referenceType' => $type, - 'referenceValue' => $reference, - 'rootDirectory' => $rootDirectory, - ])); - } - - public function createFromUrl( - Document $resource, - Document $deployment, - string $url, - string $rootDirectory = '', - ): Document { - // The Builds worker clones the repo via the deployment's own - // installationId/providerRepositoryId instead of $url. - return $this->dispatch($resource, $deployment, null); - } - - private function dispatch(Document $resource, Document $deployment, ?Document $template): Document - { - $deployment = $this->upload($resource, $deployment); - $this->deactivateOthers($resource, $deployment); - - // The Builds worker promotes 'waiting' → 'building' on pickup. - $deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'status' => 'waiting', - ])); - - $this->publisherForBuilds->enqueue(new BuildMessage( - project: $this->project, - resource: $resource, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - template: $template, - platform: $this->platform, - )); - - return $deployment; - } - - public function cancel(string $deploymentId): void - { - $this->executor->deleteRuntime($this->project->getId(), $deploymentId . '-build'); - } -} diff --git a/src/Appwrite/Deployment/Backend/Orchestrator.php b/src/Appwrite/Deployment/Deployments.php similarity index 78% rename from src/Appwrite/Deployment/Backend/Orchestrator.php rename to src/Appwrite/Deployment/Deployments.php index 6d30f2d6a4..4c18f3860c 100644 --- a/src/Appwrite/Deployment/Backend/Orchestrator.php +++ b/src/Appwrite/Deployment/Deployments.php @@ -1,10 +1,8 @@ getSequence() === null) { + return $this->dbForProject->createDocument('deployments', new Document([ + '$permissions' => self::permissions(), + ...self::resourceFields($resource), + ...$deployment->getArrayCopy(), + ])); + } + return $this->dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + } + + /** + * Finalizes the deployment document (see upload() for what `$deployment` + * should carry) and dispatches it for building from its own uploaded + * source. Deactivates any other active deployment for $resource, marks + * it queued and writes its buildPath. Returns the persisted, updated + * deployment. + */ public function createFromUpload(Document $resource, Document $deployment): Document { return $this->submit($resource, $deployment, null); } + /** + * Same as createFromUpload(), but builds from a public git reference + * (a template's repository) instead of the deployment's own uploaded + * source. $reference is already resolved to a concrete commit/branch/tag + * — resolving a version range (e.g. "0.3.*") is the caller's job, since + * only it holds the GitHub client that can do so. + */ public function createFromRef( Document $resource, Document $deployment, @@ -73,6 +113,10 @@ readonly class Orchestrator extends Backend return $this->submit($resource, $deployment, ['url' => $url, 'subdir' => $rootDirectory]); } + /** + * Same as createFromUpload(), but builds from a remote tarball at $url + * (a VCS presigned URL) instead of the deployment's own uploaded source. + */ public function createFromUrl( Document $resource, Document $deployment, @@ -107,11 +151,60 @@ readonly class Orchestrator extends Backend return $deployment; } + /** + * Best-effort cancel of an in-flight build. The deployment is already + * marked canceled by the caller; this only needs to stop the backend from + * still writing to it. + */ public function cancel(string $deploymentId): void { $this->jobs->delete(static::id($this->project->getId(), $deploymentId)); } + /** + * Deactivates any other active deployment for $resource before this one + * goes live, called right before the deployment is marked queued. + */ + protected function deactivateOthers(Document $resource, Document $deployment): void + { + if (!$deployment->getAttribute('activate', false)) { + return; + } + + $others = $this->dbForProject->find('deployments', [ + Query::equal('activate', [true]), + Query::equal('resourceId', [$resource->getId()]), + Query::equal('resourceType', [$resource->getCollection()]), + Query::notEqual('$id', $deployment->getId()), + ]); + + foreach ($others as $other) { + $this->dbForProject->updateDocument('deployments', $other->getId(), new Document([ + 'activate' => false, + ])); + } + } + + /** + * The build command for a deployment: its buildCommands, wrapped for + * sites with the framework's env and bundle commands. + */ + public static function command(Document $resource, Document $deployment): string + { + $command = $deployment->getAttribute('buildCommands', ''); + if ($resource->getCollection() !== 'sites') { + return $command; + } + + $framework = Config::getParam('frameworks', [])[$resource->getAttribute('framework', '')] ?? []; + + return \implode(' && ', \array_filter([ + $framework['envCommand'] ?? '', + $command, + $framework['bundleCommand'] ?? '', + ], fn ($command) => !empty($command))); + } + /** * @return array Named arguments for OpenRuntimes\Orchestrator\Jobs::create(). */ @@ -398,4 +491,28 @@ readonly class Orchestrator extends Backend 'OPEN_RUNTIMES_NFT' => System::getEnv('_APP_OPEN_RUNTIMES_NFT', 'enabled'), ]); } + + /** + * @return array + */ + private static function permissions(): array + { + return [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ]; + } + + /** + * @return array + */ + private static function resourceFields(Document $resource): array + { + return [ + 'resourceInternalId' => $resource->getSequence(), + 'resourceId' => $resource->getId(), + 'resourceType' => $resource->getCollection(), + ]; + } } diff --git a/src/Appwrite/Platform/Modules/Compute/Base.php b/src/Appwrite/Platform/Modules/Compute/Base.php index ed69997d8c..d38efa41f1 100644 --- a/src/Appwrite/Platform/Modules/Compute/Base.php +++ b/src/Appwrite/Platform/Modules/Compute/Base.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Compute; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -10,6 +10,7 @@ use Appwrite\Filter\BranchDomain as BranchDomainFilter; use Appwrite\Platform\Action; use Appwrite\Platform\Modules\Compute\Validator\Specification as SpecificationValidator; use Appwrite\Platform\Permission as AppwritePermission; +use Appwrite\Vcs\Factory as VcsFactory; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; @@ -68,7 +69,29 @@ class Base extends Action return $allowedSpecifications[0]; } - public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, Backend $deployments, array $platform = [], string $referenceType = 'branch', string $reference = ''): Document + /** + * Templates can pin a version range (e.g. "0.3.*"); codeload only takes a + * concrete ref, so resolve a range to the highest matching tag. Templates + * are public github.com repositories regardless of the resource's own + * provider, so this always uses the GitHub adapter. + */ + public static function resolveTemplateRef(VcsFactory $vcsFactory, string $owner, string $repository, string $type, string $reference): string + { + if ($type !== Git::CLONE_TYPE_TAG || ! \str_contains($reference, '*')) { + return $reference; + } + + try { + $tags = $vcsFactory->fromProvider('github')->listTags($owner, $repository, $reference); + + return \end($tags) ?: $reference; + } catch (\Throwable) { + // Fall back to the raw reference; the build surfaces a bad ref. + return $reference; + } + } + + public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, Deployments $deployments, array $platform = [], string $referenceType = 'branch', string $reference = ''): Document { $deploymentId = ID::unique(); $entrypoint = $function->getAttribute('entrypoint', ''); @@ -144,11 +167,9 @@ class Base extends Action 'activate' => $activate, ]); - // Build a plain (non-template) VCS deployment through $deployments - // (executor or jobs-service, decided by _APP_BUILDS_BACKEND). - // Template-into-repo pushes go through the Builds worker (which does - // the git write, then hands the build to the jobs-service itself - // when on orchestrator). + // Build a plain (non-template) VCS deployment through $deployments. + // Template-into-repo pushes go through the Builds worker, which does the + // git write and then hands the build to the jobs-service itself. if ($template->isEmpty()) { $ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch'); $deployment = $deployments->createFromUrl( @@ -181,7 +202,7 @@ class Base extends Action return $deployment; } - public function redeployVcsSite(Request $request, Document $site, Document $project, Document $installation, Database $dbForProject, Database $dbForPlatform, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, Authorization $authorization, Backend $deployments, array $platform, string $referenceType = 'branch', string $reference = ''): Document + public function redeployVcsSite(Request $request, Document $site, Document $project, Document $installation, Database $dbForProject, Database $dbForPlatform, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, Authorization $authorization, Deployments $deployments, array $platform, string $referenceType = 'branch', string $reference = ''): Document { $deploymentId = ID::unique(); $providerInstallationId = $installation->getAttribute('providerInstallationId', ''); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php index 499d18248a..dce271a92f 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; @@ -106,7 +106,7 @@ class Create extends Action Document $project, Device $deviceForFunctions, Device $deviceForLocal, - Backend $deployments, + Deployments $deployments, array $plan, Authorization $authorization, callable $locks diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php index 96aa02c9ed..a0721d56cf 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Duplicate; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; @@ -78,7 +78,7 @@ class Create extends Action Database $dbForProject, Database $dbForPlatform, Event $queueForEvents, - Backend $deployments, + Deployments $deployments, Device $deviceForFunctions, VcsFactory $vcsFactory, ) { diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php index 968381057c..5342fe1381 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php @@ -2,8 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Status; -use Appwrite\Deployment\Backend; -use Appwrite\Deployment\Backend\Orchestrator; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; @@ -72,7 +71,7 @@ class Update extends Action Response $response, Database $dbForProject, Event $queueForEvents, - Backend $deployments, + Deployments $deployments, callable $locks, ) { $function = $dbForProject->getDocument('functions', $functionId); @@ -98,9 +97,9 @@ class Update extends Action // Write under the Jobs worker's per-deployment lock: its handlers // read-modify-write buildLogs, and an unserialized cancel write here // loses the closing log line to an in-flight append. - $cancel = function () use ($dbForProject, $deployment, $duration, $deployments) { + $cancel = function () use ($dbForProject, $deployment, $duration) { try { - return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); + return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration))); } catch (TransactionException) { $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); @@ -113,7 +112,7 @@ class Update extends Action } if ($deployment->getAttribute('status') !== 'canceled') { - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration))); } return $deployment; @@ -140,25 +139,20 @@ class Update extends Action } /** - * The sparse update marking a build canceled. Jobs-backed builds have no - * cancel worker to write the closing log line the executor's Builds worker - * adds, so it is appended here; executor deployments get it from their worker. + * The sparse update marking a build canceled. No worker writes the closing + * log line for a canceled build, so it is appended here. * * @return array */ - private function cancel(Document $deployment, int $duration, bool $appendLog): array + private function cancel(Document $deployment, int $duration): array { - $update = [ + $logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n"; + + return [ 'buildEndedAt' => DateTime::now(), 'buildDuration' => $duration, 'status' => 'canceled', + 'buildLogs' => \substr($logs, -APP_LOG_LENGTH_LIMIT), ]; - - if ($appendLog) { - $logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n"; - $update['buildLogs'] = \substr($logs, -APP_LOG_LENGTH_LIMIT); - } - - return $update; } } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php index 8ce0644678..39397ef8e2 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Template; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -24,7 +24,6 @@ use Utopia\Platform\Scope\HTTP; use Utopia\Validator\Boolean; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; -use Utopia\VCS\Adapter\Git\GitHub; class Create extends Base { @@ -102,7 +101,7 @@ class Create extends Base Document $project, BuildPublisher $publisherForBuilds, VcsFactory $vcsFactory, - Backend $deployments, + Deployments $deployments, Authorization $authorization, array $platform ) { @@ -126,9 +125,7 @@ class Create extends Base if (!empty($function->getAttribute('providerRepositoryId'))) { // VCS-connected function: the Builds worker merges the template into - // the user's repo, pushes it as a commit, then builds that commit — - // on the executor itself, or by submitting a job when - // _APP_BUILDS_BACKEND=orchestrator. + // the user's repo, pushes it as a commit, then builds that commit. $installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId')); $deployment = $this->redeployVcsFunction( @@ -160,24 +157,10 @@ class Create extends Base $deploymentId = ID::unique(); - // Templates can pin a version range (e.g. "0.3.*"); codeload only - // takes a concrete ref, so resolve the range to the highest matching - // tag (mirrors the executor's `git ls-remote --tags | tail -1`). - // Templates are public github.com repositories regardless of the - // function's own provider, so this always uses the GitHub adapter. - $ref = $reference; - if ($type === GitHub::CLONE_TYPE_TAG && \str_contains($reference, '*')) { - try { - $tags = $vcsFactory->fromProvider('github')->listTags($owner, $repository, $reference); - $ref = \end($tags) ?: $reference; - } catch (\Throwable) { - // Fall back to the raw reference; the build surfaces a bad ref. - } - } + $ref = Base::resolveTemplateRef($vcsFactory, $owner, $repository, $type, $reference); // Public template: pull the source straight from GitHub's public repo - // (codeload tarball on the jobs backend, a plain git clone on the - // executor); unarchive/checkout strips down to the rootDirectory. + // as a codeload tarball; unarchive strips down to the rootDirectory. $deployment = $deployments->createFromRef( $function, new Document([ diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php index f539373048..3c54b38ebc 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Vcs; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -93,7 +93,7 @@ class Create extends Base Event $queueForEvents, BuildPublisher $publisherForBuilds, VcsFactory $vcsFactory, - Backend $deployments, + Deployments $deployments, array $platform, ) { $function = $dbForProject->getDocument('functions', $functionId); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php index d486ce2e8b..eb23873b79 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php @@ -2,9 +2,8 @@ namespace Appwrite\Platform\Modules\Functions\Http\Functions; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; -use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Message\Func as FunctionMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Event\Publisher\Func as FunctionPublisher; @@ -30,8 +29,6 @@ use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Helpers\ID; -use Utopia\Database\Helpers\Permission; -use Utopia\Database\Helpers\Role; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Roles; use Utopia\Http\Request; @@ -172,7 +169,7 @@ class Create extends Base Document $project, Event $queueForEvents, BuildPublisher $publisherForBuilds, - Backend $deployments, + Deployments $deployments, Realtime $queueForRealtime, Webhook $queueForWebhooks, FunctionPublisher $publisherForFunctions, @@ -372,33 +369,33 @@ class Create extends Base ); } elseif (!$template->isEmpty()) { - // Deploy non-VCS from template - $deploymentId = ID::unique(); - $deployment = $dbForProject->createDocument('deployments', new Document([ - '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceId' => $function->getId(), - 'resourceInternalId' => $function->getSequence(), - 'resourceType' => 'functions', - 'entrypoint' => $function->getAttribute('entrypoint', ''), - 'buildCommands' => $function->getAttribute('commands', ''), - 'startCommand' => $function->getAttribute('startCommand', ''), - 'type' => 'manual', - 'activate' => true, - ])); + // Deploy non-VCS from the template's public GitHub repository. + $templateVersion = Base::resolveTemplateRef($vcsFactory, $templateOwner, $templateRepository, Git::CLONE_TYPE_TAG, $templateVersion); - $publisherForBuilds->enqueue(new BuildMessage( - project: $project, - resource: $function, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - template: $template, - platform: $platform, - )); + $deployment = $deployments->createFromRef( + $function, + new Document([ + '$id' => ID::unique(), + 'entrypoint' => $function->getAttribute('entrypoint', ''), + 'buildCommands' => $function->getAttribute('commands', ''), + 'startCommand' => $function->getAttribute('startCommand', ''), + 'providerRepositoryName' => $templateRepository, + 'providerRepositoryOwner' => $templateOwner, + 'providerRepositoryUrl' => "https://github.com/{$templateOwner}/{$templateRepository}", + 'providerBranchUrl' => "https://github.com/{$templateOwner}/{$templateRepository}/blob/{$templateVersion}", + // The coordinates a redeploy needs: remote-source builds + // never store a tarball. + 'providerBranch' => $templateVersion, + 'providerRootDirectory' => $templateRootDirectory, + 'type' => 'vcs', + 'activate' => true, + ]), + $templateOwner, + $templateRepository, + Git::CLONE_TYPE_TAG, + $templateVersion, + $templateRootDirectory, + ); } $functionsDomain = $platform['functionsDomain']; diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php index a6f7c3e047..1fa7583f18 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Functions; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Event\Validator\FunctionEvent; @@ -153,7 +153,7 @@ class Update extends Base Document $project, Event $queueForEvents, BuildPublisher $publisherForBuilds, - Backend $deployments, + Deployments $deployments, Database $dbForPlatform, VcsFactory $vcsFactory, RepositoryWebhooks $repositoryWebhooks, diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/BuildException.php b/src/Appwrite/Platform/Modules/Functions/Workers/BuildException.php index 1496fd7b72..51a854e2f2 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/BuildException.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/BuildException.php @@ -9,7 +9,7 @@ use Appwrite\Extend\Exception; * * Every build error the worker raises intentionally is a BuildException, so the * failure handler can show its message to users while masking any other - * throwable (e.g. database or executor failures) behind a generic message. + * throwable (e.g. database or jobs-service failures) behind a generic message. */ class BuildException extends Exception { diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index b5192b7628..6134155a9b 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -2,28 +2,14 @@ namespace Appwrite\Platform\Modules\Functions\Workers; -use Ahc\Jwt\JWT; -use Appwrite\Deployment\Backend; -use Appwrite\Deployment\Detection; +use Appwrite\Deployment\Deployments; use Appwrite\Deployment\GitAction; -use Appwrite\Event\Event; -use Appwrite\Event\Message\Func as FunctionMessage; -use Appwrite\Event\Publisher\Func as FunctionPublisher; -use Appwrite\Event\Publisher\Screenshot; use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; -use Appwrite\Event\Webhook; -use Appwrite\Extend\Exception as AppwriteException; -use Appwrite\Platform\Modules\Compute\Base; use Appwrite\Usage\Build as BuildUsage; use Appwrite\Usage\Context; -use Appwrite\Utopia\Response\Model\Deployment; use Appwrite\Vcs\Factory as VcsFactory; use Exception; -use Executor\Exception as ExecutorException; -use Executor\Exception\Timeout as ExecutorTimeout; -use Executor\Executor; -use Swoole\Coroutine as Co; use Utopia\Config\Config; use Utopia\Console; use Utopia\Database\Database; @@ -34,13 +20,10 @@ use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; use Utopia\Database\Exception\Transaction as TransactionException; use Utopia\Database\Query; -use Utopia\Detector\Detection\Rendering\SSR; use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\Span\Span; -use Utopia\Storage\Device; -use Utopia\Storage\Device\Local; use Utopia\System\System; use Utopia\VCS\Adapter\Git; @@ -82,23 +65,14 @@ class Builds extends Action ->inject('message') ->inject('project') ->inject('dbForPlatform') - ->inject('queueForEvents') - ->inject('publisherForScreenshots') - ->inject('queueForWebhooks') - ->inject('publisherForFunctions') ->inject('queueForRealtime') ->inject('usage') ->inject('publisherForUsage') ->inject('vcsFactory') ->inject('dbForProject') - ->inject('deviceForFunctions') - ->inject('deviceForSites') ->inject('getIsResourceBlocked') - ->inject('deviceForFiles') ->inject('log') - ->inject('executor') ->inject('deployments') - ->inject('plan') ->callback($this->action(...)); } @@ -109,23 +83,14 @@ class Builds extends Action Message $message, Document $project, Database $dbForPlatform, - Event $queueForEvents, - Screenshot $publisherForScreenshots, - Webhook $queueForWebhooks, - FunctionPublisher $publisherForFunctions, Realtime $queueForRealtime, Context $usage, UsagePublisher $publisherForUsage, VcsFactory $vcsFactory, Database $dbForProject, - Device $deviceForFunctions, - Device $deviceForSites, callable $getIsResourceBlocked, - Device $deviceForFiles, Log $log, - Executor $executor, - Backend $deployments, - array $plan + Deployments $deployments, ): void { $payload = $message->getPayload(); @@ -149,14 +114,7 @@ class Builds extends Action case BUILD_TYPE_RETRY: $templateVcs = $vcsFactory->fromProvider('github'); $this->buildDeployment( - $deviceForFunctions, - $deviceForSites, - $deviceForFiles, - $publisherForScreenshots, - $queueForWebhooks, - $publisherForFunctions, $queueForRealtime, - $queueForEvents, $usage, $publisherForUsage, $dbForPlatform, @@ -169,9 +127,7 @@ class Builds extends Action $template, $getIsResourceBlocked, $log, - $executor, $deployments, - $plan, $platform, (int) ($payload['timeout'] ?? System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900)) ); @@ -187,14 +143,7 @@ class Builds extends Action * @throws Exception */ protected function buildDeployment( - Device $deviceForFunctions, - Device $deviceForSites, - Device $deviceForFiles, - Screenshot $publisherForScreenshots, - Webhook $queueForWebhooks, - FunctionPublisher $publisherForFunctions, Realtime $queueForRealtime, - Event $queueForEvents, Context $usage, UsagePublisher $publisherForUsage, Database $dbForPlatform, @@ -207,9 +156,7 @@ class Builds extends Action Document $template, callable $getIsResourceBlocked, Log $log, - Executor $executor, - Backend $deployments, - array $plan, + Deployments $deployments, array $platform, int $timeout ): void { @@ -229,11 +176,6 @@ class Builds extends Action default => throw new \Exception('Invalid resource type') }; - $device = match ($resource->getCollection()) { - 'sites' => $deviceForSites, - 'functions' => $deviceForFunctions, - }; - $log->addTag($resourceKey, $resource->getId()); $resource = $dbForProject->getDocument($resource->getCollection(), $resource->getId()); @@ -257,7 +199,6 @@ class Builds extends Action } $version = $this->getVersion($resource); - $runtime = $this->getRuntime($resource, $version); Span::add('build.runtime', $resource->getAttribute($resource->getCollection() === 'sites' ? 'buildRuntime' : 'runtime', '')); Span::add('build.version', $version); @@ -307,843 +248,167 @@ class Builds extends Action ->setPayload($deployment->getArrayCopy()) ->trigger(); - $source = $deployment->getAttribute('sourcePath', ''); $installationId = $deployment->getAttribute('installationId', ''); $providerRepositoryId = $deployment->getAttribute('providerRepositoryId', ''); $providerCommitHash = $deployment->getAttribute('providerCommitHash', ''); - $isVcsEnabled = ! empty($providerRepositoryId); $owner = ''; $repositoryName = ''; - $providerAdapter = null; - if ($isVcsEnabled) { - $installation = $dbForPlatform->getDocument('installations', $installationId); - $providerInstallationId = $installation->getAttribute('providerInstallationId'); + // Every other build flow submits its job straight from the request, so a + // deployment without a repository to push the template into never + // belongs here. + if (empty($providerRepositoryId)) { + throw new \Exception('Only template-into-repository deployments build through this worker'); + } - try { - $providerAdapter = $vcsFactory->fromInstallation($installation); - } catch (\Exception $e) { - if ($e->getCode() === 404 - && $resource->getAttribute('installationId', '') === $installationId) { - $this->disconnectVcs($resource, $dbForProject, $dbForPlatform); - } - throw $e; + $installation = $dbForPlatform->getDocument('installations', $installationId); + $providerInstallationId = $installation->getAttribute('providerInstallationId'); + + try { + $providerAdapter = $vcsFactory->fromInstallation($installation); + } catch (\Exception $e) { + if ($e->getCode() === 404 + && $resource->getAttribute('installationId', '') === $installationId) { + $this->disconnectVcs($resource, $dbForProject, $dbForPlatform); } + throw $e; } Span::add('timings.setup', \round(\microtime(true) - $phaseStart, 3)); $phaseStart = \microtime(true); try { - if (! $isVcsEnabled) { - // Non-VCS + Template - $templateRepositoryName = $template->getAttribute('repositoryName', ''); - $templateOwnerName = $template->getAttribute('ownerName', ''); - $templateReferenceType = $template->getAttribute('referenceType', ''); - $templateReferenceValue = $template->getAttribute('referenceValue', ''); + // VCS and VCS+Temaplte + $tmpDirectory = '/tmp/builds/' . $deploymentId . '/code'; + $rootDirectory = $resource->getAttribute('providerRootDirectory', ''); + $rootDirectory = \rtrim($rootDirectory, '/'); + $rootDirectory = \ltrim($rootDirectory, '.'); + $rootDirectory = \ltrim($rootDirectory, '/'); - $templateRootDirectory = $template->getAttribute('rootDirectory', ''); - $templateRootDirectory = \rtrim($templateRootDirectory, '/'); - $templateRootDirectory = \ltrim($templateRootDirectory, '.'); - $templateRootDirectory = \ltrim($templateRootDirectory, '/'); + $owner = $providerAdapter->getOwnerName($providerInstallationId); + $repositoryName = $providerAdapter->getRepositoryName($providerRepositoryId); - if (! empty($templateRepositoryName) && ! empty($templateOwnerName) && ! empty($templateReferenceType) && ! empty($templateReferenceValue)) { - $stdout = ''; - $stderr = ''; + $cloneOwner = $deployment->getAttribute('providerRepositoryOwner', $owner); + $cloneRepository = $deployment->getAttribute('providerRepositoryName', $repositoryName); - // Clone template repo - $tmpTemplateDirectory = '/tmp/builds/' . $deploymentId . '-template'; + $branchName = $deployment->getAttribute('providerBranch'); + $commitHash = $deployment->getAttribute('providerCommitHash', ''); - $gitCloneCommandForTemplate = $templateVcs->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateReferenceValue, $templateReferenceType, $tmpTemplateDirectory, $templateRootDirectory); + $cloneVersion = $branchName; + $cloneType = Git::CLONE_TYPE_BRANCH; + if (! empty($commitHash)) { + $cloneVersion = $commitHash; + $cloneType = Git::CLONE_TYPE_COMMIT; + } - $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr); + $gitCloneCommand = $providerAdapter->generateCloneCommand($cloneOwner, $cloneRepository, $cloneVersion, $cloneType, $tmpDirectory, $rootDirectory); + $stdout = ''; + $stderr = ''; - if ($exit !== 0) { - throw new \Exception('Unable to clone code repository: ' . $stderr); - } + Console::execute('mkdir -p ' . \escapeshellarg('/tmp/builds/' . $deploymentId), '', $stdout, $stderr); - Console::execute('find ' . \escapeshellarg($tmpTemplateDirectory) . ' -type d -name ".git" -exec rm -rf {} +', '', $stdout, $stderr); + if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { + $this->finalizeCanceledDeployment($deployment->getId(), $dbForProject, $queueForRealtime); - // Ensure directories - Console::execute('mkdir -p ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr); + return; + } - $tmpPathFile = $tmpTemplateDirectory . '/code.tar.gz'; + $exit = Console::execute($gitCloneCommand, '', $stdout, $stderr); - $localDevice = new Local(); + if ($exit !== 0) { + throw new BuildException('Unable to clone code repository: ' . $stderr); + } - if (substr($tmpTemplateDirectory, -1) !== '/') { - $tmpTemplateDirectory .= '/'; - } - - $tarParamDirectory = \escapeshellarg($tmpTemplateDirectory . (empty($templateRootDirectory) ? '' : '/' . $templateRootDirectory)); - Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax - - $source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); - $result = $localDevice->transfer($tmpPathFile, $source, $device); - - if (! $result) { - throw new \Exception('Unable to move file'); - } - - $directorySize = $localDevice->getFileSize($tmpPathFile); - - Console::execute('rm -rf ' . \escapeshellarg($tmpTemplateDirectory), '', $stdout, $stderr); - $deployment - ->setAttribute('sourcePath', $source) - ->setAttribute('sourceSize', $directorySize) - ->setAttribute('totalSize', $directorySize); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'sourcePath' => $deployment->getAttribute('sourcePath'), - 'sourceSize' => $deployment->getAttribute('sourceSize'), - 'totalSize' => $deployment->getAttribute('totalSize'), - ])); - - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); - - Span::add('build.source_size', $deployment->getAttribute('sourceSize')); - } - } elseif ($isVcsEnabled) { - // VCS and VCS+Temaplte - $tmpDirectory = '/tmp/builds/' . $deploymentId . '/code'; - $rootDirectory = $resource->getAttribute('providerRootDirectory', ''); - $rootDirectory = \rtrim($rootDirectory, '/'); - $rootDirectory = \ltrim($rootDirectory, '.'); - $rootDirectory = \ltrim($rootDirectory, '/'); - - $owner = $providerAdapter->getOwnerName($providerInstallationId); - $repositoryName = $providerAdapter->getRepositoryName($providerRepositoryId); - - $cloneOwner = $deployment->getAttribute('providerRepositoryOwner', $owner); - $cloneRepository = $deployment->getAttribute('providerRepositoryName', $repositoryName); - - $branchName = $deployment->getAttribute('providerBranch'); - $commitHash = $deployment->getAttribute('providerCommitHash', ''); - - $cloneVersion = $branchName; - $cloneType = Git::CLONE_TYPE_BRANCH; - if (! empty($commitHash)) { - $cloneVersion = $commitHash; - $cloneType = Git::CLONE_TYPE_COMMIT; - } - - $gitCloneCommand = $providerAdapter->generateCloneCommand($cloneOwner, $cloneRepository, $cloneVersion, $cloneType, $tmpDirectory, $rootDirectory); - $stdout = ''; - $stderr = ''; - - Console::execute('mkdir -p ' . \escapeshellarg('/tmp/builds/' . $deploymentId), '', $stdout, $stderr); - - if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { - $this->finalizeCanceledDeployment($deployment->getId(), $dbForProject, $queueForRealtime); - - return; - } - - $exit = Console::execute($gitCloneCommand, '', $stdout, $stderr); + // Local refactoring for function folder with spaces + if (str_contains($rootDirectory, ' ')) { + $rootDirectoryWithoutSpaces = str_replace(' ', '', $rootDirectory); + $from = $tmpDirectory . '/' . $rootDirectory; + $to = $tmpDirectory . '/' . $rootDirectoryWithoutSpaces; + $exit = Console::execute('mv ' . \escapeshellarg($from) . ' ' . \escapeshellarg($to), '', $stdout, $stderr); if ($exit !== 0) { - throw new BuildException('Unable to clone code repository: ' . $stderr); + throw new \Exception('Unable to move function with spaces' . $stderr); + } + $rootDirectory = $rootDirectoryWithoutSpaces; + } + + // Build from template + $templateRepositoryName = $template->getAttribute('repositoryName', ''); + $templateOwnerName = $template->getAttribute('ownerName', ''); + $templateReferenceType = $template->getAttribute('referenceType', ''); + $templateReferenceValue = $template->getAttribute('referenceValue', ''); + + $templateRootDirectory = $template->getAttribute('rootDirectory', ''); + $templateRootDirectory = \rtrim($templateRootDirectory, '/'); + $templateRootDirectory = \ltrim($templateRootDirectory, '.'); + $templateRootDirectory = \ltrim($templateRootDirectory, '/'); + + if (! empty($templateRepositoryName) && ! empty($templateOwnerName) && ! empty($templateReferenceType) && ! empty($templateReferenceValue)) { + // Clone template repo + $tmpTemplateDirectory = '/tmp/builds/' . $deploymentId . '/template'; + + $gitCloneCommandForTemplate = $templateVcs->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateReferenceValue, $templateReferenceType, $tmpTemplateDirectory, $templateRootDirectory); + $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr); + + if ($exit !== 0) { + throw new \Exception('Unable to clone code repository: ' . $stderr); } - // Local refactoring for function folder with spaces - if (str_contains($rootDirectory, ' ')) { - $rootDirectoryWithoutSpaces = str_replace(' ', '', $rootDirectory); - $from = $tmpDirectory . '/' . $rootDirectory; - $to = $tmpDirectory . '/' . $rootDirectoryWithoutSpaces; - $exit = Console::execute('mv ' . \escapeshellarg($from) . ' ' . \escapeshellarg($to), '', $stdout, $stderr); + // Ensure directories + Console::execute('mkdir -p ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr); + Console::execute('mkdir -p ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); - if ($exit !== 0) { - throw new \Exception('Unable to move function with spaces' . $stderr); - } - $rootDirectory = $rootDirectoryWithoutSpaces; + // Merge template into user repo + Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); + + // Commit and push + $commitMessage = \escapeshellarg('Create ' . $resource->getAttribute('name', '') . ' function'); + $exit = Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($tmpDirectory) . ' && git checkout -b ' . \escapeshellarg($branchName) . ' && git add . && git commit -m ' . $commitMessage . ' && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr); + + if ($exit !== 0) { + throw new \Exception('Unable to push code repository: ' . $stderr); } - // Build from template - $templateRepositoryName = $template->getAttribute('repositoryName', ''); - $templateOwnerName = $template->getAttribute('ownerName', ''); - $templateReferenceType = $template->getAttribute('referenceType', ''); - $templateReferenceValue = $template->getAttribute('referenceValue', ''); + $exit = Console::execute('cd ' . \escapeshellarg($tmpDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr); - $templateRootDirectory = $template->getAttribute('rootDirectory', ''); - $templateRootDirectory = \rtrim($templateRootDirectory, '/'); - $templateRootDirectory = \ltrim($templateRootDirectory, '.'); - $templateRootDirectory = \ltrim($templateRootDirectory, '/'); - - if (! empty($templateRepositoryName) && ! empty($templateOwnerName) && ! empty($templateReferenceType) && ! empty($templateReferenceValue)) { - // Clone template repo - $tmpTemplateDirectory = '/tmp/builds/' . $deploymentId . '/template'; - - $gitCloneCommandForTemplate = $templateVcs->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateReferenceValue, $templateReferenceType, $tmpTemplateDirectory, $templateRootDirectory); - $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr); - - if ($exit !== 0) { - throw new \Exception('Unable to clone code repository: ' . $stderr); - } - - // Ensure directories - Console::execute('mkdir -p ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr); - Console::execute('mkdir -p ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); - - // Merge template into user repo - Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); - - // Commit and push - $commitMessage = \escapeshellarg('Create ' . $resource->getAttribute('name', '') . ' function'); - $exit = Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($tmpDirectory) . ' && git checkout -b ' . \escapeshellarg($branchName) . ' && git add . && git commit -m ' . $commitMessage . ' && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr); - - if ($exit !== 0) { - throw new \Exception('Unable to push code repository: ' . $stderr); - } - - $exit = Console::execute('cd ' . \escapeshellarg($tmpDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr); - - if ($exit !== 0) { - throw new \Exception('Unable to get vcs commit SHA: ' . $stderr); - } - - $providerCommitHash = \trim($stdout); - - $deployment->setAttribute('providerCommitHash', $providerCommitHash); - $deployment->setAttribute('providerCommitAuthorUrl', APP_VCS_GITHUB_URL); - $deployment->setAttribute('providerCommitAuthor', APP_VCS_GITHUB_USERNAME); - $deployment->setAttribute('providerCommitMessage', "Create '" . $resource->getAttribute('name', '') . "' function"); - $deployment->setAttribute('providerCommitUrl', $providerAdapter->getCommitUrl($cloneOwner, $cloneRepository, $providerCommitHash)); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'providerCommitHash' => $deployment->getAttribute('providerCommitHash'), - 'providerCommitAuthorUrl' => $deployment->getAttribute('providerCommitAuthorUrl'), - 'providerCommitAuthor' => $deployment->getAttribute('providerCommitAuthor'), - 'providerCommitMessage' => $deployment->getAttribute('providerCommitMessage'), - 'providerCommitUrl' => $deployment->getAttribute('providerCommitUrl'), - ])); - - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); + if ($exit !== 0) { + throw new \Exception('Unable to get vcs commit SHA: ' . $stderr); } - // On the jobs backend the only build reaching this worker is the - // template-into-repo push above (a git *write* the jobs-service - // artifact system has no primitive for). With the commit pushed, - // hand the build to the jobs-service like any other VCS commit, - // via the same Deployments service the HTTP endpoints use. - if (System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator') { - $ref = $deployment->getAttribute('providerCommitHash') ?: $branchName; - $deployments->createFromUrl( - $resource, - $deployment, - $providerAdapter->getRepositoryPresignedUrl($cloneOwner, $cloneRepository, $ref), - $resource->getAttribute('providerRootDirectory', ''), - ); + $providerCommitHash = \trim($stdout); - Console::execute('rm -rf ' . \escapeshellarg('/tmp/builds/' . $deploymentId), '', $stdout, $stderr); - - return; - } - - $tmpPath = '/tmp/builds/' . $deploymentId; - $tmpPathFile = $tmpPath . '/code.tar.gz'; - $localDevice = new Local(); - - if (substr($tmpDirectory, -1) !== '/') { - $tmpDirectory .= '/'; - } - - $directorySize = $localDevice->getDirectorySize($tmpDirectory); - $sizeLimit = (int) System::getEnv('_APP_COMPUTE_SIZE_LIMIT', '30000000'); - - if (isset($plan['deploymentSize'])) { - $sizeLimit = (int) $plan['deploymentSize'] * 1000 * 1000; - } - - if ($directorySize > $sizeLimit && $sizeLimit !== 0) { - throw new BuildException('Repository directory size should be less than ' . number_format($sizeLimit / (1000 * 1000), 2) . ' MBs.'); - } - - Console::execute('find ' . \escapeshellarg($tmpDirectory) . ' -type d -name ".git" -exec rm -rf {} +', '', $stdout, $stderr); - - $tarParamDirectory = '/tmp/builds/' . $deploymentId . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory); - Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax - - $source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); - $result = $localDevice->transfer($tmpPathFile, $source, $device); - - if (! $result) { - throw new \Exception('Unable to move file'); - } - - $directorySize = $localDevice->getFileSize($tmpPathFile); - - Console::execute('rm -rf ' . \escapeshellarg($tmpPath), '', $stdout, $stderr); - - $deployment - ->setAttribute('sourcePath', $source) - ->setAttribute('sourceSize', $directorySize) - ->setAttribute('totalSize', $directorySize); + $deployment->setAttribute('providerCommitHash', $providerCommitHash); + $deployment->setAttribute('providerCommitAuthorUrl', APP_VCS_GITHUB_URL); + $deployment->setAttribute('providerCommitAuthor', APP_VCS_GITHUB_USERNAME); + $deployment->setAttribute('providerCommitMessage', "Create '" . $resource->getAttribute('name', '') . "' function"); + $deployment->setAttribute('providerCommitUrl', $providerAdapter->getCommitUrl($cloneOwner, $cloneRepository, $providerCommitHash)); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'sourcePath' => $deployment->getAttribute('sourcePath'), - 'sourceSize' => $deployment->getAttribute('sourceSize'), - 'totalSize' => $deployment->getAttribute('totalSize'), + 'providerCommitHash' => $deployment->getAttribute('providerCommitHash'), + 'providerCommitAuthorUrl' => $deployment->getAttribute('providerCommitAuthorUrl'), + 'providerCommitAuthor' => $deployment->getAttribute('providerCommitAuthor'), + 'providerCommitMessage' => $deployment->getAttribute('providerCommitMessage'), + 'providerCommitUrl' => $deployment->getAttribute('providerCommitUrl'), ])); $queueForRealtime ->setPayload($deployment->getArrayCopy()) ->trigger(); - - Span::add('build.source_size', $deployment->getAttribute('sourceSize')); - - $this->runGitAction('processing', $providerAdapter, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform, $queueForRealtime, $platform); } - Span::add('timings.source', \round(\microtime(true) - $phaseStart, 3)); - $phaseStart = \microtime(true); - - /** Request the executor to build the code... */ - $updated = $dbForProject->updateDocuments('deployments', new Document([ - 'status' => 'building', - ]), [ - Query::equal('$id', [$deploymentId]), - Query::notEqual('status', 'canceled'), - ]); - - if ($updated === 0) { - $this->finalizeCanceledDeployment($deploymentId, $dbForProject, $queueForRealtime); - return; - } - - $deployment = $dbForProject->getDocument('deployments', $deploymentId); - Span::add('deployment.status', 'building'); - - $resource = $this->updateLatestDeployment($dbForProject, $resource); - - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); - - if ($isVcsEnabled) { - $this->runGitAction('building', $providerAdapter, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform, $queueForRealtime, $platform); - } - - $deploymentModel = new Deployment(); - $deploymentUpdate = - $queueForEvents - ->setProject($project) - ->setEvent("{$resource->getCollection()}.[{$resourceKey}].deployments.[deploymentId].update") - ->setParam($resourceKey, $resource->getId()) - ->setParam('deploymentId', $deployment->getId()) - ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))); - - /** Trigger Webhook */ - $queueForWebhooks - ->from($deploymentUpdate) - ->trigger(); - - /** Trigger Functions */ - $publisherForFunctions->enqueue(FunctionMessage::fromEvent( - event: $deploymentUpdate->getEvent(), - params: $deploymentUpdate->getParams(), - project: $deploymentUpdate->getProject(), - user: $deploymentUpdate->getUser(), - userId: $deploymentUpdate->getUserId(), - payload: $deploymentUpdate->getPayload(), - platform: $deploymentUpdate->getPlatform(), - )); - - /** Trigger Realtime Event */ - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); - - $vars = []; - - // Shared vars - foreach ($resource->getAttribute('varsProject', []) as $var) { - $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); - } - - // Function vars - foreach ($resource->getAttribute('vars', []) as $var) { - $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); - } - - // Some runtimes/frameworks can't compile with less memory than this - $minMemory = $resource->getCollection() === 'sites' ? 2048 : 1024; - - if ( - $resource->getAttribute('framework', '') === 'analog' || - $resource->getAttribute('framework', '') === 'tanstack-start' - ) { - $minMemory = 4096; - } - - $cpus = $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT; - $memory = max($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT, $minMemory); - $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $timeout, 0); - - $apiKey = $jwtObj->encode([ - 'projectId' => $project->getId(), - 'scopes' => $resource->getAttribute('scopes', []), - ]); - - // Appwrite vars - $vars = \array_merge($vars, [ - 'APPWRITE_VERSION' => APP_VERSION_STABLE, - 'APPWRITE_REGION' => $project->getAttribute('region'), - 'APPWRITE_DEPLOYMENT_TYPE' => $deployment->getAttribute('type', ''), - 'APPWRITE_VCS_REPOSITORY_ID' => $deployment->getAttribute('providerRepositoryId', ''), - 'APPWRITE_VCS_REPOSITORY_NAME' => $deployment->getAttribute('providerRepositoryName', ''), - 'APPWRITE_VCS_REPOSITORY_OWNER' => $deployment->getAttribute('providerRepositoryOwner', ''), - 'APPWRITE_VCS_REPOSITORY_URL' => $deployment->getAttribute('providerRepositoryUrl', ''), - 'APPWRITE_VCS_REPOSITORY_BRANCH' => $deployment->getAttribute('providerBranch', ''), - 'APPWRITE_VCS_REPOSITORY_BRANCH_URL' => $deployment->getAttribute('providerBranchUrl', ''), - 'APPWRITE_VCS_COMMIT_HASH' => $deployment->getAttribute('providerCommitHash', ''), - 'APPWRITE_VCS_COMMIT_MESSAGE' => $deployment->getAttribute('providerCommitMessage', ''), - 'APPWRITE_VCS_COMMIT_URL' => $deployment->getAttribute('providerCommitUrl', ''), - 'APPWRITE_VCS_COMMIT_AUTHOR_NAME' => $deployment->getAttribute('providerCommitAuthor', ''), - 'APPWRITE_VCS_COMMIT_AUTHOR_URL' => $deployment->getAttribute('providerCommitAuthorUrl', ''), - 'APPWRITE_VCS_ROOT_DIRECTORY' => $deployment->getAttribute('providerRootDirectory', ''), - ]); - - $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; - $endpoint = "$protocol://{$platform['apiHostname']}/v1"; - - switch ($resource->getCollection()) { - case 'functions': - $vars = [ - ...$vars, - 'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint, - 'APPWRITE_FUNCTION_API_KEY' => API_KEY_EPHEMERAL . '_' . $apiKey, - 'APPWRITE_FUNCTION_ID' => $resource->getId(), - 'APPWRITE_FUNCTION_NAME' => $resource->getAttribute('name'), - 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), - 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', - 'APPWRITE_FUNCTION_CPUS' => $cpus, - 'APPWRITE_FUNCTION_MEMORY' => $memory, - 'OPEN_RUNTIMES_NFT' => System::getEnv('_APP_OPEN_RUNTIMES_NFT', 'enabled'), - ]; - break; - case 'sites': - $vars = [ - ...$vars, - 'APPWRITE_SITE_API_ENDPOINT' => $endpoint, - 'APPWRITE_SITE_API_KEY' => API_KEY_EPHEMERAL . '_' . $apiKey, - 'APPWRITE_SITE_ID' => $resource->getId(), - 'APPWRITE_SITE_NAME' => $resource->getAttribute('name'), - 'APPWRITE_SITE_DEPLOYMENT' => $deployment->getId(), - 'APPWRITE_SITE_PROJECT_ID' => $project->getId(), - 'APPWRITE_SITE_RUNTIME_NAME' => $runtime['name'] ?? '', - 'APPWRITE_SITE_RUNTIME_VERSION' => $runtime['version'] ?? '', - 'APPWRITE_SITE_CPUS' => $cpus, - 'APPWRITE_SITE_MEMORY' => $memory, - 'OPEN_RUNTIMES_NFT' => System::getEnv('_APP_OPEN_RUNTIMES_NFT', 'enabled'), - ]; - break; - } - - $command = Backend::command($resource, $deployment); - - $cacheKey = $this->getNodeModulesCacheKey($project, $resource, $runtime, $version, $command); - - Span::add('build.node_modules_cache.enabled', $cacheKey !== ''); - Span::add('build.node_modules_cache.key', $cacheKey); - - $response = null; - $err = null; - - if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { - $this->finalizeCanceledDeployment($deployment->getId(), $dbForProject, $queueForRealtime); - - return; - } - - $isCanceled = false; - $span = Span::current(); - - Co::join([ - Co\go(function () use ($executor, &$response, $project, $deployment, $source, $resource, $runtime, $vars, $command, $cacheKey, $cpus, $memory, $timeout, &$err, $version, $span) { - try { - if ($version === 'v2') { - $command = 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh'; - } else { - $outputDirectory = $deployment->getAttribute('buildOutput') ?? $resource->getAttribute('outputDirectory'); - if ($resource->getCollection() === 'sites') { - $command = $this->prepareSiteBuildCommand($command, $outputDirectory ?? '', $resource->getAttribute('framework', '')); - } - - $command = 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh ' . \trim(\escapeshellarg($command)); - } - - $response = $executor->createRuntime( - deploymentId: $deployment->getId(), - projectId: $project->getId(), - source: $source, - image: $runtime['image'], - version: $version, - cpus: $cpus, - memory: $memory, - timeout: $timeout, - remove: true, - entrypoint: $deployment->getAttribute('entrypoint', ''), - destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", - variables: $vars, - command: $command, - outputDirectory: $outputDirectory ?? '', - cacheKey: $cacheKey - ); - - } catch (ExecutorTimeout $error) { - $span?->set('build.runtime.timed_out', true); - $span?->set('build.runtime.error_type', $error::class); - $span?->set('build.runtime.error_message', $error->getMessage()); - $err = new BuildException(type: AppwriteException::BUILD_TIMEOUT, previous: $error); - } catch (ExecutorException $error) { - $span?->set('build.runtime.error_type', $error::class); - $span?->set('build.runtime.error_message', $error->getMessage()); - $span?->set('build.runtime.executor_error_type', $error->getType()); - - $err = $error->getType() === ExecutorException::BUILD_FAILED - ? new BuildException($error->getMessage(), previous: $error) - : $error; - } catch (\Throwable $error) { - $span?->set('build.runtime.error_type', $error::class); - $span?->set('build.runtime.error_message', $error->getMessage()); - $err = $error; - } - }), - Co\go(function () use ($executor, $project, &$deployment, &$response, $dbForProject, $timeout, &$err, $queueForRealtime, &$isCanceled, $span) { - try { - $insideSeparation = false; - - $executor->getLogs( - deploymentId: $deployment->getId(), - projectId: $project->getId(), - timeout: $timeout, - callback: function ($logs) use (&$response, &$err, $dbForProject, &$isCanceled, &$deployment, $queueForRealtime, &$insideSeparation, $span) { - if ($isCanceled) { - return; - } - - // If we have response or error from concurrent coroutine, we already have latest logs - if ($response === null && $err === null) { - $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); - - if ($deployment->getAttribute('status') === 'canceled') { - $isCanceled = true; - $span?->set('build.logs.ignored_reason', 'canceled'); - - return; - } - - // Get only valid UTF8 part - removes leftover half-multibytes causing SQL errors - $logs = \mb_substr($logs, 0, null, 'UTF-8'); - - // Do not stream logs added for SSR detection - if (! $insideSeparation) { - $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR_START}'); - if ($separator !== false) { - $logs = \substr($logs, 0, $separator); - $insideSeparation = true; - - $leftover = \substr($logs, $separator + strlen('{APPWRITE_DETECTION_SEPARATOR_START}')); - $separator = \strpos($leftover, '{APPWRITE_DETECTION_SEPARATOR_END}'); - if ($separator !== false) { - $logs .= \substr($leftover, $separator + strlen('{APPWRITE_DETECTION_SEPARATOR_END}')); - $insideSeparation = false; - } - } - } else { - $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR_END}'); - if ($separator !== false) { - $logs = \substr($logs, $separator + strlen('{APPWRITE_DETECTION_SEPARATOR_END}')); - $insideSeparation = false; - } else { - $logs = ''; - } - } - - if (empty($logs)) { - return; - } - - $currentLogs = $deployment->getAttribute('buildLogs', ''); - $affected = false; - - $streamLogs = \str_replace('\\n', '{APPWRITE_LINEBREAK_PLACEHOLDER}', $logs); - foreach (\explode("\n", $streamLogs) as $streamLog) { - if (empty($streamLog)) { - continue; - } - - $streamLog = \str_replace('{APPWRITE_LINEBREAK_PLACEHOLDER}', "\n", $streamLog); - $streamParts = \explode(' ', $streamLog, 2); - - if (! isset($streamParts[1])) { - continue; - } - - // TODO: use part[0] as timestamp when switching to dbForLogs for build logs - $currentLogs .= $streamParts[1]; - - if (! empty($streamParts[1])) { - $affected = true; - } - } - - if ($affected) { - $deployment = $deployment->setAttribute('buildLogs', $this->truncateBuildLogs($currentLogs)); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'buildLogs' => $this->truncateBuildLogs($currentLogs), - ])); - - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); - } - } - } - ); - $span?->set('build.logs.finished', true); - } catch (\Throwable $error) { - $span?->set('build.logs.error_type', $error::class); - $span?->set('build.logs.error_message', $error->getMessage()); - if (empty($err)) { - $err = $error; - } - } - }), - ]); - - Span::add('timings.build', \round(\microtime(true) - $phaseStart, 3)); - $phaseStart = \microtime(true); - - $latestDeployment = $dbForProject->getDocument('deployments', $deploymentId); - if ($latestDeployment->getAttribute('status') === 'canceled') { - $this->finalizeCanceledDeployment($deployment->getId(), $dbForProject, $queueForRealtime); - - return; - } - - if ($err) { - throw $err; - } - - $buildSizeLimit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000'); - if (isset($plan['buildSize'])) { - $buildSizeLimit = $plan['buildSize'] * 1000 * 1000; - } - if ($response['size'] > $buildSizeLimit && $buildSizeLimit !== 0) { - throw new BuildException('Build size should be less than ' . number_format($buildSizeLimit / (1000 * 1000), 2) . ' MBs.'); - } - - $deployment->setAttribute('buildPath', $response['path']); - $deployment->setAttribute('buildSize', $response['size']); - $deployment->setAttribute('totalSize', $deployment->getAttribute('buildSize', 0) + $deployment->getAttribute('sourceSize', 0)); - Span::add('build.size', $deployment->getAttribute('buildSize')); - Span::add('build.total_size', $deployment->getAttribute('totalSize')); - - $logs = ''; - foreach ($response['output'] as $log) { - $logs .= $log['content']; - } - - ['logs' => $logs, 'detectionLogs' => $detectionLogs] = $this->splitSiteDetectionLogs($logs); - - $deployment->setAttribute('buildLogs', $this->truncateBuildLogs($logs)); - - $adapter = null; - if ($resource->getCollection() === 'sites' && ! empty($detectionLogs)) { - $detection = Detection::rendering($resource->getAttribute('framework', ''), \explode("\n", $detectionLogs)); - - $adapter = $resource->getAttribute('adapter', ''); - if (empty($adapter)) { - $resource = $dbForProject->updateDocument('sites', $resource->getId(), new Document(['adapter' => $detection->getName(), 'fallbackFile' => $detection->getFallbackFile() ?? ''])); - - $deployment->setAttribute('adapter', $detection->getName()); - $deployment->setAttribute('fallbackFile', $detection->getFallbackFile() ?? ''); - Span::add('build.adapter', $deployment->getAttribute('adapter')); - Span::add('build.fallback_file', $deployment->getAttribute('fallbackFile')); - } elseif ($adapter === 'ssr' && $detection->getName() === 'static') { - throw new BuildException('Adapter mismatch. Detected: ' . $detection->getName() . ' does not match with the set adapter: ' . $adapter); - } - } - - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'buildPath' => $deployment->getAttribute('buildPath'), - 'buildSize' => $deployment->getAttribute('buildSize'), - 'totalSize' => $deployment->getAttribute('totalSize'), - 'buildLogs' => $deployment->getAttribute('buildLogs'), - 'adapter' => $deployment->getAttribute('adapter'), - 'fallbackFile' => $deployment->getAttribute('fallbackFile'), - ])); - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); - - $this->afterBuildSuccess($queueForRealtime, $dbForProject, $deployment, $runtime, $adapter); - - $logs = $deployment->getAttribute('buildLogs', ''); - $date = \date('H:i:s'); - $logs .= "\033[90m[$date] \033[90m[\033[0mappwrite\033[90m]\033[32m Deployment finished. \033[0m\n"; - $deployment->setAttribute('buildLogs', $this->truncateBuildLogs($logs)); - - /** Update the status */ - $endTime = DateTime::now(); - $durationEnd = \microtime(true); - $deployment->setAttribute('buildEndedAt', $endTime); - $deployment->setAttribute('buildDuration', \intval(\ceil($durationEnd - $durationStart))); - $deployment->setAttribute('status', 'ready'); - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ - 'buildEndedAt' => $deployment->getAttribute('buildEndedAt'), - 'buildDuration' => $deployment->getAttribute('buildDuration'), - 'buildLogs' => $deployment->getAttribute('buildLogs'), - 'status' => 'ready', - ])); - Span::add('deployment.status', 'ready'); - Span::add('build.duration', $deployment->getAttribute('buildDuration')); - - $resource = $this->updateLatestDeployment($dbForProject, $resource); - - if ($isVcsEnabled) { - $this->runGitAction('ready', $providerAdapter, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform, $queueForRealtime, $platform); - } - - /** Set auto deploy */ - $activateBuild = false; - if ($deployment->getAttribute('activate') === true) { - // Check if current active deployment started later than this deployment - $resource = $dbForProject->getDocument($resource->getCollection(), $resource->getId()); - $currentActiveDeploymentId = $resource->getAttribute('deploymentId', ''); - if (! empty($currentActiveDeploymentId)) { - $currentActiveDeployment = $dbForProject->getDocument('deployments', $currentActiveDeploymentId); - if (! $currentActiveDeployment->isEmpty()) { - $currentActiveStartTime = $currentActiveDeployment->getCreatedAt(); - $deploymentStartTime = $deployment->getCreatedAt(); - - // Skip auto-activation if current active deployment started later than deployment that is being activated - if ($currentActiveStartTime < $deploymentStartTime) { - $activateBuild = true; - } else { - Span::add('build.auto_activation.skipped_reason', 'current_deployment_newer'); - } - } - } else { - $activateBuild = true; - } - } - - if ($activateBuild) { - switch ($resource->getCollection()) { - case 'functions': - $resource = $dbForProject->updateDocument('functions', $resource->getId(), new Document([ - 'live' => true, - 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getSequence(), - 'deploymentCreatedAt' => $deployment->getCreatedAt(), - ])); - - $queries = [ - Query::equal('projectInternalId', [$project->getSequence()]), - Query::equal('type', ['deployment']), - Query::equal('deploymentResourceInternalId', [$resource->getSequence()]), - Query::equal('deploymentResourceType', ['function']), - Query::equal('trigger', ['manual']), - Query::equal('deploymentVcsProviderBranch', ['']), - ]; - - $rulesUpdated = false; - $dbForPlatform->forEach('rules', function (Document $rule) use ($dbForPlatform, $deployment, &$rulesUpdated) { - $rulesUpdated = true; - $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), new Document([ - 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getSequence(), - ])); - }, $queries); - break; - case 'sites': - $resource = $dbForProject->updateDocument('sites', $resource->getId(), new Document([ - 'live' => true, - 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getSequence(), - 'deploymentCreatedAt' => $deployment->getCreatedAt(), - ])); - $queries = [ - Query::equal('projectInternalId', [$project->getSequence()]), - Query::equal('type', ['deployment']), - Query::equal('deploymentResourceInternalId', [$resource->getSequence()]), - Query::equal('deploymentResourceType', ['site']), - Query::equal('trigger', ['manual']), - Query::equal('deploymentVcsProviderBranch', ['']), - ]; - - $dbForPlatform->forEach('rules', function (Document $rule) use ($dbForPlatform, $deployment) { - $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), new Document([ - 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getSequence(), - ])); - }, $queries); - - break; - } - - Span::add('build.activated', true); - } - - $resource = $this->updateLatestDeployment($dbForProject, $resource); - - $this->afterDeploymentSuccess( - $project, + // The only build reaching this worker is the template-into-repo push + // above (a git *write* the jobs-service artifact system has no + // primitive for). With the commit pushed, hand the build to the + // jobs-service like any other VCS commit, via the same Deployments + // service the HTTP endpoints use. + $ref = $deployment->getAttribute('providerCommitHash') ?: $branchName; + $deployments->createFromUrl( + $resource, $deployment, + $providerAdapter->getRepositoryPresignedUrl($cloneOwner, $cloneRepository, $ref), + $resource->getAttribute('providerRootDirectory', ''), ); - // Send realtime event after updating the associated resource so that Console will have the resource's deployment details when re-fetching. - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); - - if ($resource->getCollection() === 'sites' && ! empty($deployment->getAttribute('providerBranch'))) { - Base::activateBranchPreviewRule($project, $resource, $deployment, $dbForPlatform, $platform['sitesDomain']); - Span::add('build.preview_rule_created', true); - } - - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); - - if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { - $this->finalizeCanceledDeployment($deployment->getId(), $dbForProject, $queueForRealtime); - - return; - } - - /** Update function schedule */ - - // Inform scheduler if function is still active - if ($resource->getCollection() === 'functions') { - $schedule = $dbForPlatform->getDocument('schedules', $resource->getAttribute('scheduleId')); - $schedule - ->setAttribute('resourceUpdatedAt', DateTime::now()) - ->setAttribute('schedule', $resource->getAttribute('schedule')) - ->setAttribute('active', ! empty($resource->getAttribute('schedule')) && ! empty($resource->getAttribute('deploymentId'))); - $dbForPlatform->updateDocument('schedules', $schedule->getId(), new Document([ - 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), - 'schedule' => $schedule->getAttribute('schedule'), - 'active' => $schedule->getAttribute('active'), - ])); - } - - /** Screenshot site */ - if ($resource->getCollection() === 'sites') { - $publisherForScreenshots->enqueue(new \Appwrite\Event\Message\Screenshot( - project: $project, - deploymentId: $deployment->getId(), - )); - - Span::add('build.screenshot_queued', true); - } - - Span::add('timings.finalize', \round(\microtime(true) - $phaseStart, 3)); + Console::execute('rm -rf ' . \escapeshellarg('/tmp/builds/' . $deploymentId), '', $stdout, $stderr); } catch (\Throwable $th) { if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->finalizeCanceledDeployment($deployment->getId(), $dbForProject, $queueForRealtime); @@ -1200,9 +465,7 @@ class Builds extends Action ->setPayload($deployment->getArrayCopy()) ->trigger(); - if ($isVcsEnabled) { - $this->runGitAction('failed', $providerAdapter, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform, $queueForRealtime, $platform, true); - } + $this->runGitAction('failed', $providerAdapter, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform, $queueForRealtime, $platform, true); // Let the worker harness record internal errors via the span and logger. if (! $isUserFacing) { @@ -1228,40 +491,6 @@ class Builds extends Action BuildUsage::publish($usage, $resource, $deployment, $project, $publisherForUsage); } - /** - * Hook to run after build success - * - * @throws Exception - */ - protected function afterBuildSuccess(Realtime $queueForRealtime, Database $dbForProject, Document &$deployment, array $runtime, ?string $adapter): void - { - } - - /** - * Hook to run after deployment is activated - */ - protected function afterDeploymentSuccess( - Document $project, - Document $deployment, - ): void { - } - - protected function getRuntime(Document $resource, string $version): array - { - $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); - $key = $resource->getAttribute('runtime'); - $runtime = match ($resource->getCollection()) { - 'functions' => $runtimes[$resource->getAttribute('runtime')] ?? null, - 'sites' => $runtimes[$resource->getAttribute('buildRuntime')] ?? null, - default => null - }; - if (\is_null($runtime)) { - throw new BuildException('Runtime "' . $resource->getAttribute('runtime', '') . '" is not supported'); - } - - return $runtime; - } - protected function getVersion(Document $resource): string { return match ($resource->getCollection()) { @@ -1271,68 +500,6 @@ class Builds extends Action }; } - protected function getNodeModulesCacheKey(Document $project, Document $resource, array $runtime, string $version, string $command): string - { - if ($version !== 'v5' || $command === '' || $command === '0') { - return ''; - } - - $hashContext = [ - 'version' => 'v1', - 'projectId' => $project->getId(), - 'resourceType' => $resource->getCollection(), - 'resourceId' => $resource->getId(), - 'runtime' => $runtime['image'] ?? '', - ]; - - return \substr(\hash('sha256', \json_encode($hashContext, JSON_THROW_ON_ERROR)), 0, 48); - } - - protected function prepareSiteBuildCommand(string $command, string $outputDirectory, string $framework): string - { - $listFilesCommand = 'echo "{APPWRITE_DETECTION_SEPARATOR_START}" && cd /usr/local/build'; - - if (! empty($outputDirectory)) { - $listFilesCommand .= ' && cd ' . \escapeshellarg($outputDirectory); - } - - foreach (SSR::FRAMEWORK_FILES[$framework] ?? [] as $file) { - $listFilesCommand .= ' && ( [ -e ' . \escapeshellarg($file) . ' ] && echo ' . \escapeshellarg($file) . ' || true )'; - } - - // Static fallback detection only needs to distinguish 0, 1, or 2+ HTML files, so cap the output - $listFilesCommand .= ' && find . -name \'node_modules\' -prune -o -type f -name \'*.html\' -print | head -n 2'; - - $listFilesCommand .= ' && echo "{APPWRITE_DETECTION_SEPARATOR_END}"'; - - if (empty($command)) { - return $listFilesCommand; - } - - return "{$command} && {$listFilesCommand}"; - } - - /** - * @return array{logs: string, detectionLogs: string} - */ - protected function splitSiteDetectionLogs(string $logs): array - { - if (! \str_contains($logs, '{APPWRITE_DETECTION_SEPARATOR_START}')) { - return [ - 'logs' => $logs, - 'detectionLogs' => '', - ]; - } - - [$logsBefore, $detectionLogsStart] = \explode('{APPWRITE_DETECTION_SEPARATOR_START}', $logs, 2); - [$detectionLogs, $logsAfter] = \explode('{APPWRITE_DETECTION_SEPARATOR_END}', $detectionLogsStart, 2); - - return [ - 'logs' => "{$logsBefore}{$logsAfter}", - 'detectionLogs' => $detectionLogs, - ]; - } - /** * @throws Structure * @throws \Utopia\Database\Exception diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index 76b8880dcc..41c81ee3a5 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -30,8 +30,7 @@ use Utopia\System\System; /** * Applies open-runtimes jobs-service callbacks to a deployment. The API * callback endpoint verifies + enqueues the CloudEvent; this worker owns the - * deployment state transitions (log streaming and the ready/failed outcome) - * that the executor path performs inline in the Builds worker. + * deployment state transitions: log streaming and the ready/failed outcome. * * Callbacks arrive concurrently and out of order (one log line each), so * processing for a deployment is serialized under a per-deployment lock — that diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php index ba97b6cd7a..5581e00751 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; @@ -107,7 +107,7 @@ class Create extends Action Event $queueForEvents, Device $deviceForSites, Device $deviceForLocal, - Backend $deployments, + Deployments $deployments, array $plan, Authorization $authorization, array $platform, diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php index 905764bd92..092cec9119 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Duplicate; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; @@ -83,7 +83,7 @@ class Create extends Action Database $dbForProject, Database $dbForPlatform, Event $queueForEvents, - Backend $deployments, + Deployments $deployments, Device $deviceForSites, VcsFactory $vcsFactory, Authorization $authorization, diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php index 0d562258f5..ad9a33829a 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php @@ -2,8 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Status; -use Appwrite\Deployment\Backend; -use Appwrite\Deployment\Backend\Orchestrator; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; @@ -70,7 +69,7 @@ class Update extends Action Response $response, Database $dbForProject, Event $queueForEvents, - Backend $deployments, + Deployments $deployments, callable $locks ) { $site = $dbForProject->getDocument('sites', $siteId); @@ -96,9 +95,9 @@ class Update extends Action // Write under the Jobs worker's per-deployment lock: its handlers // read-modify-write buildLogs, and an unserialized cancel write here // loses the closing log line to an in-flight append. - $cancel = function () use ($dbForProject, $deployment, $duration, $deployments) { + $cancel = function () use ($dbForProject, $deployment, $duration) { try { - return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); + return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration))); } catch (TransactionException) { $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); @@ -111,7 +110,7 @@ class Update extends Action } if ($deployment->getAttribute('status') !== 'canceled') { - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration))); } return $deployment; @@ -138,25 +137,20 @@ class Update extends Action } /** - * The sparse update marking a build canceled. Jobs-backed builds have no - * cancel worker to write the closing log line the executor's Builds worker - * adds, so it is appended here; executor deployments get it from their worker. + * The sparse update marking a build canceled. No worker writes the closing + * log line for a canceled build, so it is appended here. * * @return array */ - private function cancel(Document $deployment, int $duration, bool $appendLog): array + private function cancel(Document $deployment, int $duration): array { - $update = [ + $logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n"; + + return [ 'buildEndedAt' => DateTime::now(), 'buildDuration' => $duration, 'status' => 'canceled', + 'buildLogs' => \substr($logs, -APP_LOG_LENGTH_LIMIT), ]; - - if ($appendLog) { - $logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n"; - $update['buildLogs'] = \substr($logs, -APP_LOG_LENGTH_LIMIT); - } - - return $update; } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php index 93cc7270b3..166ba5b18c 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Template; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -27,7 +27,6 @@ use Utopia\System\System; use Utopia\Validator\Boolean; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; -use Utopia\VCS\Adapter\Git; class Create extends Base { @@ -106,7 +105,7 @@ class Create extends Base BuildPublisher $publisherForBuilds, VcsFactory $vcsFactory, Authorization $authorization, - Backend $deployments, + Deployments $deployments, array $platform ) { $site = $dbForProject->getDocument('sites', $siteId); @@ -164,17 +163,7 @@ class Create extends Base $commands[] = $site->getAttribute('buildCommand', ''); } - // Templates can pin a version range (e.g. "0.3.*"); codeload only takes - // a concrete ref, so resolve it to the highest matching tag via GitHub. - $ref = $reference; - if ($type === Git::CLONE_TYPE_TAG && \str_contains($reference, '*')) { - try { - $tags = $vcsFactory->fromProvider('github')->listTags($owner, $repository, $reference); - $ref = \end($tags) ?: $reference; - } catch (\Throwable) { - // Fall back to the raw reference; the build surfaces a bad ref. - } - } + $ref = Base::resolveTemplateRef($vcsFactory, $owner, $repository, $type, $reference); $deploymentId = ID::unique(); $deployment = $dbForProject->createDocument('deployments', new Document([ @@ -233,8 +222,7 @@ class Create extends Base $this->updateEmptyManualRule($project, $site, $deployment, $dbForPlatform, $authorization); // Public template: pull the source straight from GitHub's public repo - // (codeload tarball on the jobs backend, a plain git clone on the - // executor); unarchive/checkout strips down to the rootDirectory. + // as a codeload tarball; unarchive strips down to the rootDirectory. $deployment = $deployments->createFromRef( $site, $deployment, diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Vcs/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Vcs/Create.php index 4e10ed7822..f2c0f7879d 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Vcs/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Vcs/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Vcs; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -96,7 +96,7 @@ class Create extends Base BuildPublisher $publisherForBuilds, VcsFactory $vcsFactory, Authorization $authorization, - Backend $deployments, + Deployments $deployments, array $platform ) { $site = $dbForProject->getDocument('sites', $siteId); diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php index 216c861a82..a009e7c0ed 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Sites; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -156,7 +156,7 @@ class Update extends Base RepositoryWebhooks $repositoryWebhooks, Executor $executor, Authorization $authorization, - Backend $deployments, + Deployments $deployments, array $platform ) { if (!empty($adapter)) { diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php index a4194292b9..e2d87dea34 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php @@ -389,7 +389,7 @@ trait Deployment 'activate' => $activate, ]); - // The Backend is built per repository: a webhook fans out to + // The Deployments service is built per repository: a webhook fans out to // many tenant projects, each with its own database. $deployment = $authorization->skip(fn () => $deploymentsFactory($dbForProject, $project) ->createFromUrl( diff --git a/src/Appwrite/Usage/Build.php b/src/Appwrite/Usage/Build.php index 41da65e5e1..e2ab30893b 100644 --- a/src/Appwrite/Usage/Build.php +++ b/src/Appwrite/Usage/Build.php @@ -8,9 +8,9 @@ use Utopia\Config\Config; use Utopia\Database\Document; /** - * Emits build usage/billing metrics for a completed deployment. Shared by both - * build backends — the executor (Builds worker) and the jobs-service (Jobs - * worker) — so a build is counted identically however it was produced. + * Emits build usage/billing metrics for a completed deployment. Shared by the + * Jobs worker and the Builds worker, so a build is counted identically whether + * it finished on the jobs-service or failed before reaching it. */ final class Build { diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index 1ca966416b..64c50f7ce9 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -41,100 +41,6 @@ class Executor ]; } - /** - * Create runtime - * - * Launches a runtime container for a deployment ready for execution - * - * @param string $deploymentId - * @param string $projectId - * @param string $source - * @param string $image - * @param bool $remove - * @param string $entrypoint - * @param string $destination - * @param array $variables - * @param string $command - */ - public function createRuntime( - string $deploymentId, - string $projectId, - string $source, - string $image, - string $version, - float $cpus, - int $memory, - int $timeout, - bool $remove = false, - string $entrypoint = '', - string $destination = '', - array $variables = [], - ?string $command = null, - string $outputDirectory = '', - string $runtimeEntrypoint = '', - string $cacheKey = '' - ) { - $runtimeId = "$projectId-$deploymentId-build"; - $route = "/runtimes"; - - // Remove after migration - if ($version === 'v3' || $version === 'v4') { - $version = 'v5'; - } - - $params = [ - 'runtimeId' => $runtimeId, - 'source' => $source, - 'destination' => $destination, - 'image' => $image, - 'entrypoint' => $entrypoint, - 'variables' => $variables, - 'remove' => $remove, - 'command' => $command, - 'cpus' => $cpus, - 'memory' => $memory, - 'version' => $version, - 'timeout' => $timeout, - 'outputDirectory' => $outputDirectory, - 'runtimeEntrypoint' => $runtimeEntrypoint, - 'cacheKey' => $cacheKey - ]; - - - $response = $this->call($this->endpoint, self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout); - - $status = $response['headers']['status-code']; - if ($status >= 400) { - $message = \is_string($response['body']) ? $response['body'] : ($response['body']['message'] ?? ''); - $type = \is_array($response['body']) ? ($response['body']['type'] ?? ExecutorException::GENERAL_UNKNOWN) : ExecutorException::GENERAL_UNKNOWN; - throw new ExecutorException($message, $status, type: $type); - } - - return $response['body']; - } - - /** - * Listen to realtime logs stream of a runtime - * - * @param string $deploymentId - * @param string $projectId - * @param callable $callback - */ - public function getLogs( - string $deploymentId, - string $projectId, - string $timeout, - callable $callback - ) { - $runtimeId = "$projectId-$deploymentId-build"; - $route = "/runtimes/{$runtimeId}/logs"; - $params = [ - 'timeout' => $timeout - ]; - - $this->call($this->endpoint, self::METHOD_GET, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout, $callback); - } - /** * Delete Runtime * diff --git a/tests/unit/Deployment/BackendTest.php b/tests/unit/Deployment/DeploymentsTest.php similarity index 81% rename from tests/unit/Deployment/BackendTest.php rename to tests/unit/Deployment/DeploymentsTest.php index db024a4ef0..7fefe10893 100644 --- a/tests/unit/Deployment/BackendTest.php +++ b/tests/unit/Deployment/DeploymentsTest.php @@ -4,12 +4,12 @@ declare(strict_types=1); namespace Tests\Unit\Deployment; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use PHPUnit\Framework\TestCase; use Utopia\Config\Config; use Utopia\Database\Document; -final class BackendTest extends TestCase +final class DeploymentsTest extends TestCase { public function testSiteCommandIncludesFrameworkAndDeploymentCommands(): void { @@ -30,7 +30,7 @@ final class BackendTest extends TestCase $this->assertSame( 'cp .env.example .env && npm run build && npm run bundle', - Backend::command($resource, $deployment) + Deployments::command($resource, $deployment) ); } @@ -39,6 +39,6 @@ final class BackendTest extends TestCase $resource = new Document(['$collection' => 'functions']); $deployment = new Document(['buildCommands' => 'npm install']); - $this->assertSame('npm install', Backend::command($resource, $deployment)); + $this->assertSame('npm install', Deployments::command($resource, $deployment)); } } From 396a03bc8a319ab1f0cf9fa887dac74ea96bc9f9 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:09:19 +0100 Subject: [PATCH 2/3] fix: don't resurrect a canceled deployment on dispatch A deployment canceled while its source was being prepared (the Builds worker's template-into-repo push can take a while) was still built: submit() wrote the caller's stale status through upload() and then unconditionally moved it to 'waiting', so the Jobs worker saw a live build and could activate it. Drop the caller's stale status before upload() and guard the queued transition with notEqual('status', 'canceled'), so a cancel can no longer be clobbered. When the guard rejects the write, no job is submitted and the active deployment is left alone. The failure write is guarded the same way, so a cancel landing while the job is being submitted stays canceled. Co-Authored-By: Claude Opus 5 (1M context) --- src/Appwrite/Deployment/Deployments.php | 41 +++++++++++++++++++------ 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/src/Appwrite/Deployment/Deployments.php b/src/Appwrite/Deployment/Deployments.php index 4c18f3860c..82707f05bb 100644 --- a/src/Appwrite/Deployment/Deployments.php +++ b/src/Appwrite/Deployment/Deployments.php @@ -80,9 +80,10 @@ readonly class Deployments /** * Finalizes the deployment document (see upload() for what `$deployment` * should carry) and dispatches it for building from its own uploaded - * source. Deactivates any other active deployment for $resource, marks - * it queued and writes its buildPath. Returns the persisted, updated - * deployment. + * source. Marks it queued, writes its buildPath and deactivates any other + * active deployment for $resource. A deployment canceled before it is + * queued is left canceled and never dispatched. Returns the persisted, + * updated deployment. */ public function createFromUpload(Document $resource, Document $deployment): Document { @@ -128,22 +129,44 @@ readonly class Deployments private function submit(Document $resource, Document $deployment, ?array $source): Document { + // The caller may have been holding this deployment for a while (the + // Builds worker pushes a template commit first), so its status is stale + // by now — dropping it keeps upload() from writing a cancel away, and + // the queued transition below is guarded instead. + $deployment->removeAttribute('status'); $deployment = $this->upload($resource, $deployment); - $this->deactivateOthers($resource, $deployment); - $deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ + $queued = $this->dbForProject->updateDocuments('deployments', new Document([ 'status' => 'waiting', 'buildPath' => static::buildPath($this->project->getId(), $deployment->getId()), - ])); + ]), [ + Query::equal('$id', [$deployment->getId()]), + Query::notEqual('status', 'canceled'), + ]); + + $deployment = $this->dbForProject->getDocument('deployments', $deployment->getId()); + + // Canceled before it could be queued: leave it canceled, submit nothing, + // and leave the currently active deployment alone. + if ($queued === 0) { + return $deployment; + } + + $this->deactivateOthers($resource, $deployment); try { $this->jobs->create(...static::payload($this->project, $resource, $deployment, $this->platform, $source)); } catch (\Throwable $error) { - $deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ + // Guarded like the transition above: a cancel that landed while the + // job was being submitted must not be reported as a failure. + $this->dbForProject->updateDocuments('deployments', new Document([ 'status' => 'failed', 'buildLogs' => "\nAn internal error occurred while building. Please try again, and contact support if the problem persists.\n", 'buildEndedAt' => DateTime::now(), - ])); + ]), [ + Query::equal('$id', [$deployment->getId()]), + Query::notEqual('status', 'canceled'), + ]); throw $error; } @@ -163,7 +186,7 @@ readonly class Deployments /** * Deactivates any other active deployment for $resource before this one - * goes live, called right before the deployment is marked queued. + * goes live, called once the deployment is queued for building. */ protected function deactivateOthers(Document $resource, Document $deployment): void { From f0be2d9bb31e6ac89d8143b6a2450782133d483f Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:50:41 +0100 Subject: [PATCH 3/3] fix: hand activation back when a queued deployment is canceled deactivateOthers() clears `activate` on the other pending deployments so the one being dispatched wins when it finishes. A cancel landing between the queued transition and that call left the resource with nothing able to go live: the replacement was canceled, and the deployments it displaced could no longer activate when their own builds finished. Return the ids deactivateOthers() cleared, and hand the claim back to exactly those deployments when the dispatch turns out to have been canceled. No job is submitted in that case either. Co-Authored-By: Claude Opus 5 (1M context) --- src/Appwrite/Deployment/Deployments.php | 31 ++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Deployment/Deployments.php b/src/Appwrite/Deployment/Deployments.php index 82707f05bb..336a71c6ca 100644 --- a/src/Appwrite/Deployment/Deployments.php +++ b/src/Appwrite/Deployment/Deployments.php @@ -152,7 +152,20 @@ readonly class Deployments return $deployment; } - $this->deactivateOthers($resource, $deployment); + // Claiming activation takes it away from the other pending deployments, + // so a cancel landing while we do it would leave nothing able to go + // live. Hand the claim back to exactly the deployments it was taken + // from, and stop before submitting a job for a canceled deployment. + $deactivated = $this->deactivateOthers($resource, $deployment); + if ($deactivated !== [] && $this->status($deployment->getId()) === 'canceled') { + foreach ($deactivated as $other) { + $this->dbForProject->updateDocument('deployments', $other, new Document([ + 'activate' => true, + ])); + } + + return $this->dbForProject->getDocument('deployments', $deployment->getId()); + } try { $this->jobs->create(...static::payload($this->project, $resource, $deployment, $this->platform, $source)); @@ -187,11 +200,14 @@ readonly class Deployments /** * Deactivates any other active deployment for $resource before this one * goes live, called once the deployment is queued for building. + * + * @return array The ids it deactivated, so the caller can hand the + * claim back if this deployment never gets to build. */ - protected function deactivateOthers(Document $resource, Document $deployment): void + protected function deactivateOthers(Document $resource, Document $deployment): array { if (!$deployment->getAttribute('activate', false)) { - return; + return []; } $others = $this->dbForProject->find('deployments', [ @@ -201,11 +217,20 @@ readonly class Deployments Query::notEqual('$id', $deployment->getId()), ]); + $deactivated = []; foreach ($others as $other) { $this->dbForProject->updateDocument('deployments', $other->getId(), new Document([ 'activate' => false, ])); + $deactivated[] = $other->getId(); } + + return $deactivated; + } + + private function status(string $deploymentId): string + { + return $this->dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status', ''); } /**