diff --git a/.env b/.env index b48a023df1..4a4b7245a1 100644 --- a/.env +++ b/.env @@ -117,7 +117,6 @@ _APP_SITES_TIMEOUT=30 _APP_EXECUTOR_HOST=http://exc1/v1 _APP_EXECUTOR_SECRET=your-secret-key _APP_EXECUTOR_IMAGES=openruntimes/node:v5-22,openruntimes/static:v5-1 -_APP_BUILDS_BACKEND=orchestrator _APP_BUILDS_VOLUME=appwrite-builds _APP_JOBS_HOST=http://orchestrator-jobs:8080 _APP_JOBS_SECRET=your-secret-key diff --git a/app/config/variables.php b/app/config/variables.php index e25a0f8fd2..789890ae55 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -988,18 +988,9 @@ return [ 'question' => '', 'filter' => '' ], - [ - 'name' => '_APP_BUILDS_BACKEND', - 'description' => 'Backend that builds manual-upload function deployments: "executor" (default; the open-runtimes executor, via the Builds worker) or "orchestrator" (the open-runtimes jobs-service, submitted in the request flow). Other build flows always use the executor.', - 'introduction' => '1.9.0', - 'default' => 'executor', - 'required' => false, - 'question' => '', - 'filter' => '' - ], [ 'name' => '_APP_BUILDS_VOLUME', - 'description' => 'The Docker volume (or Kubernetes PersistentVolumeClaim) holding build storage, attached to jobs-service build workers so they write output directly onto it. Must match the storage the "builds" device is backed by. Only used when _APP_BUILDS_BACKEND is "orchestrator".', + 'description' => 'The Docker volume (or Kubernetes PersistentVolumeClaim) holding build storage, attached to jobs-service build workers so they write output directly onto it. Must match the storage the "builds" device is backed by.', 'introduction' => '1.9.0', 'default' => 'appwrite-builds', 'required' => false, diff --git a/app/init/resources.php b/app/init/resources.php index cc01f3df40..f11e96c0ce 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -77,9 +77,8 @@ $container->set('jobs', function () { ->withBearerAuth(System::getEnv('_APP_JOBS_SECRET', '')) ->withTimeout(30); - // No host on executor-only installs: keep the injection resolvable and - // fail at call time instead (the client is only used when - // _APP_BUILDS_BACKEND=orchestrator, which requires _APP_JOBS_HOST). + // Keep the injection resolvable without _APP_JOBS_HOST and fail at call + // time instead, so installs that never build stay bootable. $host = System::getEnv('_APP_JOBS_HOST', ''); if ($host !== '') { $client = $client->withBaseUri($host); diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 3208c0e096..f64c8dcd23 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -5,13 +5,10 @@ use Ahc\Jwt\JWTException; use Appwrite\Auth\Key; use Appwrite\Database\Factory as DatabaseFactory; use Appwrite\Databases\TransactionState; -use Appwrite\Deployment\Backend; -use Appwrite\Deployment\Backend\Executor as ExecutorBackend; -use Appwrite\Deployment\Backend\Orchestrator; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Context\Audit as AuditContext; use Appwrite\Event\Event; use Appwrite\Event\Message\Func as FunctionMessage; -use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Event\Publisher\Func as FunctionPublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Webhook; @@ -28,7 +25,6 @@ use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; -use Executor\Executor; use OpenRuntimes\Orchestrator\Jobs; use Utopia\Agents\Adapters\Appwrite as AppwriteAdapter; use Utopia\Agents\Agent; @@ -197,13 +193,11 @@ return function (Container $context): void { $publisher, new Queue(System::getEnv('_APP_FUNCTIONS_QUEUE_NAME', Event::FUNCTIONS_QUEUE_NAME), 'utopia-queue', Event::FUNCTIONS_QUEUE_TTL) ), ['publisher']); - // Builds a Backend bound to a given project — webhook handlers resolve + // Builds a Deployments bound to a given project — webhook handlers resolve // their tenant projects mid-request, after this container is initialized. - $context->set('deploymentsFactory', function (BuildPublisher $publisherForBuilds, Jobs $jobs, Executor $executor, array $platform) { - return fn (Database $dbForProject, Document $project): Backend => System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator' - ? new Orchestrator($jobs, $dbForProject, $project, $platform) - : new ExecutorBackend($publisherForBuilds, $dbForProject, $project, $executor, $platform); - }, ['publisherForBuilds', 'jobs', 'executor', 'platform']); + $context->set('deploymentsFactory', function (Jobs $jobs, array $platform) { + return fn (Database $dbForProject, Document $project): Deployments => new Deployments($jobs, $dbForProject, $project, $platform); + }, ['jobs', 'platform']); $context->set('deployments', fn (callable $deploymentsFactory, Database $dbForProject, Document $project) => $deploymentsFactory($dbForProject, $project), ['deploymentsFactory', 'dbForProject', 'project']); $context->set('eventProcessor', fn () => new EventProcessor(), []); $context->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory( diff --git a/app/init/worker/message.php b/app/init/worker/message.php index d581820549..6f6fc39eef 100644 --- a/app/init/worker/message.php +++ b/app/init/worker/message.php @@ -1,7 +1,7 @@ set('deployments', function (Jobs $jobs, Database $dbForProject, Document $project, array $platform) { - return new Orchestrator($jobs, $dbForProject, $project, $platform); + return new Deployments($jobs, $dbForProject, $project, $platform); }, ['jobs', 'dbForProject', 'project', 'platform']); $container->set('logError', function (Registry $register, Document $project) { diff --git a/docker-compose.yml b/docker-compose.yml index b415610233..55dc9b0c0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -171,7 +171,6 @@ services: - _APP_DOMAIN_SITES - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - - _APP_BUILDS_BACKEND - _APP_BUILDS_VOLUME - _APP_JOBS_HOST - _APP_JOBS_SECRET @@ -538,7 +537,6 @@ services: - _APP_STORAGE_WASABI_BUCKET - _APP_DATABASE_SHARED_TABLES - _APP_DOMAIN_SITES - - _APP_BUILDS_BACKEND - _APP_BUILDS_VOLUME - _APP_JOBS_HOST - _APP_JOBS_SECRET diff --git a/src/Appwrite/Deployment/Backend.php b/src/Appwrite/Deployment/Backend.php deleted file mode 100644 index a8f29ae8f0..0000000000 --- a/src/Appwrite/Deployment/Backend.php +++ /dev/null @@ -1,164 +0,0 @@ -getSequence() === null) { - return $this->dbForProject->createDocument('deployments', new Document([ - '$permissions' => self::permissions(), - ...self::resourceFields($resource), - ...$deployment->getArrayCopy(), - ])); - } - - return $this->dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); - } - - /** - * Finalizes the deployment document (see upload() for what `$deployment` - * should carry) and dispatches it for building from its own uploaded - * source. Deactivates any other active deployment for $resource, marks - * it queued (and writes whatever else the backend needs, e.g. - * buildPath). Returns the persisted, updated deployment. - */ - abstract public function createFromUpload(Document $resource, Document $deployment): Document; - - /** - * Same as createFromUpload(), but builds from a public git reference - * (a template's repository) instead of the deployment's own uploaded - * source. $reference is already resolved to a concrete commit/branch/tag - * — resolving a version range (e.g. "0.3.*") is the caller's job, since - * only it holds the GitHub client that can do so. - */ - abstract public function createFromRef( - Document $resource, - Document $deployment, - string $owner, - string $repository, - string $type, - string $reference, - string $rootDirectory = '', - ): Document; - - /** - * Same as createFromUpload(), but builds from a remote tarball at $url - * (a VCS presigned URL) instead of the deployment's own uploaded source. - */ - abstract public function createFromUrl( - Document $resource, - Document $deployment, - string $url, - string $rootDirectory = '', - ): Document; - - /** - * Best-effort cancel of an in-flight build. The deployment is already - * marked canceled by the caller; this only needs to stop the backend from - * still writing to it. - */ - abstract public function cancel(string $deploymentId): void; - - /** - * Deactivates any other active deployment for $resource before this one - * goes live. Shared by both backends, called right before they mark the - * deployment queued. - */ - protected function deactivateOthers(Document $resource, Document $deployment): void - { - if (!$deployment->getAttribute('activate', false)) { - return; - } - - $others = $this->dbForProject->find('deployments', [ - Query::equal('activate', [true]), - Query::equal('resourceId', [$resource->getId()]), - Query::equal('resourceType', [$resource->getCollection()]), - Query::notEqual('$id', $deployment->getId()), - ]); - - foreach ($others as $other) { - $this->dbForProject->updateDocument('deployments', $other->getId(), new Document([ - 'activate' => false, - ])); - } - } - - /** - * The build command for a deployment: its buildCommands, wrapped for - * sites with the framework's env and bundle commands. - */ - public static function command(Document $resource, Document $deployment): string - { - $command = $deployment->getAttribute('buildCommands', ''); - if ($resource->getCollection() !== 'sites') { - return $command; - } - - $framework = Config::getParam('frameworks', [])[$resource->getAttribute('framework', '')] ?? []; - - return \implode(' && ', \array_filter([ - $framework['envCommand'] ?? '', - $command, - $framework['bundleCommand'] ?? '', - ], fn ($command) => !empty($command))); - } - - /** - * @return array - */ - private static function permissions(): array - { - return [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ]; - } - - /** - * @return array - */ - private static function resourceFields(Document $resource): array - { - return [ - 'resourceInternalId' => $resource->getSequence(), - 'resourceId' => $resource->getId(), - 'resourceType' => $resource->getCollection(), - ]; - } -} diff --git a/src/Appwrite/Deployment/Backend/Executor.php b/src/Appwrite/Deployment/Backend/Executor.php deleted file mode 100644 index f665d9e979..0000000000 --- a/src/Appwrite/Deployment/Backend/Executor.php +++ /dev/null @@ -1,87 +0,0 @@ -dispatch($resource, $deployment, null); - } - - public function createFromRef( - Document $resource, - Document $deployment, - string $owner, - string $repository, - string $type, - string $reference, - string $rootDirectory = '', - ): Document { - // No presigned URL to fetch here — the Builds worker clones the - // public repo itself via these raw git coordinates. - return $this->dispatch($resource, $deployment, new Document([ - 'ownerName' => $owner, - 'repositoryName' => $repository, - 'referenceType' => $type, - 'referenceValue' => $reference, - 'rootDirectory' => $rootDirectory, - ])); - } - - public function createFromUrl( - Document $resource, - Document $deployment, - string $url, - string $rootDirectory = '', - ): Document { - // The Builds worker clones the repo via the deployment's own - // installationId/providerRepositoryId instead of $url. - return $this->dispatch($resource, $deployment, null); - } - - private function dispatch(Document $resource, Document $deployment, ?Document $template): Document - { - $deployment = $this->upload($resource, $deployment); - $this->deactivateOthers($resource, $deployment); - - // The Builds worker promotes 'waiting' → 'building' on pickup. - $deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'status' => 'waiting', - ])); - - $this->publisherForBuilds->enqueue(new BuildMessage( - project: $this->project, - resource: $resource, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - template: $template, - platform: $this->platform, - )); - - return $deployment; - } - - public function cancel(string $deploymentId): void - { - $this->executor->deleteRuntime($this->project->getId(), $deploymentId . '-build'); - } -} diff --git a/src/Appwrite/Deployment/Backend/Orchestrator.php b/src/Appwrite/Deployment/Deployments.php similarity index 70% rename from src/Appwrite/Deployment/Backend/Orchestrator.php rename to src/Appwrite/Deployment/Deployments.php index 6d30f2d6a4..336a71c6ca 100644 --- a/src/Appwrite/Deployment/Backend/Orchestrator.php +++ b/src/Appwrite/Deployment/Deployments.php @@ -1,10 +1,8 @@ getSequence() === null) { + return $this->dbForProject->createDocument('deployments', new Document([ + '$permissions' => self::permissions(), + ...self::resourceFields($resource), + ...$deployment->getArrayCopy(), + ])); + } + return $this->dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + } + + /** + * Finalizes the deployment document (see upload() for what `$deployment` + * should carry) and dispatches it for building from its own uploaded + * source. Marks it queued, writes its buildPath and deactivates any other + * active deployment for $resource. A deployment canceled before it is + * queued is left canceled and never dispatched. Returns the persisted, + * updated deployment. + */ public function createFromUpload(Document $resource, Document $deployment): Document { return $this->submit($resource, $deployment, null); } + /** + * Same as createFromUpload(), but builds from a public git reference + * (a template's repository) instead of the deployment's own uploaded + * source. $reference is already resolved to a concrete commit/branch/tag + * — resolving a version range (e.g. "0.3.*") is the caller's job, since + * only it holds the GitHub client that can do so. + */ public function createFromRef( Document $resource, Document $deployment, @@ -73,6 +114,10 @@ readonly class Orchestrator extends Backend return $this->submit($resource, $deployment, ['url' => $url, 'subdir' => $rootDirectory]); } + /** + * Same as createFromUpload(), but builds from a remote tarball at $url + * (a VCS presigned URL) instead of the deployment's own uploaded source. + */ public function createFromUrl( Document $resource, Document $deployment, @@ -84,22 +129,57 @@ readonly class Orchestrator extends Backend private function submit(Document $resource, Document $deployment, ?array $source): Document { + // The caller may have been holding this deployment for a while (the + // Builds worker pushes a template commit first), so its status is stale + // by now — dropping it keeps upload() from writing a cancel away, and + // the queued transition below is guarded instead. + $deployment->removeAttribute('status'); $deployment = $this->upload($resource, $deployment); - $this->deactivateOthers($resource, $deployment); - $deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ + $queued = $this->dbForProject->updateDocuments('deployments', new Document([ 'status' => 'waiting', 'buildPath' => static::buildPath($this->project->getId(), $deployment->getId()), - ])); + ]), [ + Query::equal('$id', [$deployment->getId()]), + Query::notEqual('status', 'canceled'), + ]); + + $deployment = $this->dbForProject->getDocument('deployments', $deployment->getId()); + + // Canceled before it could be queued: leave it canceled, submit nothing, + // and leave the currently active deployment alone. + if ($queued === 0) { + return $deployment; + } + + // Claiming activation takes it away from the other pending deployments, + // so a cancel landing while we do it would leave nothing able to go + // live. Hand the claim back to exactly the deployments it was taken + // from, and stop before submitting a job for a canceled deployment. + $deactivated = $this->deactivateOthers($resource, $deployment); + if ($deactivated !== [] && $this->status($deployment->getId()) === 'canceled') { + foreach ($deactivated as $other) { + $this->dbForProject->updateDocument('deployments', $other, new Document([ + 'activate' => true, + ])); + } + + return $this->dbForProject->getDocument('deployments', $deployment->getId()); + } try { $this->jobs->create(...static::payload($this->project, $resource, $deployment, $this->platform, $source)); } catch (\Throwable $error) { - $deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ + // Guarded like the transition above: a cancel that landed while the + // job was being submitted must not be reported as a failure. + $this->dbForProject->updateDocuments('deployments', new Document([ 'status' => 'failed', 'buildLogs' => "\nAn internal error occurred while building. Please try again, and contact support if the problem persists.\n", 'buildEndedAt' => DateTime::now(), - ])); + ]), [ + Query::equal('$id', [$deployment->getId()]), + Query::notEqual('status', 'canceled'), + ]); throw $error; } @@ -107,11 +187,72 @@ readonly class Orchestrator extends Backend return $deployment; } + /** + * Best-effort cancel of an in-flight build. The deployment is already + * marked canceled by the caller; this only needs to stop the backend from + * still writing to it. + */ public function cancel(string $deploymentId): void { $this->jobs->delete(static::id($this->project->getId(), $deploymentId)); } + /** + * Deactivates any other active deployment for $resource before this one + * goes live, called once the deployment is queued for building. + * + * @return array The ids it deactivated, so the caller can hand the + * claim back if this deployment never gets to build. + */ + protected function deactivateOthers(Document $resource, Document $deployment): array + { + if (!$deployment->getAttribute('activate', false)) { + return []; + } + + $others = $this->dbForProject->find('deployments', [ + Query::equal('activate', [true]), + Query::equal('resourceId', [$resource->getId()]), + Query::equal('resourceType', [$resource->getCollection()]), + Query::notEqual('$id', $deployment->getId()), + ]); + + $deactivated = []; + foreach ($others as $other) { + $this->dbForProject->updateDocument('deployments', $other->getId(), new Document([ + 'activate' => false, + ])); + $deactivated[] = $other->getId(); + } + + return $deactivated; + } + + private function status(string $deploymentId): string + { + return $this->dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status', ''); + } + + /** + * The build command for a deployment: its buildCommands, wrapped for + * sites with the framework's env and bundle commands. + */ + public static function command(Document $resource, Document $deployment): string + { + $command = $deployment->getAttribute('buildCommands', ''); + if ($resource->getCollection() !== 'sites') { + return $command; + } + + $framework = Config::getParam('frameworks', [])[$resource->getAttribute('framework', '')] ?? []; + + return \implode(' && ', \array_filter([ + $framework['envCommand'] ?? '', + $command, + $framework['bundleCommand'] ?? '', + ], fn ($command) => !empty($command))); + } + /** * @return array Named arguments for OpenRuntimes\Orchestrator\Jobs::create(). */ @@ -398,4 +539,28 @@ readonly class Orchestrator extends Backend 'OPEN_RUNTIMES_NFT' => System::getEnv('_APP_OPEN_RUNTIMES_NFT', 'enabled'), ]); } + + /** + * @return array + */ + private static function permissions(): array + { + return [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ]; + } + + /** + * @return array + */ + private static function resourceFields(Document $resource): array + { + return [ + 'resourceInternalId' => $resource->getSequence(), + 'resourceId' => $resource->getId(), + 'resourceType' => $resource->getCollection(), + ]; + } } diff --git a/src/Appwrite/Platform/Modules/Compute/Base.php b/src/Appwrite/Platform/Modules/Compute/Base.php index ed69997d8c..d38efa41f1 100644 --- a/src/Appwrite/Platform/Modules/Compute/Base.php +++ b/src/Appwrite/Platform/Modules/Compute/Base.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Compute; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -10,6 +10,7 @@ use Appwrite\Filter\BranchDomain as BranchDomainFilter; use Appwrite\Platform\Action; use Appwrite\Platform\Modules\Compute\Validator\Specification as SpecificationValidator; use Appwrite\Platform\Permission as AppwritePermission; +use Appwrite\Vcs\Factory as VcsFactory; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; @@ -68,7 +69,29 @@ class Base extends Action return $allowedSpecifications[0]; } - public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, Backend $deployments, array $platform = [], string $referenceType = 'branch', string $reference = ''): Document + /** + * Templates can pin a version range (e.g. "0.3.*"); codeload only takes a + * concrete ref, so resolve a range to the highest matching tag. Templates + * are public github.com repositories regardless of the resource's own + * provider, so this always uses the GitHub adapter. + */ + public static function resolveTemplateRef(VcsFactory $vcsFactory, string $owner, string $repository, string $type, string $reference): string + { + if ($type !== Git::CLONE_TYPE_TAG || ! \str_contains($reference, '*')) { + return $reference; + } + + try { + $tags = $vcsFactory->fromProvider('github')->listTags($owner, $repository, $reference); + + return \end($tags) ?: $reference; + } catch (\Throwable) { + // Fall back to the raw reference; the build surfaces a bad ref. + return $reference; + } + } + + public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, Deployments $deployments, array $platform = [], string $referenceType = 'branch', string $reference = ''): Document { $deploymentId = ID::unique(); $entrypoint = $function->getAttribute('entrypoint', ''); @@ -144,11 +167,9 @@ class Base extends Action 'activate' => $activate, ]); - // Build a plain (non-template) VCS deployment through $deployments - // (executor or jobs-service, decided by _APP_BUILDS_BACKEND). - // Template-into-repo pushes go through the Builds worker (which does - // the git write, then hands the build to the jobs-service itself - // when on orchestrator). + // Build a plain (non-template) VCS deployment through $deployments. + // Template-into-repo pushes go through the Builds worker, which does the + // git write and then hands the build to the jobs-service itself. if ($template->isEmpty()) { $ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch'); $deployment = $deployments->createFromUrl( @@ -181,7 +202,7 @@ class Base extends Action return $deployment; } - public function redeployVcsSite(Request $request, Document $site, Document $project, Document $installation, Database $dbForProject, Database $dbForPlatform, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, Authorization $authorization, Backend $deployments, array $platform, string $referenceType = 'branch', string $reference = ''): Document + public function redeployVcsSite(Request $request, Document $site, Document $project, Document $installation, Database $dbForProject, Database $dbForPlatform, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, Authorization $authorization, Deployments $deployments, array $platform, string $referenceType = 'branch', string $reference = ''): Document { $deploymentId = ID::unique(); $providerInstallationId = $installation->getAttribute('providerInstallationId', ''); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php index 499d18248a..dce271a92f 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; @@ -106,7 +106,7 @@ class Create extends Action Document $project, Device $deviceForFunctions, Device $deviceForLocal, - Backend $deployments, + Deployments $deployments, array $plan, Authorization $authorization, callable $locks diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php index 96aa02c9ed..a0721d56cf 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Duplicate; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; @@ -78,7 +78,7 @@ class Create extends Action Database $dbForProject, Database $dbForPlatform, Event $queueForEvents, - Backend $deployments, + Deployments $deployments, Device $deviceForFunctions, VcsFactory $vcsFactory, ) { diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php index 968381057c..5342fe1381 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php @@ -2,8 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Status; -use Appwrite\Deployment\Backend; -use Appwrite\Deployment\Backend\Orchestrator; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; @@ -72,7 +71,7 @@ class Update extends Action Response $response, Database $dbForProject, Event $queueForEvents, - Backend $deployments, + Deployments $deployments, callable $locks, ) { $function = $dbForProject->getDocument('functions', $functionId); @@ -98,9 +97,9 @@ class Update extends Action // Write under the Jobs worker's per-deployment lock: its handlers // read-modify-write buildLogs, and an unserialized cancel write here // loses the closing log line to an in-flight append. - $cancel = function () use ($dbForProject, $deployment, $duration, $deployments) { + $cancel = function () use ($dbForProject, $deployment, $duration) { try { - return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); + return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration))); } catch (TransactionException) { $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); @@ -113,7 +112,7 @@ class Update extends Action } if ($deployment->getAttribute('status') !== 'canceled') { - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration))); } return $deployment; @@ -140,25 +139,20 @@ class Update extends Action } /** - * The sparse update marking a build canceled. Jobs-backed builds have no - * cancel worker to write the closing log line the executor's Builds worker - * adds, so it is appended here; executor deployments get it from their worker. + * The sparse update marking a build canceled. No worker writes the closing + * log line for a canceled build, so it is appended here. * * @return array */ - private function cancel(Document $deployment, int $duration, bool $appendLog): array + private function cancel(Document $deployment, int $duration): array { - $update = [ + $logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n"; + + return [ 'buildEndedAt' => DateTime::now(), 'buildDuration' => $duration, 'status' => 'canceled', + 'buildLogs' => \substr($logs, -APP_LOG_LENGTH_LIMIT), ]; - - if ($appendLog) { - $logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n"; - $update['buildLogs'] = \substr($logs, -APP_LOG_LENGTH_LIMIT); - } - - return $update; } } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php index 8ce0644678..39397ef8e2 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Template; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -24,7 +24,6 @@ use Utopia\Platform\Scope\HTTP; use Utopia\Validator\Boolean; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; -use Utopia\VCS\Adapter\Git\GitHub; class Create extends Base { @@ -102,7 +101,7 @@ class Create extends Base Document $project, BuildPublisher $publisherForBuilds, VcsFactory $vcsFactory, - Backend $deployments, + Deployments $deployments, Authorization $authorization, array $platform ) { @@ -126,9 +125,7 @@ class Create extends Base if (!empty($function->getAttribute('providerRepositoryId'))) { // VCS-connected function: the Builds worker merges the template into - // the user's repo, pushes it as a commit, then builds that commit — - // on the executor itself, or by submitting a job when - // _APP_BUILDS_BACKEND=orchestrator. + // the user's repo, pushes it as a commit, then builds that commit. $installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId')); $deployment = $this->redeployVcsFunction( @@ -160,24 +157,10 @@ class Create extends Base $deploymentId = ID::unique(); - // Templates can pin a version range (e.g. "0.3.*"); codeload only - // takes a concrete ref, so resolve the range to the highest matching - // tag (mirrors the executor's `git ls-remote --tags | tail -1`). - // Templates are public github.com repositories regardless of the - // function's own provider, so this always uses the GitHub adapter. - $ref = $reference; - if ($type === GitHub::CLONE_TYPE_TAG && \str_contains($reference, '*')) { - try { - $tags = $vcsFactory->fromProvider('github')->listTags($owner, $repository, $reference); - $ref = \end($tags) ?: $reference; - } catch (\Throwable) { - // Fall back to the raw reference; the build surfaces a bad ref. - } - } + $ref = Base::resolveTemplateRef($vcsFactory, $owner, $repository, $type, $reference); // Public template: pull the source straight from GitHub's public repo - // (codeload tarball on the jobs backend, a plain git clone on the - // executor); unarchive/checkout strips down to the rootDirectory. + // as a codeload tarball; unarchive strips down to the rootDirectory. $deployment = $deployments->createFromRef( $function, new Document([ diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php index f539373048..3c54b38ebc 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Vcs; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -93,7 +93,7 @@ class Create extends Base Event $queueForEvents, BuildPublisher $publisherForBuilds, VcsFactory $vcsFactory, - Backend $deployments, + Deployments $deployments, array $platform, ) { $function = $dbForProject->getDocument('functions', $functionId); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php index d486ce2e8b..eb23873b79 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php @@ -2,9 +2,8 @@ namespace Appwrite\Platform\Modules\Functions\Http\Functions; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; -use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Message\Func as FunctionMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Event\Publisher\Func as FunctionPublisher; @@ -30,8 +29,6 @@ use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Helpers\ID; -use Utopia\Database\Helpers\Permission; -use Utopia\Database\Helpers\Role; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Roles; use Utopia\Http\Request; @@ -172,7 +169,7 @@ class Create extends Base Document $project, Event $queueForEvents, BuildPublisher $publisherForBuilds, - Backend $deployments, + Deployments $deployments, Realtime $queueForRealtime, Webhook $queueForWebhooks, FunctionPublisher $publisherForFunctions, @@ -372,33 +369,33 @@ class Create extends Base ); } elseif (!$template->isEmpty()) { - // Deploy non-VCS from template - $deploymentId = ID::unique(); - $deployment = $dbForProject->createDocument('deployments', new Document([ - '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceId' => $function->getId(), - 'resourceInternalId' => $function->getSequence(), - 'resourceType' => 'functions', - 'entrypoint' => $function->getAttribute('entrypoint', ''), - 'buildCommands' => $function->getAttribute('commands', ''), - 'startCommand' => $function->getAttribute('startCommand', ''), - 'type' => 'manual', - 'activate' => true, - ])); + // Deploy non-VCS from the template's public GitHub repository. + $templateVersion = Base::resolveTemplateRef($vcsFactory, $templateOwner, $templateRepository, Git::CLONE_TYPE_TAG, $templateVersion); - $publisherForBuilds->enqueue(new BuildMessage( - project: $project, - resource: $function, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - template: $template, - platform: $platform, - )); + $deployment = $deployments->createFromRef( + $function, + new Document([ + '$id' => ID::unique(), + 'entrypoint' => $function->getAttribute('entrypoint', ''), + 'buildCommands' => $function->getAttribute('commands', ''), + 'startCommand' => $function->getAttribute('startCommand', ''), + 'providerRepositoryName' => $templateRepository, + 'providerRepositoryOwner' => $templateOwner, + 'providerRepositoryUrl' => "https://github.com/{$templateOwner}/{$templateRepository}", + 'providerBranchUrl' => "https://github.com/{$templateOwner}/{$templateRepository}/blob/{$templateVersion}", + // The coordinates a redeploy needs: remote-source builds + // never store a tarball. + 'providerBranch' => $templateVersion, + 'providerRootDirectory' => $templateRootDirectory, + 'type' => 'vcs', + 'activate' => true, + ]), + $templateOwner, + $templateRepository, + Git::CLONE_TYPE_TAG, + $templateVersion, + $templateRootDirectory, + ); } $functionsDomain = $platform['functionsDomain']; diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php index a6f7c3e047..1fa7583f18 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Functions; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Event\Validator\FunctionEvent; @@ -153,7 +153,7 @@ class Update extends Base Document $project, Event $queueForEvents, BuildPublisher $publisherForBuilds, - Backend $deployments, + Deployments $deployments, Database $dbForPlatform, VcsFactory $vcsFactory, RepositoryWebhooks $repositoryWebhooks, diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/BuildException.php b/src/Appwrite/Platform/Modules/Functions/Workers/BuildException.php index 1496fd7b72..51a854e2f2 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/BuildException.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/BuildException.php @@ -9,7 +9,7 @@ use Appwrite\Extend\Exception; * * Every build error the worker raises intentionally is a BuildException, so the * failure handler can show its message to users while masking any other - * throwable (e.g. database or executor failures) behind a generic message. + * throwable (e.g. database or jobs-service failures) behind a generic message. */ class BuildException extends Exception { diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index b5192b7628..6134155a9b 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -2,28 +2,14 @@ namespace Appwrite\Platform\Modules\Functions\Workers; -use Ahc\Jwt\JWT; -use Appwrite\Deployment\Backend; -use Appwrite\Deployment\Detection; +use Appwrite\Deployment\Deployments; use Appwrite\Deployment\GitAction; -use Appwrite\Event\Event; -use Appwrite\Event\Message\Func as FunctionMessage; -use Appwrite\Event\Publisher\Func as FunctionPublisher; -use Appwrite\Event\Publisher\Screenshot; use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; -use Appwrite\Event\Webhook; -use Appwrite\Extend\Exception as AppwriteException; -use Appwrite\Platform\Modules\Compute\Base; use Appwrite\Usage\Build as BuildUsage; use Appwrite\Usage\Context; -use Appwrite\Utopia\Response\Model\Deployment; use Appwrite\Vcs\Factory as VcsFactory; use Exception; -use Executor\Exception as ExecutorException; -use Executor\Exception\Timeout as ExecutorTimeout; -use Executor\Executor; -use Swoole\Coroutine as Co; use Utopia\Config\Config; use Utopia\Console; use Utopia\Database\Database; @@ -34,13 +20,10 @@ use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; use Utopia\Database\Exception\Transaction as TransactionException; use Utopia\Database\Query; -use Utopia\Detector\Detection\Rendering\SSR; use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\Span\Span; -use Utopia\Storage\Device; -use Utopia\Storage\Device\Local; use Utopia\System\System; use Utopia\VCS\Adapter\Git; @@ -82,23 +65,14 @@ class Builds extends Action ->inject('message') ->inject('project') ->inject('dbForPlatform') - ->inject('queueForEvents') - ->inject('publisherForScreenshots') - ->inject('queueForWebhooks') - ->inject('publisherForFunctions') ->inject('queueForRealtime') ->inject('usage') ->inject('publisherForUsage') ->inject('vcsFactory') ->inject('dbForProject') - ->inject('deviceForFunctions') - ->inject('deviceForSites') ->inject('getIsResourceBlocked') - ->inject('deviceForFiles') ->inject('log') - ->inject('executor') ->inject('deployments') - ->inject('plan') ->callback($this->action(...)); } @@ -109,23 +83,14 @@ class Builds extends Action Message $message, Document $project, Database $dbForPlatform, - Event $queueForEvents, - Screenshot $publisherForScreenshots, - Webhook $queueForWebhooks, - FunctionPublisher $publisherForFunctions, Realtime $queueForRealtime, Context $usage, UsagePublisher $publisherForUsage, VcsFactory $vcsFactory, Database $dbForProject, - Device $deviceForFunctions, - Device $deviceForSites, callable $getIsResourceBlocked, - Device $deviceForFiles, Log $log, - Executor $executor, - Backend $deployments, - array $plan + Deployments $deployments, ): void { $payload = $message->getPayload(); @@ -149,14 +114,7 @@ class Builds extends Action case BUILD_TYPE_RETRY: $templateVcs = $vcsFactory->fromProvider('github'); $this->buildDeployment( - $deviceForFunctions, - $deviceForSites, - $deviceForFiles, - $publisherForScreenshots, - $queueForWebhooks, - $publisherForFunctions, $queueForRealtime, - $queueForEvents, $usage, $publisherForUsage, $dbForPlatform, @@ -169,9 +127,7 @@ class Builds extends Action $template, $getIsResourceBlocked, $log, - $executor, $deployments, - $plan, $platform, (int) ($payload['timeout'] ?? System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900)) ); @@ -187,14 +143,7 @@ class Builds extends Action * @throws Exception */ protected function buildDeployment( - Device $deviceForFunctions, - Device $deviceForSites, - Device $deviceForFiles, - Screenshot $publisherForScreenshots, - Webhook $queueForWebhooks, - FunctionPublisher $publisherForFunctions, Realtime $queueForRealtime, - Event $queueForEvents, Context $usage, UsagePublisher $publisherForUsage, Database $dbForPlatform, @@ -207,9 +156,7 @@ class Builds extends Action Document $template, callable $getIsResourceBlocked, Log $log, - Executor $executor, - Backend $deployments, - array $plan, + Deployments $deployments, array $platform, int $timeout ): void { @@ -229,11 +176,6 @@ class Builds extends Action default => throw new \Exception('Invalid resource type') }; - $device = match ($resource->getCollection()) { - 'sites' => $deviceForSites, - 'functions' => $deviceForFunctions, - }; - $log->addTag($resourceKey, $resource->getId()); $resource = $dbForProject->getDocument($resource->getCollection(), $resource->getId()); @@ -257,7 +199,6 @@ class Builds extends Action } $version = $this->getVersion($resource); - $runtime = $this->getRuntime($resource, $version); Span::add('build.runtime', $resource->getAttribute($resource->getCollection() === 'sites' ? 'buildRuntime' : 'runtime', '')); Span::add('build.version', $version); @@ -307,843 +248,167 @@ class Builds extends Action ->setPayload($deployment->getArrayCopy()) ->trigger(); - $source = $deployment->getAttribute('sourcePath', ''); $installationId = $deployment->getAttribute('installationId', ''); $providerRepositoryId = $deployment->getAttribute('providerRepositoryId', ''); $providerCommitHash = $deployment->getAttribute('providerCommitHash', ''); - $isVcsEnabled = ! empty($providerRepositoryId); $owner = ''; $repositoryName = ''; - $providerAdapter = null; - if ($isVcsEnabled) { - $installation = $dbForPlatform->getDocument('installations', $installationId); - $providerInstallationId = $installation->getAttribute('providerInstallationId'); + // Every other build flow submits its job straight from the request, so a + // deployment without a repository to push the template into never + // belongs here. + if (empty($providerRepositoryId)) { + throw new \Exception('Only template-into-repository deployments build through this worker'); + } - try { - $providerAdapter = $vcsFactory->fromInstallation($installation); - } catch (\Exception $e) { - if ($e->getCode() === 404 - && $resource->getAttribute('installationId', '') === $installationId) { - $this->disconnectVcs($resource, $dbForProject, $dbForPlatform); - } - throw $e; + $installation = $dbForPlatform->getDocument('installations', $installationId); + $providerInstallationId = $installation->getAttribute('providerInstallationId'); + + try { + $providerAdapter = $vcsFactory->fromInstallation($installation); + } catch (\Exception $e) { + if ($e->getCode() === 404 + && $resource->getAttribute('installationId', '') === $installationId) { + $this->disconnectVcs($resource, $dbForProject, $dbForPlatform); } + throw $e; } Span::add('timings.setup', \round(\microtime(true) - $phaseStart, 3)); $phaseStart = \microtime(true); try { - if (! $isVcsEnabled) { - // Non-VCS + Template - $templateRepositoryName = $template->getAttribute('repositoryName', ''); - $templateOwnerName = $template->getAttribute('ownerName', ''); - $templateReferenceType = $template->getAttribute('referenceType', ''); - $templateReferenceValue = $template->getAttribute('referenceValue', ''); + // VCS and VCS+Temaplte + $tmpDirectory = '/tmp/builds/' . $deploymentId . '/code'; + $rootDirectory = $resource->getAttribute('providerRootDirectory', ''); + $rootDirectory = \rtrim($rootDirectory, '/'); + $rootDirectory = \ltrim($rootDirectory, '.'); + $rootDirectory = \ltrim($rootDirectory, '/'); - $templateRootDirectory = $template->getAttribute('rootDirectory', ''); - $templateRootDirectory = \rtrim($templateRootDirectory, '/'); - $templateRootDirectory = \ltrim($templateRootDirectory, '.'); - $templateRootDirectory = \ltrim($templateRootDirectory, '/'); + $owner = $providerAdapter->getOwnerName($providerInstallationId); + $repositoryName = $providerAdapter->getRepositoryName($providerRepositoryId); - if (! empty($templateRepositoryName) && ! empty($templateOwnerName) && ! empty($templateReferenceType) && ! empty($templateReferenceValue)) { - $stdout = ''; - $stderr = ''; + $cloneOwner = $deployment->getAttribute('providerRepositoryOwner', $owner); + $cloneRepository = $deployment->getAttribute('providerRepositoryName', $repositoryName); - // Clone template repo - $tmpTemplateDirectory = '/tmp/builds/' . $deploymentId . '-template'; + $branchName = $deployment->getAttribute('providerBranch'); + $commitHash = $deployment->getAttribute('providerCommitHash', ''); - $gitCloneCommandForTemplate = $templateVcs->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateReferenceValue, $templateReferenceType, $tmpTemplateDirectory, $templateRootDirectory); + $cloneVersion = $branchName; + $cloneType = Git::CLONE_TYPE_BRANCH; + if (! empty($commitHash)) { + $cloneVersion = $commitHash; + $cloneType = Git::CLONE_TYPE_COMMIT; + } - $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr); + $gitCloneCommand = $providerAdapter->generateCloneCommand($cloneOwner, $cloneRepository, $cloneVersion, $cloneType, $tmpDirectory, $rootDirectory); + $stdout = ''; + $stderr = ''; - if ($exit !== 0) { - throw new \Exception('Unable to clone code repository: ' . $stderr); - } + Console::execute('mkdir -p ' . \escapeshellarg('/tmp/builds/' . $deploymentId), '', $stdout, $stderr); - Console::execute('find ' . \escapeshellarg($tmpTemplateDirectory) . ' -type d -name ".git" -exec rm -rf {} +', '', $stdout, $stderr); + if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { + $this->finalizeCanceledDeployment($deployment->getId(), $dbForProject, $queueForRealtime); - // Ensure directories - Console::execute('mkdir -p ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr); + return; + } - $tmpPathFile = $tmpTemplateDirectory . '/code.tar.gz'; + $exit = Console::execute($gitCloneCommand, '', $stdout, $stderr); - $localDevice = new Local(); + if ($exit !== 0) { + throw new BuildException('Unable to clone code repository: ' . $stderr); + } - if (substr($tmpTemplateDirectory, -1) !== '/') { - $tmpTemplateDirectory .= '/'; - } - - $tarParamDirectory = \escapeshellarg($tmpTemplateDirectory . (empty($templateRootDirectory) ? '' : '/' . $templateRootDirectory)); - Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax - - $source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); - $result = $localDevice->transfer($tmpPathFile, $source, $device); - - if (! $result) { - throw new \Exception('Unable to move file'); - } - - $directorySize = $localDevice->getFileSize($tmpPathFile); - - Console::execute('rm -rf ' . \escapeshellarg($tmpTemplateDirectory), '', $stdout, $stderr); - $deployment - ->setAttribute('sourcePath', $source) - ->setAttribute('sourceSize', $directorySize) - ->setAttribute('totalSize', $directorySize); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'sourcePath' => $deployment->getAttribute('sourcePath'), - 'sourceSize' => $deployment->getAttribute('sourceSize'), - 'totalSize' => $deployment->getAttribute('totalSize'), - ])); - - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); - - Span::add('build.source_size', $deployment->getAttribute('sourceSize')); - } - } elseif ($isVcsEnabled) { - // VCS and VCS+Temaplte - $tmpDirectory = '/tmp/builds/' . $deploymentId . '/code'; - $rootDirectory = $resource->getAttribute('providerRootDirectory', ''); - $rootDirectory = \rtrim($rootDirectory, '/'); - $rootDirectory = \ltrim($rootDirectory, '.'); - $rootDirectory = \ltrim($rootDirectory, '/'); - - $owner = $providerAdapter->getOwnerName($providerInstallationId); - $repositoryName = $providerAdapter->getRepositoryName($providerRepositoryId); - - $cloneOwner = $deployment->getAttribute('providerRepositoryOwner', $owner); - $cloneRepository = $deployment->getAttribute('providerRepositoryName', $repositoryName); - - $branchName = $deployment->getAttribute('providerBranch'); - $commitHash = $deployment->getAttribute('providerCommitHash', ''); - - $cloneVersion = $branchName; - $cloneType = Git::CLONE_TYPE_BRANCH; - if (! empty($commitHash)) { - $cloneVersion = $commitHash; - $cloneType = Git::CLONE_TYPE_COMMIT; - } - - $gitCloneCommand = $providerAdapter->generateCloneCommand($cloneOwner, $cloneRepository, $cloneVersion, $cloneType, $tmpDirectory, $rootDirectory); - $stdout = ''; - $stderr = ''; - - Console::execute('mkdir -p ' . \escapeshellarg('/tmp/builds/' . $deploymentId), '', $stdout, $stderr); - - if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { - $this->finalizeCanceledDeployment($deployment->getId(), $dbForProject, $queueForRealtime); - - return; - } - - $exit = Console::execute($gitCloneCommand, '', $stdout, $stderr); + // Local refactoring for function folder with spaces + if (str_contains($rootDirectory, ' ')) { + $rootDirectoryWithoutSpaces = str_replace(' ', '', $rootDirectory); + $from = $tmpDirectory . '/' . $rootDirectory; + $to = $tmpDirectory . '/' . $rootDirectoryWithoutSpaces; + $exit = Console::execute('mv ' . \escapeshellarg($from) . ' ' . \escapeshellarg($to), '', $stdout, $stderr); if ($exit !== 0) { - throw new BuildException('Unable to clone code repository: ' . $stderr); + throw new \Exception('Unable to move function with spaces' . $stderr); + } + $rootDirectory = $rootDirectoryWithoutSpaces; + } + + // Build from template + $templateRepositoryName = $template->getAttribute('repositoryName', ''); + $templateOwnerName = $template->getAttribute('ownerName', ''); + $templateReferenceType = $template->getAttribute('referenceType', ''); + $templateReferenceValue = $template->getAttribute('referenceValue', ''); + + $templateRootDirectory = $template->getAttribute('rootDirectory', ''); + $templateRootDirectory = \rtrim($templateRootDirectory, '/'); + $templateRootDirectory = \ltrim($templateRootDirectory, '.'); + $templateRootDirectory = \ltrim($templateRootDirectory, '/'); + + if (! empty($templateRepositoryName) && ! empty($templateOwnerName) && ! empty($templateReferenceType) && ! empty($templateReferenceValue)) { + // Clone template repo + $tmpTemplateDirectory = '/tmp/builds/' . $deploymentId . '/template'; + + $gitCloneCommandForTemplate = $templateVcs->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateReferenceValue, $templateReferenceType, $tmpTemplateDirectory, $templateRootDirectory); + $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr); + + if ($exit !== 0) { + throw new \Exception('Unable to clone code repository: ' . $stderr); } - // Local refactoring for function folder with spaces - if (str_contains($rootDirectory, ' ')) { - $rootDirectoryWithoutSpaces = str_replace(' ', '', $rootDirectory); - $from = $tmpDirectory . '/' . $rootDirectory; - $to = $tmpDirectory . '/' . $rootDirectoryWithoutSpaces; - $exit = Console::execute('mv ' . \escapeshellarg($from) . ' ' . \escapeshellarg($to), '', $stdout, $stderr); + // Ensure directories + Console::execute('mkdir -p ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr); + Console::execute('mkdir -p ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); - if ($exit !== 0) { - throw new \Exception('Unable to move function with spaces' . $stderr); - } - $rootDirectory = $rootDirectoryWithoutSpaces; + // Merge template into user repo + Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); + + // Commit and push + $commitMessage = \escapeshellarg('Create ' . $resource->getAttribute('name', '') . ' function'); + $exit = Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($tmpDirectory) . ' && git checkout -b ' . \escapeshellarg($branchName) . ' && git add . && git commit -m ' . $commitMessage . ' && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr); + + if ($exit !== 0) { + throw new \Exception('Unable to push code repository: ' . $stderr); } - // Build from template - $templateRepositoryName = $template->getAttribute('repositoryName', ''); - $templateOwnerName = $template->getAttribute('ownerName', ''); - $templateReferenceType = $template->getAttribute('referenceType', ''); - $templateReferenceValue = $template->getAttribute('referenceValue', ''); + $exit = Console::execute('cd ' . \escapeshellarg($tmpDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr); - $templateRootDirectory = $template->getAttribute('rootDirectory', ''); - $templateRootDirectory = \rtrim($templateRootDirectory, '/'); - $templateRootDirectory = \ltrim($templateRootDirectory, '.'); - $templateRootDirectory = \ltrim($templateRootDirectory, '/'); - - if (! empty($templateRepositoryName) && ! empty($templateOwnerName) && ! empty($templateReferenceType) && ! empty($templateReferenceValue)) { - // Clone template repo - $tmpTemplateDirectory = '/tmp/builds/' . $deploymentId . '/template'; - - $gitCloneCommandForTemplate = $templateVcs->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateReferenceValue, $templateReferenceType, $tmpTemplateDirectory, $templateRootDirectory); - $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr); - - if ($exit !== 0) { - throw new \Exception('Unable to clone code repository: ' . $stderr); - } - - // Ensure directories - Console::execute('mkdir -p ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr); - Console::execute('mkdir -p ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); - - // Merge template into user repo - Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); - - // Commit and push - $commitMessage = \escapeshellarg('Create ' . $resource->getAttribute('name', '') . ' function'); - $exit = Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($tmpDirectory) . ' && git checkout -b ' . \escapeshellarg($branchName) . ' && git add . && git commit -m ' . $commitMessage . ' && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr); - - if ($exit !== 0) { - throw new \Exception('Unable to push code repository: ' . $stderr); - } - - $exit = Console::execute('cd ' . \escapeshellarg($tmpDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr); - - if ($exit !== 0) { - throw new \Exception('Unable to get vcs commit SHA: ' . $stderr); - } - - $providerCommitHash = \trim($stdout); - - $deployment->setAttribute('providerCommitHash', $providerCommitHash); - $deployment->setAttribute('providerCommitAuthorUrl', APP_VCS_GITHUB_URL); - $deployment->setAttribute('providerCommitAuthor', APP_VCS_GITHUB_USERNAME); - $deployment->setAttribute('providerCommitMessage', "Create '" . $resource->getAttribute('name', '') . "' function"); - $deployment->setAttribute('providerCommitUrl', $providerAdapter->getCommitUrl($cloneOwner, $cloneRepository, $providerCommitHash)); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'providerCommitHash' => $deployment->getAttribute('providerCommitHash'), - 'providerCommitAuthorUrl' => $deployment->getAttribute('providerCommitAuthorUrl'), - 'providerCommitAuthor' => $deployment->getAttribute('providerCommitAuthor'), - 'providerCommitMessage' => $deployment->getAttribute('providerCommitMessage'), - 'providerCommitUrl' => $deployment->getAttribute('providerCommitUrl'), - ])); - - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); + if ($exit !== 0) { + throw new \Exception('Unable to get vcs commit SHA: ' . $stderr); } - // On the jobs backend the only build reaching this worker is the - // template-into-repo push above (a git *write* the jobs-service - // artifact system has no primitive for). With the commit pushed, - // hand the build to the jobs-service like any other VCS commit, - // via the same Deployments service the HTTP endpoints use. - if (System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator') { - $ref = $deployment->getAttribute('providerCommitHash') ?: $branchName; - $deployments->createFromUrl( - $resource, - $deployment, - $providerAdapter->getRepositoryPresignedUrl($cloneOwner, $cloneRepository, $ref), - $resource->getAttribute('providerRootDirectory', ''), - ); + $providerCommitHash = \trim($stdout); - Console::execute('rm -rf ' . \escapeshellarg('/tmp/builds/' . $deploymentId), '', $stdout, $stderr); - - return; - } - - $tmpPath = '/tmp/builds/' . $deploymentId; - $tmpPathFile = $tmpPath . '/code.tar.gz'; - $localDevice = new Local(); - - if (substr($tmpDirectory, -1) !== '/') { - $tmpDirectory .= '/'; - } - - $directorySize = $localDevice->getDirectorySize($tmpDirectory); - $sizeLimit = (int) System::getEnv('_APP_COMPUTE_SIZE_LIMIT', '30000000'); - - if (isset($plan['deploymentSize'])) { - $sizeLimit = (int) $plan['deploymentSize'] * 1000 * 1000; - } - - if ($directorySize > $sizeLimit && $sizeLimit !== 0) { - throw new BuildException('Repository directory size should be less than ' . number_format($sizeLimit / (1000 * 1000), 2) . ' MBs.'); - } - - Console::execute('find ' . \escapeshellarg($tmpDirectory) . ' -type d -name ".git" -exec rm -rf {} +', '', $stdout, $stderr); - - $tarParamDirectory = '/tmp/builds/' . $deploymentId . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory); - Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax - - $source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); - $result = $localDevice->transfer($tmpPathFile, $source, $device); - - if (! $result) { - throw new \Exception('Unable to move file'); - } - - $directorySize = $localDevice->getFileSize($tmpPathFile); - - Console::execute('rm -rf ' . \escapeshellarg($tmpPath), '', $stdout, $stderr); - - $deployment - ->setAttribute('sourcePath', $source) - ->setAttribute('sourceSize', $directorySize) - ->setAttribute('totalSize', $directorySize); + $deployment->setAttribute('providerCommitHash', $providerCommitHash); + $deployment->setAttribute('providerCommitAuthorUrl', APP_VCS_GITHUB_URL); + $deployment->setAttribute('providerCommitAuthor', APP_VCS_GITHUB_USERNAME); + $deployment->setAttribute('providerCommitMessage', "Create '" . $resource->getAttribute('name', '') . "' function"); + $deployment->setAttribute('providerCommitUrl', $providerAdapter->getCommitUrl($cloneOwner, $cloneRepository, $providerCommitHash)); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'sourcePath' => $deployment->getAttribute('sourcePath'), - 'sourceSize' => $deployment->getAttribute('sourceSize'), - 'totalSize' => $deployment->getAttribute('totalSize'), + 'providerCommitHash' => $deployment->getAttribute('providerCommitHash'), + 'providerCommitAuthorUrl' => $deployment->getAttribute('providerCommitAuthorUrl'), + 'providerCommitAuthor' => $deployment->getAttribute('providerCommitAuthor'), + 'providerCommitMessage' => $deployment->getAttribute('providerCommitMessage'), + 'providerCommitUrl' => $deployment->getAttribute('providerCommitUrl'), ])); $queueForRealtime ->setPayload($deployment->getArrayCopy()) ->trigger(); - - Span::add('build.source_size', $deployment->getAttribute('sourceSize')); - - $this->runGitAction('processing', $providerAdapter, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform, $queueForRealtime, $platform); } - Span::add('timings.source', \round(\microtime(true) - $phaseStart, 3)); - $phaseStart = \microtime(true); - - /** Request the executor to build the code... */ - $updated = $dbForProject->updateDocuments('deployments', new Document([ - 'status' => 'building', - ]), [ - Query::equal('$id', [$deploymentId]), - Query::notEqual('status', 'canceled'), - ]); - - if ($updated === 0) { - $this->finalizeCanceledDeployment($deploymentId, $dbForProject, $queueForRealtime); - return; - } - - $deployment = $dbForProject->getDocument('deployments', $deploymentId); - Span::add('deployment.status', 'building'); - - $resource = $this->updateLatestDeployment($dbForProject, $resource); - - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); - - if ($isVcsEnabled) { - $this->runGitAction('building', $providerAdapter, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform, $queueForRealtime, $platform); - } - - $deploymentModel = new Deployment(); - $deploymentUpdate = - $queueForEvents - ->setProject($project) - ->setEvent("{$resource->getCollection()}.[{$resourceKey}].deployments.[deploymentId].update") - ->setParam($resourceKey, $resource->getId()) - ->setParam('deploymentId', $deployment->getId()) - ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))); - - /** Trigger Webhook */ - $queueForWebhooks - ->from($deploymentUpdate) - ->trigger(); - - /** Trigger Functions */ - $publisherForFunctions->enqueue(FunctionMessage::fromEvent( - event: $deploymentUpdate->getEvent(), - params: $deploymentUpdate->getParams(), - project: $deploymentUpdate->getProject(), - user: $deploymentUpdate->getUser(), - userId: $deploymentUpdate->getUserId(), - payload: $deploymentUpdate->getPayload(), - platform: $deploymentUpdate->getPlatform(), - )); - - /** Trigger Realtime Event */ - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); - - $vars = []; - - // Shared vars - foreach ($resource->getAttribute('varsProject', []) as $var) { - $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); - } - - // Function vars - foreach ($resource->getAttribute('vars', []) as $var) { - $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); - } - - // Some runtimes/frameworks can't compile with less memory than this - $minMemory = $resource->getCollection() === 'sites' ? 2048 : 1024; - - if ( - $resource->getAttribute('framework', '') === 'analog' || - $resource->getAttribute('framework', '') === 'tanstack-start' - ) { - $minMemory = 4096; - } - - $cpus = $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT; - $memory = max($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT, $minMemory); - $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $timeout, 0); - - $apiKey = $jwtObj->encode([ - 'projectId' => $project->getId(), - 'scopes' => $resource->getAttribute('scopes', []), - ]); - - // Appwrite vars - $vars = \array_merge($vars, [ - 'APPWRITE_VERSION' => APP_VERSION_STABLE, - 'APPWRITE_REGION' => $project->getAttribute('region'), - 'APPWRITE_DEPLOYMENT_TYPE' => $deployment->getAttribute('type', ''), - 'APPWRITE_VCS_REPOSITORY_ID' => $deployment->getAttribute('providerRepositoryId', ''), - 'APPWRITE_VCS_REPOSITORY_NAME' => $deployment->getAttribute('providerRepositoryName', ''), - 'APPWRITE_VCS_REPOSITORY_OWNER' => $deployment->getAttribute('providerRepositoryOwner', ''), - 'APPWRITE_VCS_REPOSITORY_URL' => $deployment->getAttribute('providerRepositoryUrl', ''), - 'APPWRITE_VCS_REPOSITORY_BRANCH' => $deployment->getAttribute('providerBranch', ''), - 'APPWRITE_VCS_REPOSITORY_BRANCH_URL' => $deployment->getAttribute('providerBranchUrl', ''), - 'APPWRITE_VCS_COMMIT_HASH' => $deployment->getAttribute('providerCommitHash', ''), - 'APPWRITE_VCS_COMMIT_MESSAGE' => $deployment->getAttribute('providerCommitMessage', ''), - 'APPWRITE_VCS_COMMIT_URL' => $deployment->getAttribute('providerCommitUrl', ''), - 'APPWRITE_VCS_COMMIT_AUTHOR_NAME' => $deployment->getAttribute('providerCommitAuthor', ''), - 'APPWRITE_VCS_COMMIT_AUTHOR_URL' => $deployment->getAttribute('providerCommitAuthorUrl', ''), - 'APPWRITE_VCS_ROOT_DIRECTORY' => $deployment->getAttribute('providerRootDirectory', ''), - ]); - - $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; - $endpoint = "$protocol://{$platform['apiHostname']}/v1"; - - switch ($resource->getCollection()) { - case 'functions': - $vars = [ - ...$vars, - 'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint, - 'APPWRITE_FUNCTION_API_KEY' => API_KEY_EPHEMERAL . '_' . $apiKey, - 'APPWRITE_FUNCTION_ID' => $resource->getId(), - 'APPWRITE_FUNCTION_NAME' => $resource->getAttribute('name'), - 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), - 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', - 'APPWRITE_FUNCTION_CPUS' => $cpus, - 'APPWRITE_FUNCTION_MEMORY' => $memory, - 'OPEN_RUNTIMES_NFT' => System::getEnv('_APP_OPEN_RUNTIMES_NFT', 'enabled'), - ]; - break; - case 'sites': - $vars = [ - ...$vars, - 'APPWRITE_SITE_API_ENDPOINT' => $endpoint, - 'APPWRITE_SITE_API_KEY' => API_KEY_EPHEMERAL . '_' . $apiKey, - 'APPWRITE_SITE_ID' => $resource->getId(), - 'APPWRITE_SITE_NAME' => $resource->getAttribute('name'), - 'APPWRITE_SITE_DEPLOYMENT' => $deployment->getId(), - 'APPWRITE_SITE_PROJECT_ID' => $project->getId(), - 'APPWRITE_SITE_RUNTIME_NAME' => $runtime['name'] ?? '', - 'APPWRITE_SITE_RUNTIME_VERSION' => $runtime['version'] ?? '', - 'APPWRITE_SITE_CPUS' => $cpus, - 'APPWRITE_SITE_MEMORY' => $memory, - 'OPEN_RUNTIMES_NFT' => System::getEnv('_APP_OPEN_RUNTIMES_NFT', 'enabled'), - ]; - break; - } - - $command = Backend::command($resource, $deployment); - - $cacheKey = $this->getNodeModulesCacheKey($project, $resource, $runtime, $version, $command); - - Span::add('build.node_modules_cache.enabled', $cacheKey !== ''); - Span::add('build.node_modules_cache.key', $cacheKey); - - $response = null; - $err = null; - - if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { - $this->finalizeCanceledDeployment($deployment->getId(), $dbForProject, $queueForRealtime); - - return; - } - - $isCanceled = false; - $span = Span::current(); - - Co::join([ - Co\go(function () use ($executor, &$response, $project, $deployment, $source, $resource, $runtime, $vars, $command, $cacheKey, $cpus, $memory, $timeout, &$err, $version, $span) { - try { - if ($version === 'v2') { - $command = 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh'; - } else { - $outputDirectory = $deployment->getAttribute('buildOutput') ?? $resource->getAttribute('outputDirectory'); - if ($resource->getCollection() === 'sites') { - $command = $this->prepareSiteBuildCommand($command, $outputDirectory ?? '', $resource->getAttribute('framework', '')); - } - - $command = 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh ' . \trim(\escapeshellarg($command)); - } - - $response = $executor->createRuntime( - deploymentId: $deployment->getId(), - projectId: $project->getId(), - source: $source, - image: $runtime['image'], - version: $version, - cpus: $cpus, - memory: $memory, - timeout: $timeout, - remove: true, - entrypoint: $deployment->getAttribute('entrypoint', ''), - destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", - variables: $vars, - command: $command, - outputDirectory: $outputDirectory ?? '', - cacheKey: $cacheKey - ); - - } catch (ExecutorTimeout $error) { - $span?->set('build.runtime.timed_out', true); - $span?->set('build.runtime.error_type', $error::class); - $span?->set('build.runtime.error_message', $error->getMessage()); - $err = new BuildException(type: AppwriteException::BUILD_TIMEOUT, previous: $error); - } catch (ExecutorException $error) { - $span?->set('build.runtime.error_type', $error::class); - $span?->set('build.runtime.error_message', $error->getMessage()); - $span?->set('build.runtime.executor_error_type', $error->getType()); - - $err = $error->getType() === ExecutorException::BUILD_FAILED - ? new BuildException($error->getMessage(), previous: $error) - : $error; - } catch (\Throwable $error) { - $span?->set('build.runtime.error_type', $error::class); - $span?->set('build.runtime.error_message', $error->getMessage()); - $err = $error; - } - }), - Co\go(function () use ($executor, $project, &$deployment, &$response, $dbForProject, $timeout, &$err, $queueForRealtime, &$isCanceled, $span) { - try { - $insideSeparation = false; - - $executor->getLogs( - deploymentId: $deployment->getId(), - projectId: $project->getId(), - timeout: $timeout, - callback: function ($logs) use (&$response, &$err, $dbForProject, &$isCanceled, &$deployment, $queueForRealtime, &$insideSeparation, $span) { - if ($isCanceled) { - return; - } - - // If we have response or error from concurrent coroutine, we already have latest logs - if ($response === null && $err === null) { - $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); - - if ($deployment->getAttribute('status') === 'canceled') { - $isCanceled = true; - $span?->set('build.logs.ignored_reason', 'canceled'); - - return; - } - - // Get only valid UTF8 part - removes leftover half-multibytes causing SQL errors - $logs = \mb_substr($logs, 0, null, 'UTF-8'); - - // Do not stream logs added for SSR detection - if (! $insideSeparation) { - $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR_START}'); - if ($separator !== false) { - $logs = \substr($logs, 0, $separator); - $insideSeparation = true; - - $leftover = \substr($logs, $separator + strlen('{APPWRITE_DETECTION_SEPARATOR_START}')); - $separator = \strpos($leftover, '{APPWRITE_DETECTION_SEPARATOR_END}'); - if ($separator !== false) { - $logs .= \substr($leftover, $separator + strlen('{APPWRITE_DETECTION_SEPARATOR_END}')); - $insideSeparation = false; - } - } - } else { - $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR_END}'); - if ($separator !== false) { - $logs = \substr($logs, $separator + strlen('{APPWRITE_DETECTION_SEPARATOR_END}')); - $insideSeparation = false; - } else { - $logs = ''; - } - } - - if (empty($logs)) { - return; - } - - $currentLogs = $deployment->getAttribute('buildLogs', ''); - $affected = false; - - $streamLogs = \str_replace('\\n', '{APPWRITE_LINEBREAK_PLACEHOLDER}', $logs); - foreach (\explode("\n", $streamLogs) as $streamLog) { - if (empty($streamLog)) { - continue; - } - - $streamLog = \str_replace('{APPWRITE_LINEBREAK_PLACEHOLDER}', "\n", $streamLog); - $streamParts = \explode(' ', $streamLog, 2); - - if (! isset($streamParts[1])) { - continue; - } - - // TODO: use part[0] as timestamp when switching to dbForLogs for build logs - $currentLogs .= $streamParts[1]; - - if (! empty($streamParts[1])) { - $affected = true; - } - } - - if ($affected) { - $deployment = $deployment->setAttribute('buildLogs', $this->truncateBuildLogs($currentLogs)); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'buildLogs' => $this->truncateBuildLogs($currentLogs), - ])); - - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); - } - } - } - ); - $span?->set('build.logs.finished', true); - } catch (\Throwable $error) { - $span?->set('build.logs.error_type', $error::class); - $span?->set('build.logs.error_message', $error->getMessage()); - if (empty($err)) { - $err = $error; - } - } - }), - ]); - - Span::add('timings.build', \round(\microtime(true) - $phaseStart, 3)); - $phaseStart = \microtime(true); - - $latestDeployment = $dbForProject->getDocument('deployments', $deploymentId); - if ($latestDeployment->getAttribute('status') === 'canceled') { - $this->finalizeCanceledDeployment($deployment->getId(), $dbForProject, $queueForRealtime); - - return; - } - - if ($err) { - throw $err; - } - - $buildSizeLimit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000'); - if (isset($plan['buildSize'])) { - $buildSizeLimit = $plan['buildSize'] * 1000 * 1000; - } - if ($response['size'] > $buildSizeLimit && $buildSizeLimit !== 0) { - throw new BuildException('Build size should be less than ' . number_format($buildSizeLimit / (1000 * 1000), 2) . ' MBs.'); - } - - $deployment->setAttribute('buildPath', $response['path']); - $deployment->setAttribute('buildSize', $response['size']); - $deployment->setAttribute('totalSize', $deployment->getAttribute('buildSize', 0) + $deployment->getAttribute('sourceSize', 0)); - Span::add('build.size', $deployment->getAttribute('buildSize')); - Span::add('build.total_size', $deployment->getAttribute('totalSize')); - - $logs = ''; - foreach ($response['output'] as $log) { - $logs .= $log['content']; - } - - ['logs' => $logs, 'detectionLogs' => $detectionLogs] = $this->splitSiteDetectionLogs($logs); - - $deployment->setAttribute('buildLogs', $this->truncateBuildLogs($logs)); - - $adapter = null; - if ($resource->getCollection() === 'sites' && ! empty($detectionLogs)) { - $detection = Detection::rendering($resource->getAttribute('framework', ''), \explode("\n", $detectionLogs)); - - $adapter = $resource->getAttribute('adapter', ''); - if (empty($adapter)) { - $resource = $dbForProject->updateDocument('sites', $resource->getId(), new Document(['adapter' => $detection->getName(), 'fallbackFile' => $detection->getFallbackFile() ?? ''])); - - $deployment->setAttribute('adapter', $detection->getName()); - $deployment->setAttribute('fallbackFile', $detection->getFallbackFile() ?? ''); - Span::add('build.adapter', $deployment->getAttribute('adapter')); - Span::add('build.fallback_file', $deployment->getAttribute('fallbackFile')); - } elseif ($adapter === 'ssr' && $detection->getName() === 'static') { - throw new BuildException('Adapter mismatch. Detected: ' . $detection->getName() . ' does not match with the set adapter: ' . $adapter); - } - } - - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'buildPath' => $deployment->getAttribute('buildPath'), - 'buildSize' => $deployment->getAttribute('buildSize'), - 'totalSize' => $deployment->getAttribute('totalSize'), - 'buildLogs' => $deployment->getAttribute('buildLogs'), - 'adapter' => $deployment->getAttribute('adapter'), - 'fallbackFile' => $deployment->getAttribute('fallbackFile'), - ])); - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); - - $this->afterBuildSuccess($queueForRealtime, $dbForProject, $deployment, $runtime, $adapter); - - $logs = $deployment->getAttribute('buildLogs', ''); - $date = \date('H:i:s'); - $logs .= "\033[90m[$date] \033[90m[\033[0mappwrite\033[90m]\033[32m Deployment finished. \033[0m\n"; - $deployment->setAttribute('buildLogs', $this->truncateBuildLogs($logs)); - - /** Update the status */ - $endTime = DateTime::now(); - $durationEnd = \microtime(true); - $deployment->setAttribute('buildEndedAt', $endTime); - $deployment->setAttribute('buildDuration', \intval(\ceil($durationEnd - $durationStart))); - $deployment->setAttribute('status', 'ready'); - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ - 'buildEndedAt' => $deployment->getAttribute('buildEndedAt'), - 'buildDuration' => $deployment->getAttribute('buildDuration'), - 'buildLogs' => $deployment->getAttribute('buildLogs'), - 'status' => 'ready', - ])); - Span::add('deployment.status', 'ready'); - Span::add('build.duration', $deployment->getAttribute('buildDuration')); - - $resource = $this->updateLatestDeployment($dbForProject, $resource); - - if ($isVcsEnabled) { - $this->runGitAction('ready', $providerAdapter, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform, $queueForRealtime, $platform); - } - - /** Set auto deploy */ - $activateBuild = false; - if ($deployment->getAttribute('activate') === true) { - // Check if current active deployment started later than this deployment - $resource = $dbForProject->getDocument($resource->getCollection(), $resource->getId()); - $currentActiveDeploymentId = $resource->getAttribute('deploymentId', ''); - if (! empty($currentActiveDeploymentId)) { - $currentActiveDeployment = $dbForProject->getDocument('deployments', $currentActiveDeploymentId); - if (! $currentActiveDeployment->isEmpty()) { - $currentActiveStartTime = $currentActiveDeployment->getCreatedAt(); - $deploymentStartTime = $deployment->getCreatedAt(); - - // Skip auto-activation if current active deployment started later than deployment that is being activated - if ($currentActiveStartTime < $deploymentStartTime) { - $activateBuild = true; - } else { - Span::add('build.auto_activation.skipped_reason', 'current_deployment_newer'); - } - } - } else { - $activateBuild = true; - } - } - - if ($activateBuild) { - switch ($resource->getCollection()) { - case 'functions': - $resource = $dbForProject->updateDocument('functions', $resource->getId(), new Document([ - 'live' => true, - 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getSequence(), - 'deploymentCreatedAt' => $deployment->getCreatedAt(), - ])); - - $queries = [ - Query::equal('projectInternalId', [$project->getSequence()]), - Query::equal('type', ['deployment']), - Query::equal('deploymentResourceInternalId', [$resource->getSequence()]), - Query::equal('deploymentResourceType', ['function']), - Query::equal('trigger', ['manual']), - Query::equal('deploymentVcsProviderBranch', ['']), - ]; - - $rulesUpdated = false; - $dbForPlatform->forEach('rules', function (Document $rule) use ($dbForPlatform, $deployment, &$rulesUpdated) { - $rulesUpdated = true; - $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), new Document([ - 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getSequence(), - ])); - }, $queries); - break; - case 'sites': - $resource = $dbForProject->updateDocument('sites', $resource->getId(), new Document([ - 'live' => true, - 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getSequence(), - 'deploymentCreatedAt' => $deployment->getCreatedAt(), - ])); - $queries = [ - Query::equal('projectInternalId', [$project->getSequence()]), - Query::equal('type', ['deployment']), - Query::equal('deploymentResourceInternalId', [$resource->getSequence()]), - Query::equal('deploymentResourceType', ['site']), - Query::equal('trigger', ['manual']), - Query::equal('deploymentVcsProviderBranch', ['']), - ]; - - $dbForPlatform->forEach('rules', function (Document $rule) use ($dbForPlatform, $deployment) { - $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), new Document([ - 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getSequence(), - ])); - }, $queries); - - break; - } - - Span::add('build.activated', true); - } - - $resource = $this->updateLatestDeployment($dbForProject, $resource); - - $this->afterDeploymentSuccess( - $project, + // The only build reaching this worker is the template-into-repo push + // above (a git *write* the jobs-service artifact system has no + // primitive for). With the commit pushed, hand the build to the + // jobs-service like any other VCS commit, via the same Deployments + // service the HTTP endpoints use. + $ref = $deployment->getAttribute('providerCommitHash') ?: $branchName; + $deployments->createFromUrl( + $resource, $deployment, + $providerAdapter->getRepositoryPresignedUrl($cloneOwner, $cloneRepository, $ref), + $resource->getAttribute('providerRootDirectory', ''), ); - // Send realtime event after updating the associated resource so that Console will have the resource's deployment details when re-fetching. - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); - - if ($resource->getCollection() === 'sites' && ! empty($deployment->getAttribute('providerBranch'))) { - Base::activateBranchPreviewRule($project, $resource, $deployment, $dbForPlatform, $platform['sitesDomain']); - Span::add('build.preview_rule_created', true); - } - - $queueForRealtime - ->setPayload($deployment->getArrayCopy()) - ->trigger(); - - if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { - $this->finalizeCanceledDeployment($deployment->getId(), $dbForProject, $queueForRealtime); - - return; - } - - /** Update function schedule */ - - // Inform scheduler if function is still active - if ($resource->getCollection() === 'functions') { - $schedule = $dbForPlatform->getDocument('schedules', $resource->getAttribute('scheduleId')); - $schedule - ->setAttribute('resourceUpdatedAt', DateTime::now()) - ->setAttribute('schedule', $resource->getAttribute('schedule')) - ->setAttribute('active', ! empty($resource->getAttribute('schedule')) && ! empty($resource->getAttribute('deploymentId'))); - $dbForPlatform->updateDocument('schedules', $schedule->getId(), new Document([ - 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), - 'schedule' => $schedule->getAttribute('schedule'), - 'active' => $schedule->getAttribute('active'), - ])); - } - - /** Screenshot site */ - if ($resource->getCollection() === 'sites') { - $publisherForScreenshots->enqueue(new \Appwrite\Event\Message\Screenshot( - project: $project, - deploymentId: $deployment->getId(), - )); - - Span::add('build.screenshot_queued', true); - } - - Span::add('timings.finalize', \round(\microtime(true) - $phaseStart, 3)); + Console::execute('rm -rf ' . \escapeshellarg('/tmp/builds/' . $deploymentId), '', $stdout, $stderr); } catch (\Throwable $th) { if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->finalizeCanceledDeployment($deployment->getId(), $dbForProject, $queueForRealtime); @@ -1200,9 +465,7 @@ class Builds extends Action ->setPayload($deployment->getArrayCopy()) ->trigger(); - if ($isVcsEnabled) { - $this->runGitAction('failed', $providerAdapter, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform, $queueForRealtime, $platform, true); - } + $this->runGitAction('failed', $providerAdapter, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform, $queueForRealtime, $platform, true); // Let the worker harness record internal errors via the span and logger. if (! $isUserFacing) { @@ -1228,40 +491,6 @@ class Builds extends Action BuildUsage::publish($usage, $resource, $deployment, $project, $publisherForUsage); } - /** - * Hook to run after build success - * - * @throws Exception - */ - protected function afterBuildSuccess(Realtime $queueForRealtime, Database $dbForProject, Document &$deployment, array $runtime, ?string $adapter): void - { - } - - /** - * Hook to run after deployment is activated - */ - protected function afterDeploymentSuccess( - Document $project, - Document $deployment, - ): void { - } - - protected function getRuntime(Document $resource, string $version): array - { - $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); - $key = $resource->getAttribute('runtime'); - $runtime = match ($resource->getCollection()) { - 'functions' => $runtimes[$resource->getAttribute('runtime')] ?? null, - 'sites' => $runtimes[$resource->getAttribute('buildRuntime')] ?? null, - default => null - }; - if (\is_null($runtime)) { - throw new BuildException('Runtime "' . $resource->getAttribute('runtime', '') . '" is not supported'); - } - - return $runtime; - } - protected function getVersion(Document $resource): string { return match ($resource->getCollection()) { @@ -1271,68 +500,6 @@ class Builds extends Action }; } - protected function getNodeModulesCacheKey(Document $project, Document $resource, array $runtime, string $version, string $command): string - { - if ($version !== 'v5' || $command === '' || $command === '0') { - return ''; - } - - $hashContext = [ - 'version' => 'v1', - 'projectId' => $project->getId(), - 'resourceType' => $resource->getCollection(), - 'resourceId' => $resource->getId(), - 'runtime' => $runtime['image'] ?? '', - ]; - - return \substr(\hash('sha256', \json_encode($hashContext, JSON_THROW_ON_ERROR)), 0, 48); - } - - protected function prepareSiteBuildCommand(string $command, string $outputDirectory, string $framework): string - { - $listFilesCommand = 'echo "{APPWRITE_DETECTION_SEPARATOR_START}" && cd /usr/local/build'; - - if (! empty($outputDirectory)) { - $listFilesCommand .= ' && cd ' . \escapeshellarg($outputDirectory); - } - - foreach (SSR::FRAMEWORK_FILES[$framework] ?? [] as $file) { - $listFilesCommand .= ' && ( [ -e ' . \escapeshellarg($file) . ' ] && echo ' . \escapeshellarg($file) . ' || true )'; - } - - // Static fallback detection only needs to distinguish 0, 1, or 2+ HTML files, so cap the output - $listFilesCommand .= ' && find . -name \'node_modules\' -prune -o -type f -name \'*.html\' -print | head -n 2'; - - $listFilesCommand .= ' && echo "{APPWRITE_DETECTION_SEPARATOR_END}"'; - - if (empty($command)) { - return $listFilesCommand; - } - - return "{$command} && {$listFilesCommand}"; - } - - /** - * @return array{logs: string, detectionLogs: string} - */ - protected function splitSiteDetectionLogs(string $logs): array - { - if (! \str_contains($logs, '{APPWRITE_DETECTION_SEPARATOR_START}')) { - return [ - 'logs' => $logs, - 'detectionLogs' => '', - ]; - } - - [$logsBefore, $detectionLogsStart] = \explode('{APPWRITE_DETECTION_SEPARATOR_START}', $logs, 2); - [$detectionLogs, $logsAfter] = \explode('{APPWRITE_DETECTION_SEPARATOR_END}', $detectionLogsStart, 2); - - return [ - 'logs' => "{$logsBefore}{$logsAfter}", - 'detectionLogs' => $detectionLogs, - ]; - } - /** * @throws Structure * @throws \Utopia\Database\Exception diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index 76b8880dcc..41c81ee3a5 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -30,8 +30,7 @@ use Utopia\System\System; /** * Applies open-runtimes jobs-service callbacks to a deployment. The API * callback endpoint verifies + enqueues the CloudEvent; this worker owns the - * deployment state transitions (log streaming and the ready/failed outcome) - * that the executor path performs inline in the Builds worker. + * deployment state transitions: log streaming and the ready/failed outcome. * * Callbacks arrive concurrently and out of order (one log line each), so * processing for a deployment is serialized under a per-deployment lock — that diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php index ba97b6cd7a..5581e00751 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; @@ -107,7 +107,7 @@ class Create extends Action Event $queueForEvents, Device $deviceForSites, Device $deviceForLocal, - Backend $deployments, + Deployments $deployments, array $plan, Authorization $authorization, array $platform, diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php index 905764bd92..092cec9119 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Duplicate; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; @@ -83,7 +83,7 @@ class Create extends Action Database $dbForProject, Database $dbForPlatform, Event $queueForEvents, - Backend $deployments, + Deployments $deployments, Device $deviceForSites, VcsFactory $vcsFactory, Authorization $authorization, diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php index 0d562258f5..ad9a33829a 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php @@ -2,8 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Status; -use Appwrite\Deployment\Backend; -use Appwrite\Deployment\Backend\Orchestrator; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; @@ -70,7 +69,7 @@ class Update extends Action Response $response, Database $dbForProject, Event $queueForEvents, - Backend $deployments, + Deployments $deployments, callable $locks ) { $site = $dbForProject->getDocument('sites', $siteId); @@ -96,9 +95,9 @@ class Update extends Action // Write under the Jobs worker's per-deployment lock: its handlers // read-modify-write buildLogs, and an unserialized cancel write here // loses the closing log line to an in-flight append. - $cancel = function () use ($dbForProject, $deployment, $duration, $deployments) { + $cancel = function () use ($dbForProject, $deployment, $duration) { try { - return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); + return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration))); } catch (TransactionException) { $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); @@ -111,7 +110,7 @@ class Update extends Action } if ($deployment->getAttribute('status') !== 'canceled') { - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration))); } return $deployment; @@ -138,25 +137,20 @@ class Update extends Action } /** - * The sparse update marking a build canceled. Jobs-backed builds have no - * cancel worker to write the closing log line the executor's Builds worker - * adds, so it is appended here; executor deployments get it from their worker. + * The sparse update marking a build canceled. No worker writes the closing + * log line for a canceled build, so it is appended here. * * @return array */ - private function cancel(Document $deployment, int $duration, bool $appendLog): array + private function cancel(Document $deployment, int $duration): array { - $update = [ + $logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n"; + + return [ 'buildEndedAt' => DateTime::now(), 'buildDuration' => $duration, 'status' => 'canceled', + 'buildLogs' => \substr($logs, -APP_LOG_LENGTH_LIMIT), ]; - - if ($appendLog) { - $logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n"; - $update['buildLogs'] = \substr($logs, -APP_LOG_LENGTH_LIMIT); - } - - return $update; } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php index 93cc7270b3..166ba5b18c 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Template; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -27,7 +27,6 @@ use Utopia\System\System; use Utopia\Validator\Boolean; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; -use Utopia\VCS\Adapter\Git; class Create extends Base { @@ -106,7 +105,7 @@ class Create extends Base BuildPublisher $publisherForBuilds, VcsFactory $vcsFactory, Authorization $authorization, - Backend $deployments, + Deployments $deployments, array $platform ) { $site = $dbForProject->getDocument('sites', $siteId); @@ -164,17 +163,7 @@ class Create extends Base $commands[] = $site->getAttribute('buildCommand', ''); } - // Templates can pin a version range (e.g. "0.3.*"); codeload only takes - // a concrete ref, so resolve it to the highest matching tag via GitHub. - $ref = $reference; - if ($type === Git::CLONE_TYPE_TAG && \str_contains($reference, '*')) { - try { - $tags = $vcsFactory->fromProvider('github')->listTags($owner, $repository, $reference); - $ref = \end($tags) ?: $reference; - } catch (\Throwable) { - // Fall back to the raw reference; the build surfaces a bad ref. - } - } + $ref = Base::resolveTemplateRef($vcsFactory, $owner, $repository, $type, $reference); $deploymentId = ID::unique(); $deployment = $dbForProject->createDocument('deployments', new Document([ @@ -233,8 +222,7 @@ class Create extends Base $this->updateEmptyManualRule($project, $site, $deployment, $dbForPlatform, $authorization); // Public template: pull the source straight from GitHub's public repo - // (codeload tarball on the jobs backend, a plain git clone on the - // executor); unarchive/checkout strips down to the rootDirectory. + // as a codeload tarball; unarchive strips down to the rootDirectory. $deployment = $deployments->createFromRef( $site, $deployment, diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Vcs/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Vcs/Create.php index 4e10ed7822..f2c0f7879d 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Vcs/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Vcs/Create.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Vcs; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -96,7 +96,7 @@ class Create extends Base BuildPublisher $publisherForBuilds, VcsFactory $vcsFactory, Authorization $authorization, - Backend $deployments, + Deployments $deployments, array $platform ) { $site = $dbForProject->getDocument('sites', $siteId); diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php index 216c861a82..a009e7c0ed 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Sites; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -156,7 +156,7 @@ class Update extends Base RepositoryWebhooks $repositoryWebhooks, Executor $executor, Authorization $authorization, - Backend $deployments, + Deployments $deployments, array $platform ) { if (!empty($adapter)) { diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php index a4194292b9..e2d87dea34 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php @@ -389,7 +389,7 @@ trait Deployment 'activate' => $activate, ]); - // The Backend is built per repository: a webhook fans out to + // The Deployments service is built per repository: a webhook fans out to // many tenant projects, each with its own database. $deployment = $authorization->skip(fn () => $deploymentsFactory($dbForProject, $project) ->createFromUrl( diff --git a/src/Appwrite/Usage/Build.php b/src/Appwrite/Usage/Build.php index 41da65e5e1..e2ab30893b 100644 --- a/src/Appwrite/Usage/Build.php +++ b/src/Appwrite/Usage/Build.php @@ -8,9 +8,9 @@ use Utopia\Config\Config; use Utopia\Database\Document; /** - * Emits build usage/billing metrics for a completed deployment. Shared by both - * build backends — the executor (Builds worker) and the jobs-service (Jobs - * worker) — so a build is counted identically however it was produced. + * Emits build usage/billing metrics for a completed deployment. Shared by the + * Jobs worker and the Builds worker, so a build is counted identically whether + * it finished on the jobs-service or failed before reaching it. */ final class Build { diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index 1ca966416b..64c50f7ce9 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -41,100 +41,6 @@ class Executor ]; } - /** - * Create runtime - * - * Launches a runtime container for a deployment ready for execution - * - * @param string $deploymentId - * @param string $projectId - * @param string $source - * @param string $image - * @param bool $remove - * @param string $entrypoint - * @param string $destination - * @param array $variables - * @param string $command - */ - public function createRuntime( - string $deploymentId, - string $projectId, - string $source, - string $image, - string $version, - float $cpus, - int $memory, - int $timeout, - bool $remove = false, - string $entrypoint = '', - string $destination = '', - array $variables = [], - ?string $command = null, - string $outputDirectory = '', - string $runtimeEntrypoint = '', - string $cacheKey = '' - ) { - $runtimeId = "$projectId-$deploymentId-build"; - $route = "/runtimes"; - - // Remove after migration - if ($version === 'v3' || $version === 'v4') { - $version = 'v5'; - } - - $params = [ - 'runtimeId' => $runtimeId, - 'source' => $source, - 'destination' => $destination, - 'image' => $image, - 'entrypoint' => $entrypoint, - 'variables' => $variables, - 'remove' => $remove, - 'command' => $command, - 'cpus' => $cpus, - 'memory' => $memory, - 'version' => $version, - 'timeout' => $timeout, - 'outputDirectory' => $outputDirectory, - 'runtimeEntrypoint' => $runtimeEntrypoint, - 'cacheKey' => $cacheKey - ]; - - - $response = $this->call($this->endpoint, self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout); - - $status = $response['headers']['status-code']; - if ($status >= 400) { - $message = \is_string($response['body']) ? $response['body'] : ($response['body']['message'] ?? ''); - $type = \is_array($response['body']) ? ($response['body']['type'] ?? ExecutorException::GENERAL_UNKNOWN) : ExecutorException::GENERAL_UNKNOWN; - throw new ExecutorException($message, $status, type: $type); - } - - return $response['body']; - } - - /** - * Listen to realtime logs stream of a runtime - * - * @param string $deploymentId - * @param string $projectId - * @param callable $callback - */ - public function getLogs( - string $deploymentId, - string $projectId, - string $timeout, - callable $callback - ) { - $runtimeId = "$projectId-$deploymentId-build"; - $route = "/runtimes/{$runtimeId}/logs"; - $params = [ - 'timeout' => $timeout - ]; - - $this->call($this->endpoint, self::METHOD_GET, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout, $callback); - } - /** * Delete Runtime * diff --git a/tests/unit/Deployment/BackendTest.php b/tests/unit/Deployment/DeploymentsTest.php similarity index 81% rename from tests/unit/Deployment/BackendTest.php rename to tests/unit/Deployment/DeploymentsTest.php index db024a4ef0..7fefe10893 100644 --- a/tests/unit/Deployment/BackendTest.php +++ b/tests/unit/Deployment/DeploymentsTest.php @@ -4,12 +4,12 @@ declare(strict_types=1); namespace Tests\Unit\Deployment; -use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Deployments; use PHPUnit\Framework\TestCase; use Utopia\Config\Config; use Utopia\Database\Document; -final class BackendTest extends TestCase +final class DeploymentsTest extends TestCase { public function testSiteCommandIncludesFrameworkAndDeploymentCommands(): void { @@ -30,7 +30,7 @@ final class BackendTest extends TestCase $this->assertSame( 'cp .env.example .env && npm run build && npm run bundle', - Backend::command($resource, $deployment) + Deployments::command($resource, $deployment) ); } @@ -39,6 +39,6 @@ final class BackendTest extends TestCase $resource = new Document(['$collection' => 'functions']); $deployment = new Document(['buildCommands' => 'npm install']); - $this->assertSame('npm install', Backend::command($resource, $deployment)); + $this->assertSame('npm install', Deployments::command($resource, $deployment)); } }