diff --git a/AGENTS.md b/AGENTS.md index fd67a542e0..a54bb22dfb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -227,6 +227,7 @@ Register new injections alongside existing ones in `app/init/resources.php` and - **E2E**: `tests/e2e/Services/{Service}/` mirrors the API service. Shared logic in `{Service}Base` traits; suites as `{Feature}{ConsoleClientTest|CustomClientTest|CustomServerTest}`. - **E2E methods**: `test{Verb}` or `test{Verb}{Qualifier}` -- `testCreateTeam`, `testListProjectsQuerySelect`. Omit the service name when the test class already implies it. - **Unit**: path mirrors source under `tests/unit/`; class `{ClassUnderTest}Test`. +- **Never use reflection in tests** (no `ReflectionMethod`/`ReflectionClass`/`setAccessible` to reach private members). A private method is not a contract; test through the public surface. ### Tracing span keys diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 580fa45602..bbb15a91df 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -5,6 +5,7 @@ use Ahc\Jwt\JWTException; use Appwrite\Auth\Key; use Appwrite\Database\Factory as DatabaseFactory; use Appwrite\Databases\TransactionState; +use Appwrite\Deployment\Backend; use Appwrite\Deployment\Backend\Executor as ExecutorBackend; use Appwrite\Deployment\Backend\Orchestrator; use Appwrite\Event\Context\Audit as AuditContext; @@ -196,18 +197,14 @@ return function (Container $context): void { $publisher, new Queue(System::getEnv('_APP_FUNCTIONS_QUEUE_NAME', Event::FUNCTIONS_QUEUE_NAME), 'utopia-queue', Event::FUNCTIONS_QUEUE_TTL) ), ['publisher']); - $context->set('deployments', function (BuildPublisher $publisherForBuilds, Jobs $jobs, Database $dbForProject, Document $project, Executor $executor, array $platform, Request $request) { - // The jobs-service orchestrator backend only understands functions - // (its payload builder reads function-shaped attributes like - // `runtime`); sites always build on the executor. Transitional: - // once sites build on the jobs-service too, this URI check and the - // nullable Backend params in the VCS trait can both go. - $isSite = \str_starts_with($request->getURI(), '/v1/sites'); - - return !$isSite && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator' + // Builds a Backend bound to a given project — webhook handlers resolve + // their tenant projects mid-request, after this container is initialized. + $context->set('deploymentsFactory', function (BuildPublisher $publisherForBuilds, Jobs $jobs, Executor $executor, array $platform) { + return fn (Database $dbForProject, Document $project): Backend => System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator' ? new Orchestrator($jobs, $dbForProject, $project, $platform) : new ExecutorBackend($publisherForBuilds, $dbForProject, $project, $executor, $platform); - }, ['publisherForBuilds', 'jobs', 'dbForProject', 'project', 'executor', 'platform', 'request']); + }, ['publisherForBuilds', 'jobs', 'executor', 'platform']); + $context->set('deployments', fn (callable $deploymentsFactory, Database $dbForProject, Document $project) => $deploymentsFactory($dbForProject, $project), ['deploymentsFactory', 'dbForProject', 'project']); $context->set('eventProcessor', fn () => new EventProcessor(), []); $context->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory( $pools, diff --git a/app/init/worker/message.php b/app/init/worker/message.php index bf2d2f6cb1..d581820549 100644 --- a/app/init/worker/message.php +++ b/app/init/worker/message.php @@ -188,10 +188,8 @@ return function (Container $container): void { return new TelemetryDevice($telemetry, getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId())); }, ['project', 'telemetry']); - // Workers only ever build functions on the jobs-service backend (sites - // always build on the executor via the Builds worker directly), so - // unlike the HTTP-side 'deployments' resource this doesn't need to pick - // a backend per-request. + // Only the Builds worker uses this, handing template-into-repo pushes to + // the jobs-service when _APP_BUILDS_BACKEND=orchestrator — no backend switch. $container->set('deployments', function (Jobs $jobs, Database $dbForProject, Document $project, array $platform) { return new Orchestrator($jobs, $dbForProject, $project, $platform); }, ['jobs', 'dbForProject', 'project', 'platform']); diff --git a/composer.json b/composer.json index cc575b2408..f9770cc9d1 100644 --- a/composer.json +++ b/composer.json @@ -106,7 +106,7 @@ "league/csv": "9.14.*", "enshrined/svg-sanitize": "0.22.*", "utopia-php/client": "^0.2.2", - "open-runtimes/sdk-for-php": "^0.4.0" + "open-runtimes/sdk-for-php": "0.6.*" }, "require-dev": { "ext-fileinfo": "*", @@ -124,10 +124,11 @@ "ext-phpiredis": "*" }, "config": { - "platform": {}, + "platform": { + }, "allow-plugins": { "php-http/discovery": true, "tbachert/spi": true } } -} \ No newline at end of file +} diff --git a/composer.lock b/composer.lock index 7febf8d75b..2144d45e02 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "56ff11d19729a2d94b5936e8b942a610", + "content-hash": "f007708a2c4301c72ae22feadcad6c8f", "packages": [ { "name": "adhocore/jwt", @@ -1147,16 +1147,16 @@ }, { "name": "open-runtimes/sdk-for-php", - "version": "0.4.0", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/open-runtimes/sdk-for-php.git", - "reference": "02c01ec6c807f43602d93534252e96c9451ecd80" + "reference": "17776aeee6321a05e83ab2704e410f8fd8ffd9cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/open-runtimes/sdk-for-php/zipball/02c01ec6c807f43602d93534252e96c9451ecd80", - "reference": "02c01ec6c807f43602d93534252e96c9451ecd80", + "url": "https://api.github.com/repos/open-runtimes/sdk-for-php/zipball/17776aeee6321a05e83ab2704e410f8fd8ffd9cf", + "reference": "17776aeee6321a05e83ab2704e410f8fd8ffd9cf", "shasum": "" }, "require": { @@ -1184,9 +1184,9 @@ "description": "PHP SDK for the Open Runtimes orchestrator Jobs API.", "support": { "issues": "https://github.com/open-runtimes/sdk-for-php/issues", - "source": "https://github.com/open-runtimes/sdk-for-php/tree/0.4.0" + "source": "https://github.com/open-runtimes/sdk-for-php/tree/0.6.0" }, - "time": "2026-07-13T21:55:56+00:00" + "time": "2026-07-17T09:11:17+00:00" }, { "name": "open-telemetry/api", diff --git a/docker-compose.yml b/docker-compose.yml index 0f3ee57e46..c5ea162b6f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1233,7 +1233,7 @@ services: options: max-file: "5" max-size: 10m - image: ghcr.io/open-runtimes/orchestrator/jobs-service:0.6.0 + image: ghcr.io/open-runtimes/orchestrator/jobs-service:0.8.0 restart: unless-stopped user: root networks: @@ -1244,7 +1244,7 @@ services: - ORCHESTRATOR_BACKEND=docker - ORCHESTRATOR_NETWORK=appwrite - ARTIFACT_ENDPOINT=http://orchestrator-jobs:8080 - - JOB_SIDECAR_IMAGE=ghcr.io/open-runtimes/orchestrator/job-sidecar:0.6.0 + - JOB_SIDECAR_IMAGE=ghcr.io/open-runtimes/orchestrator/job-sidecar:0.8.0 mariadb: image: mariadb:10.11 diff --git a/src/Appwrite/Deployment/Backend.php b/src/Appwrite/Deployment/Backend.php index b68d8f0276..a8f29ae8f0 100644 --- a/src/Appwrite/Deployment/Backend.php +++ b/src/Appwrite/Deployment/Backend.php @@ -2,6 +2,7 @@ namespace Appwrite\Deployment; +use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Helpers\Permission; @@ -23,11 +24,6 @@ abstract readonly class Backend ) { } - /** - * Return a backend bound to a project resolved after request initialization. - */ - abstract public function forProject(Database $dbForProject, Document $project): static; - /** * Saves chunked-upload progress onto the deployment — source path/size, * chunk counters, metadata. Never triggers a build; call createFromUpload() @@ -122,6 +118,26 @@ abstract readonly class Backend } } + /** + * The build command for a deployment: its buildCommands, wrapped for + * sites with the framework's env and bundle commands. + */ + public static function command(Document $resource, Document $deployment): string + { + $command = $deployment->getAttribute('buildCommands', ''); + if ($resource->getCollection() !== 'sites') { + return $command; + } + + $framework = Config::getParam('frameworks', [])[$resource->getAttribute('framework', '')] ?? []; + + return \implode(' && ', \array_filter([ + $framework['envCommand'] ?? '', + $command, + $framework['bundleCommand'] ?? '', + ], fn ($command) => !empty($command))); + } + /** * @return array */ diff --git a/src/Appwrite/Deployment/Backend/Executor.php b/src/Appwrite/Deployment/Backend/Executor.php index 5b9d077ca2..f665d9e979 100644 --- a/src/Appwrite/Deployment/Backend/Executor.php +++ b/src/Appwrite/Deployment/Backend/Executor.php @@ -11,7 +11,7 @@ use Utopia\Database\Document; readonly class Executor extends Backend { - final public function __construct( + public function __construct( private BuildPublisher $publisherForBuilds, Database $dbForProject, Document $project, @@ -21,10 +21,6 @@ readonly class Executor extends Backend parent::__construct($dbForProject, $project); } - public function forProject(Database $dbForProject, Document $project): static - { - return new static($this->publisherForBuilds, $dbForProject, $project, $this->executor, $this->platform); - } public function createFromUpload(Document $resource, Document $deployment): Document { diff --git a/src/Appwrite/Deployment/Backend/Orchestrator.php b/src/Appwrite/Deployment/Backend/Orchestrator.php index 9d029d3fab..15de813208 100644 --- a/src/Appwrite/Deployment/Backend/Orchestrator.php +++ b/src/Appwrite/Deployment/Backend/Orchestrator.php @@ -6,8 +6,10 @@ use Ahc\Jwt\JWT; use Appwrite\Deployment\Backend; use Appwrite\Deployment\Token; use OpenRuntimes\Orchestrator\Enum\CallbackEvent; +use OpenRuntimes\Orchestrator\Enum\ReadFormat; use OpenRuntimes\Orchestrator\Jobs; use OpenRuntimes\Orchestrator\Model\Artifact\DownloadArtifact; +use OpenRuntimes\Orchestrator\Model\Artifact\ReadArtifact; use OpenRuntimes\Orchestrator\Model\Artifact\StatArtifact; use OpenRuntimes\Orchestrator\Model\Artifact\UnarchiveArtifact; use OpenRuntimes\Orchestrator\Model\Callback; @@ -31,13 +33,14 @@ use Utopia\System\System; * upload/download artifacts instead of a shared volume) override storage() * — everything else about the payload stays the same. * - * Covers function deployments whose source is a tarball: manual upload, - * duplicate/rebuild, and templates (public GitHub tarball resolved from a - * git reference). + * Covers function and site deployments whose source is a tarball: manual + * upload, duplicate/rebuild, and templates (public GitHub tarball resolved + * from a git reference). Site builds also emit a JSON build manifest, read + * back post-job as an artifact callback for adapter detection. */ readonly class Orchestrator extends Backend { - final public function __construct( + public function __construct( private Jobs $jobs, Database $dbForProject, Document $project, @@ -46,10 +49,6 @@ readonly class Orchestrator extends Backend parent::__construct($dbForProject, $project); } - public function forProject(Database $dbForProject, Document $project): static - { - return new static($this->jobs, $dbForProject, $project, $this->platform); - } public function createFromUpload(Document $resource, Document $deployment): Document { @@ -107,20 +106,26 @@ readonly class Orchestrator extends Backend */ protected static function payload( Document $project, - Document $function, + Document $resource, Document $deployment, array $platform, ?array $source = null, ): array { $projectId = $project->getId(); $deploymentId = $deployment->getId(); + $isSite = $resource->getCollection() === 'sites'; $timeout = (int) System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900); - $version = $function->getAttribute('version', 'v2'); - $runtime = self::runtime($function, $version); - $spec = Config::getParam('specifications')[$function->getAttribute('buildSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT)]; + $runtime = self::runtime($resource, self::version($resource)); + $spec = Config::getParam('specifications')[$resource->getAttribute('buildSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT)]; $cpus = (float) ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT); - $memory = \max((int) ($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT), 1024); + + // Some runtimes/frameworks can't compile with less memory than this. + $minMemory = $isSite ? 2048 : 1024; + if (\in_array($resource->getAttribute('framework', ''), ['analog', 'tanstack-start'], true)) { + $minMemory = 4096; + } + $memory = \max((int) ($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT), $minMemory); // The jobs-service (and the containers it spawns) reach Appwrite over // the internal Docker network, so the presigned + callback URLs use an @@ -151,7 +156,7 @@ readonly class Orchestrator extends Backend // the sidecar. Bound to this deployment + direction; valid for the whole // build window plus transfer slack. $ttl = $timeout + 300; - $base = "{$endpoint}/v1/functions/{$function->getId()}/deployments/{$deploymentId}"; + $base = "{$endpoint}/v1/{$resource->getCollection()}/{$resource->getId()}/deployments/{$deploymentId}"; $sourceUrl = "{$base}/download?" . \http_build_query([ 'type' => Token::TYPE_SOURCE, 'project' => $projectId, @@ -166,13 +171,26 @@ readonly class Orchestrator extends Backend // Where output + cache land is a swappable strategy (see storage()) — // the default mounts the shared builds volume; nothing else here cares // which strategy is active. - $output = static::storage($project, $function, $deployment); + $output = static::storage($project, $resource, $deployment); - $command = $deployment->getAttribute('buildCommands', ''); - $env = self::variables($project, $function, $deployment, $runtime, $cpus, $memory, $endpoint, $timeout) + [ + // Site builds write a JSON build manifest into the workspace, read + // back post-job so the Jobs worker can run adapter detection. + $manifestArtifacts = $isSite ? [new ReadArtifact(id: 'manifest', in: 'manifest.json', format: ReadFormat::Json, depends: 'job')] : []; + + $command = self::command($resource, $deployment); + $env = self::variables($project, $resource, $deployment, $runtime, $cpus, $memory, $endpoint, $timeout) + [ 'OPEN_RUNTIMES_BUILD_INPUT_DIR' => '/mnt/code/source', 'OPEN_RUNTIMES_BUILD_COMPRESSION' => static::compression(), - ] + $output['environment']; + ] + ($isSite ? ['OPEN_RUNTIMES_BUILD_MANIFEST' => '/mnt/code/manifest.json'] : []) + $output['environment']; + + // Two terminal callbacks: exit carries the code (fires before + // post-job artifacts), complete confirms artifact delivery — the + // worker joins them, so readiness holds on any storage strategy. + // Artifact callbacks carry the source-size stat and the site manifest. + $events = [CallbackEvent::Log, CallbackEvent::Exit, CallbackEvent::Complete]; + if ($source !== null || $isSite) { + $events[] = CallbackEvent::Artifact; + } return [ 'id' => static::id($projectId, $deploymentId), @@ -185,24 +203,16 @@ readonly class Orchestrator extends Backend 'meta' => [ 'projectId' => $projectId, 'deploymentId' => $deploymentId, - 'resourceId' => $function->getId(), - 'resourceType' => 'functions', + 'resourceId' => $resource->getId(), + 'resourceType' => $resource->getCollection(), ], // The orchestrator expects environment as a string->string map. 'environment' => \array_map('strval', $env), - 'artifacts' => [...$sourceArtifacts, ...$output['artifacts']], + 'artifacts' => [...$sourceArtifacts, ...$manifestArtifacts, ...$output['artifacts']], 'volumes' => $output['volumes'], 'callback' => new Callback( url: "{$endpoint}/v1/jobs/event?" . \http_build_query(['project' => $projectId]), - // Two terminal callbacks: exit carries the code (fires on - // command exit, before post-job artifacts), complete confirms - // artifact delivery (carries only jobId + meta) — the worker - // joins them, so readiness holds on any storage strategy. - // Artifact callbacks only carry the source-size stat, which - // exists only for remote-source builds (templates / VCS). - events: $source !== null - ? [CallbackEvent::Log, CallbackEvent::Artifact, CallbackEvent::Exit, CallbackEvent::Complete] - : [CallbackEvent::Log, CallbackEvent::Exit, CallbackEvent::Complete], + events: $events, key: System::getEnv('_APP_JOBS_SECRET', ''), ), ]; @@ -251,12 +261,12 @@ readonly class Orchestrator extends Backend } /** - * Deterministic build-cache key, shared across a function's deployments so + * Deterministic build-cache key, shared across a resource's deployments so * package-manager caches (npm/yarn/pnpm) survive between builds. */ - public static function cacheKey(string $projectId, string $functionId, string $image): string + public static function cacheKey(string $projectId, string $resourceId, string $image): string { - return \substr(\hash('sha256', "{$projectId}:{$functionId}:{$image}"), 0, 48); + return \substr(\hash('sha256', "{$projectId}:{$resourceId}:{$image}"), 0, 48); } public static function cachePath(string $projectId, string $cacheKey): string @@ -284,7 +294,7 @@ readonly class Orchestrator extends Backend { $projectId = $project->getId(); $deploymentId = $deployment->getId(); - $runtime = self::runtime($resource, $resource->getAttribute('version', 'v2')); + $runtime = self::runtime($resource, self::version($resource)); $cacheKey = static::cacheKey($projectId, $resource->getId(), $runtime['image'] ?? ''); return [ @@ -302,12 +312,17 @@ readonly class Orchestrator extends Backend ]; } - protected static function runtime(Document $function, string $version): array + protected static function version(Document $resource): string { - $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); - $runtime = $runtimes[$function->getAttribute('runtime')] ?? null; + return $resource->getCollection() === 'sites' ? 'v5' : $resource->getAttribute('version', 'v2'); + } + + protected static function runtime(Document $resource, string $version): array + { + $key = $resource->getAttribute($resource->getCollection() === 'sites' ? 'buildRuntime' : 'runtime'); + $runtime = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', [])[$key] ?? null; if ($runtime === null) { - throw new \Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); + throw new \Exception('Runtime "' . $key . '" is not supported'); } return $runtime; @@ -315,7 +330,7 @@ readonly class Orchestrator extends Backend private static function variables( Document $project, - Document $function, + Document $resource, Document $deployment, array $runtime, float $cpus, @@ -325,35 +340,49 @@ readonly class Orchestrator extends Backend ): array { $vars = []; - foreach ($function->getAttribute('varsProject', []) as $var) { + foreach ($resource->getAttribute('varsProject', []) as $var) { $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); } - foreach ($function->getAttribute('vars', []) as $var) { + foreach ($resource->getAttribute('vars', []) as $var) { $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); } $apiKey = (new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $timeout, 0))->encode([ 'projectId' => $project->getId(), - 'scopes' => $function->getAttribute('scopes', []), + 'scopes' => $resource->getAttribute('scopes', []), ]); + $prefix = $resource->getCollection() === 'sites' ? 'SITE' : 'FUNCTION'; + return \array_merge($vars, [ // Consumed by the open-runtimes build helper (build.sh). 'OPEN_RUNTIMES_ENTRYPOINT' => $deployment->getAttribute('entrypoint', ''), - 'OPEN_RUNTIMES_OUTPUT_DIRECTORY' => $deployment->getAttribute('buildOutput', '') ?: $function->getAttribute('outputDirectory', ''), + 'OPEN_RUNTIMES_OUTPUT_DIRECTORY' => $deployment->getAttribute('buildOutput', '') ?: $resource->getAttribute('outputDirectory', ''), 'APPWRITE_VERSION' => APP_VERSION_STABLE, 'APPWRITE_REGION' => $project->getAttribute('region'), 'APPWRITE_DEPLOYMENT_TYPE' => $deployment->getAttribute('type', ''), - 'APPWRITE_FUNCTION_API_ENDPOINT' => "{$endpoint}/v1", - 'APPWRITE_FUNCTION_API_KEY' => API_KEY_EPHEMERAL . '_' . $apiKey, - 'APPWRITE_FUNCTION_ID' => $function->getId(), - 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), - 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), - 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', - 'APPWRITE_FUNCTION_CPUS' => $cpus, - 'APPWRITE_FUNCTION_MEMORY' => $memory, + 'APPWRITE_VCS_REPOSITORY_ID' => $deployment->getAttribute('providerRepositoryId', ''), + 'APPWRITE_VCS_REPOSITORY_NAME' => $deployment->getAttribute('providerRepositoryName', ''), + 'APPWRITE_VCS_REPOSITORY_OWNER' => $deployment->getAttribute('providerRepositoryOwner', ''), + 'APPWRITE_VCS_REPOSITORY_URL' => $deployment->getAttribute('providerRepositoryUrl', ''), + 'APPWRITE_VCS_REPOSITORY_BRANCH' => $deployment->getAttribute('providerBranch', ''), + 'APPWRITE_VCS_REPOSITORY_BRANCH_URL' => $deployment->getAttribute('providerBranchUrl', ''), + 'APPWRITE_VCS_COMMIT_HASH' => $deployment->getAttribute('providerCommitHash', ''), + 'APPWRITE_VCS_COMMIT_MESSAGE' => $deployment->getAttribute('providerCommitMessage', ''), + 'APPWRITE_VCS_COMMIT_URL' => $deployment->getAttribute('providerCommitUrl', ''), + 'APPWRITE_VCS_COMMIT_AUTHOR_NAME' => $deployment->getAttribute('providerCommitAuthor', ''), + 'APPWRITE_VCS_COMMIT_AUTHOR_URL' => $deployment->getAttribute('providerCommitAuthorUrl', ''), + 'APPWRITE_VCS_ROOT_DIRECTORY' => $deployment->getAttribute('providerRootDirectory', ''), + "APPWRITE_{$prefix}_API_ENDPOINT" => "{$endpoint}/v1", + "APPWRITE_{$prefix}_API_KEY" => API_KEY_EPHEMERAL . '_' . $apiKey, + "APPWRITE_{$prefix}_ID" => $resource->getId(), + "APPWRITE_{$prefix}_NAME" => $resource->getAttribute('name'), + "APPWRITE_{$prefix}_DEPLOYMENT" => $deployment->getId(), + "APPWRITE_{$prefix}_PROJECT_ID" => $project->getId(), + "APPWRITE_{$prefix}_RUNTIME_NAME" => $runtime['name'] ?? '', + "APPWRITE_{$prefix}_RUNTIME_VERSION" => $runtime['version'] ?? '', + "APPWRITE_{$prefix}_CPUS" => $cpus, + "APPWRITE_{$prefix}_MEMORY" => $memory, 'OPEN_RUNTIMES_NFT' => System::getEnv('_APP_OPEN_RUNTIMES_NFT', 'enabled'), ]); } diff --git a/src/Appwrite/Deployment/Detection.php b/src/Appwrite/Deployment/Detection.php new file mode 100644 index 0000000000..b34f30fa53 --- /dev/null +++ b/src/Appwrite/Deployment/Detection.php @@ -0,0 +1,34 @@ + $files + */ + public static function rendering(string $framework, array $files): object + { + $files = \array_map(\trim(...), $files); + $files = \array_filter($files); + $files = \array_map(fn ($file) => \str_starts_with($file, './') ? \substr($file, 2) : $file, $files); + + $detector = new Rendering($framework); + foreach ($files as $file) { + $detector->addInput($file); + } + + return $detector + ->addOption(new SSR()) + ->addOption(new XStatic()) + ->detect(); + } +} diff --git a/src/Appwrite/Deployment/GitAction.php b/src/Appwrite/Deployment/GitAction.php new file mode 100644 index 0000000000..03d679b2ba --- /dev/null +++ b/src/Appwrite/Deployment/GitAction.php @@ -0,0 +1,97 @@ +getAttribute('providerSilentMode', false) === true) { + return; + } + + $isSite = $resource->getCollection() === 'sites'; + $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; + + if (!empty($commitHash)) { + $message = match ($status) { + 'ready' => 'Build succeeded.', + 'failed' => 'Build failed.', + 'processing' => 'Building...', + default => $status + }; + $state = match ($status) { + 'ready' => 'success', + 'failed' => 'failure', + 'processing' => 'pending', + default => $status + }; + + $hostname = System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN', '')); + $region = $project->getAttribute('region', 'default'); + $segment = $isSite ? "sites/site-{$resource->getId()}" : "functions/function-{$resource->getId()}"; + $targetUrl = "{$protocol}://{$hostname}/console/project-{$region}-{$project->getId()}/{$segment}"; + $name = $resource->getAttribute('name') . ' (' . $project->getAttribute('name') . ')'; + + $vcs->updateCommitStatus($repositoryName, $commitHash, $owner, $state, $message, $targetUrl, $name); + } + + $commentId = $deployment->getAttribute('providerCommentId', ''); + if (empty($commentId)) { + return; + } + + // Serialize comment updates across concurrent builds via a lock document. + $retries = 0; + while (true) { + try { + $dbForPlatform->createDocument('vcsCommentLocks', new Document(['$id' => $commentId])); + break; + } catch (\Throwable $err) { + if (++$retries >= 9) { + throw $err; + } + \sleep(1); + } + } + + try { + $rule = $dbForPlatform->findOne('rules', [ + Query::equal('projectInternalId', [$project->getSequence()]), + Query::equal('type', ['deployment']), + Query::equal('deploymentInternalId', [$deployment->getSequence()]), + ]); + $previewUrl = $isSite && !$rule->isEmpty() ? "{$protocol}://" . $rule->getAttribute('domain', '') : ''; + + $comment = new Comment($platform); + $comment->parseComment($vcs->getComment($owner, $repositoryName, $commentId)); + $comment->addBuild($project, $resource, $isSite ? 'site' : 'function', $status, $deployment->getId(), ['type' => 'logs'], $previewUrl); + $vcs->updateComment($owner, $repositoryName, $commentId, $comment->generateComment()); + } finally { + $dbForPlatform->deleteDocument('vcsCommentLocks', $commentId); + } + } +} diff --git a/src/Appwrite/Platform/Modules/Compute/Base.php b/src/Appwrite/Platform/Modules/Compute/Base.php index 60591096ce..ed69997d8c 100644 --- a/src/Appwrite/Platform/Modules/Compute/Base.php +++ b/src/Appwrite/Platform/Modules/Compute/Base.php @@ -68,7 +68,7 @@ class Base extends Action return $allowedSpecifications[0]; } - public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Backend $deployments = null): Document + public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, Backend $deployments, array $platform = [], string $referenceType = 'branch', string $reference = ''): Document { $deploymentId = ID::unique(); $entrypoint = $function->getAttribute('entrypoint', ''); @@ -144,13 +144,12 @@ class Base extends Action 'activate' => $activate, ]); - // Build a plain (non-template) VCS function deployment through - // $deployments (executor or jobs-service, decided by - // _APP_BUILDS_BACKEND) when the caller opts in. Sites stay on the - // executor; template-into-repo pushes go through the Builds worker - // (which does the git write, then hands the build to the - // jobs-service itself when on orchestrator). - if ($deployments !== null && $function->getCollection() === 'functions' && $template->isEmpty()) { + // Build a plain (non-template) VCS deployment through $deployments + // (executor or jobs-service, decided by _APP_BUILDS_BACKEND). + // Template-into-repo pushes go through the Builds worker (which does + // the git write, then hands the build to the jobs-service itself + // when on orchestrator). + if ($template->isEmpty()) { $ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch'); $deployment = $deployments->createFromUrl( $function, @@ -182,7 +181,7 @@ class Base extends Action return $deployment; } - public function redeployVcsSite(Request $request, Document $site, Document $project, Document $installation, Database $dbForProject, Database $dbForPlatform, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, Authorization $authorization, array $platform, string $referenceType = 'branch', string $reference = ''): Document + public function redeployVcsSite(Request $request, Document $site, Document $project, Document $installation, Database $dbForProject, Database $dbForPlatform, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, Authorization $authorization, Backend $deployments, array $platform, string $referenceType = 'branch', string $reference = ''): Document { $deploymentId = ID::unique(); $providerInstallationId = $installation->getAttribute('providerInstallationId', ''); @@ -369,18 +368,93 @@ class Base extends Action $this->updateEmptyManualRule($project, $site, $deployment, $dbForPlatform, $authorization); - $publisherForBuilds->enqueue(new BuildMessage( - project: $project, - resource: $site, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - template: $template, - platform: $platform, - )); + // Plain VCS deployments build through $deployments; template-into-repo + // pushes go through the Builds worker (same split as redeployVcsFunction). + if ($template->isEmpty()) { + $ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch'); + $deployment = $deployments->createFromUrl( + $site, + $deployment, + $vcs->getRepositoryPresignedUrl($owner, $repositoryName, $ref), + $site->getAttribute('providerRootDirectory', ''), + ); + } else { + $publisherForBuilds->enqueue(new BuildMessage( + project: $project, + resource: $site, + deployment: $deployment, + type: BUILD_TYPE_DEPLOYMENT, + template: $template, + platform: $platform, + )); + } return $deployment; } + /** + * Create or repoint the site's branch-preview rule — and any manual rules + * pinned to the branch — at a successfully built deployment. + */ + public static function activateBranchPreviewRule(Document $project, Document $site, Document $deployment, Database $dbForPlatform, string $sitesDomain): void + { + // Template deployments reuse providerBranch for their resolved ref + // (tags included), which must not mint a preview domain. + $branchName = $deployment->getAttribute('providerBranch', ''); + if (empty($branchName) || empty($deployment->getAttribute('installationId'))) { + return; + } + + $domain = (new BranchDomainFilter())->apply([ + 'branch' => $branchName, + 'resourceId' => $site->getId(), + 'projectId' => $project->getId(), + 'sitesDomain' => $sitesDomain, + ]); + $ruleId = md5($domain); + + try { + $dbForPlatform->createDocument('rules', new Document([ + '$id' => $ruleId, + 'projectId' => $project->getId(), + 'projectInternalId' => $project->getSequence(), + 'domain' => $domain, + 'type' => 'deployment', + 'trigger' => 'deployment', + 'deploymentId' => $deployment->getId(), + 'deploymentInternalId' => $deployment->getSequence(), + 'deploymentResourceType' => 'site', + 'deploymentResourceId' => $site->getId(), + 'deploymentResourceInternalId' => $site->getSequence(), + 'deploymentVcsProviderBranch' => $branchName, + 'status' => 'verified', + 'certificateId' => '', + 'search' => implode(' ', [$ruleId, $domain]), + 'owner' => 'Appwrite', + 'region' => $project->getAttribute('region'), + ])); + } catch (Duplicate) { + $dbForPlatform->updateDocument('rules', $ruleId, new Document([ + 'deploymentId' => $deployment->getId(), + 'deploymentInternalId' => $deployment->getSequence(), + ])); + } + + $dbForPlatform->forEach('rules', function (Document $rule) use ($dbForPlatform, $deployment) { + $dbForPlatform->updateDocument('rules', $rule->getId(), new Document([ + 'deploymentId' => $deployment->getId(), + 'deploymentInternalId' => $deployment->getSequence(), + ])); + }, [ + Query::equal('projectInternalId', [$project->getSequence()]), + Query::equal('type', ['deployment']), + Query::equal('deploymentResourceInternalId', [$site->getSequence()]), + Query::equal('deploymentResourceType', ['site']), + Query::equal('deploymentVcsProviderBranch', [$branchName]), + Query::equal('trigger', ['manual']), + ]); + } + /** * Update empty manual rule for deployment. * In case of first deployment, deployment ID will be empty in the rules, so we need to update it here. diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php index c80249d8a9..96aa02c9ed 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php @@ -134,6 +134,9 @@ class Create extends Action 'buildSize' => null, 'buildPath' => '', 'buildLogs' => '', + // Not inherited: a redeploy always goes live, and the source's own + // flag is unset by deactivateOthers() once anything newer builds. + 'activate' => true, ]); if ($hasSource) { @@ -154,14 +157,14 @@ class Create extends Action $deployment->getAttribute('providerRootDirectory', ''), ); } else { - // Public template repo: providerBranch holds the resolved ref - // (branch, tag, or commit — see Deployments/Template/Create). + // Public template repo: providerBranch holds the resolved ref, + // fetched commit-style since a branch-type clone breaks on tags. $deployment = $deployments->createFromRef( $function, $deployment, $owner, $repository, - GitHub::CLONE_TYPE_BRANCH, + GitHub::CLONE_TYPE_COMMIT, $deployment->getAttribute('providerBranch', ''), $deployment->getAttribute('providerRootDirectory', ''), ); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php index 1380428bf7..968381057c 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php @@ -15,6 +15,7 @@ use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Transaction as TransactionException; use Utopia\Database\Validator\UID; +use Utopia\Lock\Exception\Contention; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; @@ -61,6 +62,7 @@ class Update extends Action ->inject('dbForProject') ->inject('queueForEvents') ->inject('deployments') + ->inject('locks') ->callback($this->action(...)); } @@ -71,6 +73,7 @@ class Update extends Action Database $dbForProject, Event $queueForEvents, Backend $deployments, + callable $locks, ) { $function = $dbForProject->getDocument('functions', $functionId); @@ -92,22 +95,35 @@ class Update extends Action $endTime = new \DateTime('now'); $duration = $endTime->getTimestamp() - $startTime->getTimestamp(); + // Write under the Jobs worker's per-deployment lock: its handlers + // read-modify-write buildLogs, and an unserialized cancel write here + // loses the closing log line to an in-flight append. + $cancel = function () use ($dbForProject, $deployment, $duration, $deployments) { + try { + return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); + } catch (TransactionException) { + $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); + + if ($deployment->isEmpty()) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + + if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'])) { + throw new Exception(Exception::BUILD_ALREADY_COMPLETED); + } + + if ($deployment->getAttribute('status') !== 'canceled') { + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); + } + + return $deployment; + } + }; + try { - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); - } catch (TransactionException) { - $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); - - if ($deployment->isEmpty()) { - throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); - } - - if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'])) { - throw new Exception(Exception::BUILD_ALREADY_COMPLETED); - } - - if ($deployment->getAttribute('status') !== 'canceled') { - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); - } + $deployment = $locks('jobs-deployment:' . $deploymentId, 30, $cancel, 10.0); + } catch (Contention) { + $deployment = $cancel(); } // Best-effort cleanup — the deployment is already marked 'canceled'. diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php index b44c8a7ef8..a6f7c3e047 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php @@ -329,7 +329,7 @@ class Update extends Base // Redeploy logic if (!$isConnected && !empty($providerRepositoryId)) { - $this->redeployVcsFunction($request, $function, $project, $installation, $dbForProject, $publisherForBuilds, new Document(), $vcsFactory->fromInstallation($installation), true, $platform, deployments: $deployments); + $this->redeployVcsFunction($request, $function, $project, $installation, $dbForProject, $publisherForBuilds, new Document(), $vcsFactory->fromInstallation($installation), true, $deployments, $platform); } // Inform scheduler if function is still active diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 3b2daf976b..7ce1220d66 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -4,6 +4,8 @@ namespace Appwrite\Platform\Modules\Functions\Workers; use Ahc\Jwt\JWT; use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Detection; +use Appwrite\Deployment\GitAction; use Appwrite\Event\Event; use Appwrite\Event\Message\Func as FunctionMessage; use Appwrite\Event\Publisher\Func as FunctionPublisher; @@ -12,11 +14,10 @@ use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Webhook; use Appwrite\Extend\Exception as AppwriteException; -use Appwrite\Filter\BranchDomain as BranchDomainFilter; +use Appwrite\Platform\Modules\Compute\Base; use Appwrite\Usage\Build as BuildUsage; use Appwrite\Usage\Context; use Appwrite\Utopia\Response\Model\Deployment; -use Appwrite\Vcs\Comment; use Appwrite\Vcs\Factory as VcsFactory; use Exception; use Executor\Exception as ExecutorException; @@ -29,14 +30,11 @@ use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Conflict; -use Utopia\Database\Exception\Duplicate; use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; use Utopia\Database\Exception\Transaction as TransactionException; use Utopia\Database\Query; use Utopia\Detector\Detection\Rendering\SSR; -use Utopia\Detector\Detection\Rendering\XStatic; -use Utopia\Detector\Detector\Rendering; use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; @@ -529,8 +527,7 @@ class Builds extends Action // artifact system has no primitive for). With the commit pushed, // hand the build to the jobs-service like any other VCS commit, // via the same Deployments service the HTTP endpoints use. - if ($resource->getCollection() === 'functions' - && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator') { + if (System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator') { $ref = $deployment->getAttribute('providerCommitHash') ?: $branchName; $deployments->createFromUrl( $resource, @@ -745,10 +742,7 @@ class Builds extends Action break; } - $command = $this->getCommand( - resource: $resource, - deployment: $deployment - ); + $command = Backend::command($resource, $deployment); $cacheKey = $this->getNodeModulesCacheKey($project, $resource, $runtime, $version, $command); @@ -960,7 +954,7 @@ class Builds extends Action $adapter = null; if ($resource->getCollection() === 'sites' && ! empty($detectionLogs)) { - $detection = $this->detectSiteRendering($resource->getAttribute('framework', ''), $detectionLogs); + $detection = Detection::rendering($resource->getAttribute('framework', ''), \explode("\n", $detectionLogs)); $adapter = $resource->getAttribute('adapter', ''); if (empty($adapter)) { @@ -1108,63 +1102,9 @@ class Builds extends Action ->setPayload($deployment->getArrayCopy()) ->trigger(); - if ($resource->getCollection() === 'sites') { - // VCS branch - $branchName = $deployment->getAttribute('providerBranch'); - if (! empty($branchName)) { - $domain = (new BranchDomainFilter())->apply([ - 'branch' => $branchName, - 'resourceId' => $resource->getId(), - 'projectId' => $project->getId(), - 'sitesDomain' => $platform['sitesDomain'], - ]); - $ruleId = md5($domain); - - try { - $dbForPlatform->createDocument('rules', new Document([ - '$id' => $ruleId, - 'projectId' => $project->getId(), - 'projectInternalId' => $project->getSequence(), - 'domain' => $domain, - 'type' => 'deployment', - 'trigger' => 'deployment', - 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getSequence(), - 'deploymentResourceType' => 'site', - 'deploymentResourceId' => $resource->getId(), - 'deploymentResourceInternalId' => $resource->getSequence(), - 'deploymentVcsProviderBranch' => $branchName, - 'status' => 'verified', - 'certificateId' => '', - 'search' => implode(' ', [$ruleId, $domain]), - 'owner' => 'Appwrite', - 'region' => $project->getAttribute('region'), - ])); - } catch (Duplicate $err) { - $rule = $dbForPlatform->updateDocument('rules', $ruleId, new Document([ - 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getSequence(), - ])); - } - - $queries = [ - Query::equal('projectInternalId', [$project->getSequence()]), - Query::equal('type', ['deployment']), - Query::equal('deploymentResourceInternalId', [$resource->getSequence()]), - Query::equal('deploymentResourceType', ['site']), - Query::equal('deploymentVcsProviderBranch', [$branchName]), - Query::equal('trigger', ['manual']), - ]; - - $dbForPlatform->foreach('rules', function (Document $rule) use ($dbForPlatform, $deployment) { - $rule = $dbForPlatform->updateDocument('rules', $rule->getId(), new Document([ - 'deploymentId' => $deployment->getId(), - 'deploymentInternalId' => $deployment->getSequence(), - ])); - }, $queries); - - Span::add('build.preview_rule_created', true); - } + if ($resource->getCollection() === 'sites' && ! empty($deployment->getAttribute('providerBranch'))) { + Base::activateBranchPreviewRule($project, $resource, $deployment, $dbForPlatform, $platform['sitesDomain']); + Span::add('build.preview_rule_created', true); } $queueForRealtime @@ -1331,35 +1271,6 @@ class Builds extends Action }; } - protected function getCommand(Document $resource, Document $deployment): string - { - if ($resource->getCollection() === 'functions') { - return $deployment->getAttribute('buildCommands', ''); - } elseif ($resource->getCollection() === 'sites') { - $commands = []; - - $frameworks = Config::getParam('frameworks', []); - $framework = $frameworks[$resource->getAttribute('framework', '')] ?? null; - - $envCommand = ''; - $bundleCommand = ''; - if (! is_null($framework)) { - $envCommand = $framework['envCommand'] ?? ''; - $bundleCommand = $framework['bundleCommand'] ?? ''; - } - - $commands[] = $envCommand; - $commands[] = $deployment->getAttribute('buildCommands', ''); - $commands[] = $bundleCommand; - - $commands = array_filter($commands, fn ($command) => ! empty($command)); - - return implode(' && ', $commands); - } - - return ''; - } - protected function getNodeModulesCacheKey(Document $project, Document $resource, array $runtime, string $version, string $command): string { if ($version !== 'v5' || $command === '' || $command === '0') { @@ -1422,24 +1333,6 @@ class Builds extends Action ]; } - protected function detectSiteRendering(string $framework, string $detectionLogs): object - { - $files = \explode("\n", $detectionLogs); - $files = \array_filter($files); - $files = \array_map(\trim(...), $files); - $files = \array_map(fn ($file) => \str_starts_with($file, './') ? \substr($file, 2) : $file, $files); - - $detector = new Rendering($framework); - foreach ($files as $file) { - $detector->addInput($file); - } - - return $detector - ->addOption(new SSR()) - ->addOption(new XStatic()) - ->detect(); - } - /** * @throws Structure * @throws \Utopia\Database\Exception @@ -1464,96 +1357,9 @@ class Builds extends Action $deployment = new Document(); try { - if ($resource->getAttribute('providerSilentMode', false) === true) { - return; - } - $deployment = $dbForProject->getDocument('deployments', $deploymentId); - $commentId = $deployment->getAttribute('providerCommentId', ''); - if (! empty($providerCommitHash)) { - $message = match ($status) { - 'ready' => 'Build succeeded.', - 'failed' => 'Build failed.', - 'processing' => 'Building...', - default => $status - }; - - $state = match ($status) { - 'ready' => 'success', - 'failed' => 'failure', - 'processing' => 'pending', - default => $status - }; - - $resourceName = $resource->getAttribute('name'); - $projectName = $project->getAttribute('name'); - - $name = "{$resourceName} ({$projectName})"; - - $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; - $hostname = System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN', '')); - - $projectId = $project->getId(); - $region = $project->getAttribute('region', 'default'); - $resourceId = $resource->getId(); - $providerTargetUrl = match ($resource->getCollection()) { - 'functions' => "{$protocol}://{$hostname}/console/project-{$region}-{$projectId}/functions/function-{$resourceId}", - 'sites' => "{$protocol}://{$hostname}/console/project-{$region}-{$projectId}/sites/site-{$resourceId}", - default => throw new \Exception('Invalid resource type') - }; - - $vcs->updateCommitStatus($repositoryName, $providerCommitHash, $owner, $state, $message, $providerTargetUrl, $name); - } - - if (! empty($commentId)) { - $retries = 0; - - while (true) { - $retries++; - - try { - $dbForPlatform->createDocument('vcsCommentLocks', new Document([ - '$id' => $commentId, - ])); - break; - } catch (\Throwable $err) { - if ($retries >= 9) { - throw $err; - } - - \sleep(1); - } - } - - // Wrap in try/finally to ensure lock file gets deleted - try { - $resourceType = match ($resource->getCollection()) { - 'functions' => 'function', - 'sites' => 'site', - default => throw new \Exception('Invalid resource type') - }; - - $rule = $dbForPlatform->findOne('rules', [ - Query::equal('projectInternalId', [$project->getSequence()]), - Query::equal('type', ['deployment']), - Query::equal('deploymentInternalId', [$deployment->getSequence()]), - ]); - - $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; - $previewUrl = ''; - if ($resource->getCollection() === 'sites' && !$rule->isEmpty()) { - $previewUrl = "{$protocol}://" . $rule->getAttribute('domain', ''); - } - - $comment = new Comment($platform); - $comment->parseComment($vcs->getComment($owner, $repositoryName, $commentId)); - $comment->addBuild($project, $resource, $resourceType, $status, $deployment->getId(), ['type' => 'logs'], $previewUrl); - $vcs->updateComment($owner, $repositoryName, $commentId, $comment->generateComment()); - } finally { - $dbForPlatform->deleteDocument('vcsCommentLocks', $commentId); - } - } + GitAction::run($status, $vcs, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment, $dbForPlatform, $platform); } catch (\Throwable $th) { $span = Span::current(); $errorPrefix = $secondaryError ? 'build.error.secondary' : 'build.git_action.error'; diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index 29d1a4842a..76b8880dcc 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -2,13 +2,17 @@ namespace Appwrite\Platform\Modules\Functions\Workers; +use Appwrite\Deployment\Detection; +use Appwrite\Deployment\GitAction; use Appwrite\Event\Event; use Appwrite\Event\Message\Func as FunctionMessage; use Appwrite\Event\Message\Jobs as JobsMessage; use Appwrite\Event\Publisher\Func as FunctionPublisher; +use Appwrite\Event\Publisher\Screenshot as ScreenshotPublisher; use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Webhook; +use Appwrite\Platform\Modules\Compute\Base; use Appwrite\Usage\Build as BuildUsage; use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Response\Model\Deployment; @@ -35,6 +39,10 @@ use Utopia\System\System; * deployments build in parallel. Callbacks are at-least-once, so events are * de-duplicated on the CloudEvent id (inside the lock, so dedup + apply is * atomic and a lock timeout retries cleanly). + * + * Handlers are protected extension points: downstream workers (e.g. cloud) + * override finalize() and wrap parent:: — before it for work that must + * precede 'ready' (edge distribution), after it for post-activation work. */ class Jobs extends Action { @@ -60,12 +68,15 @@ class Jobs extends Action ->inject('queueForEvents') ->inject('queueForWebhooks') ->inject('publisherForFunctions') + ->inject('publisherForScreenshots') ->inject('publisherForUsage') ->inject('usage') ->inject('deviceForBuilds') ->inject('vcsFactory') ->inject('cache') ->inject('locks') + ->inject('platform') + ->inject('plan') ->callback($this->action(...)); } @@ -78,12 +89,15 @@ class Jobs extends Action Event $queueForEvents, Webhook $queueForWebhooks, FunctionPublisher $publisherForFunctions, + ScreenshotPublisher $publisherForScreenshots, UsagePublisher $publisherForUsage, UsageContext $usage, Device $deviceForBuilds, VcsFactory $vcsFactory, Cache $cache, callable $locks, + array $platform, + array $plan, ): void { $event = JobsMessage::fromArray($message->getPayload()); @@ -92,7 +106,7 @@ class Jobs extends Action return; } - $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $deploymentId): void { + $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForScreenshots, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $deploymentId): void { if ($event->id !== '') { $key = 'jobs-event-' . $event->id; if ($cache->load($key, self::DEDUPE_TTL) !== false) { @@ -109,10 +123,10 @@ class Jobs extends Action $statusBefore = $deployment->getAttribute('status'); $deployment = match ($event->event) { - 'orchestrator.job.log' => $this->onLog($dbForProject, $deployment, $event->data), - 'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $deployment, $event->data), - 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds, $vcsFactory, $cache), - 'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $deviceForBuilds, $vcsFactory, $cache), + 'orchestrator.job.log' => $this->onLog($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $vcsFactory, $platform), + 'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan), + 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan), + 'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan), default => $deployment, }; @@ -120,8 +134,8 @@ class Jobs extends Action $queueForRealtime ->setSubscribers(['console']) ->setProject($project) - ->setEvent('functions.[functionId].deployments.[deploymentId].update') - ->setParam('functionId', $deployment->getAttribute('resourceId')) + ->setEvent(self::event($deployment)) + ->setParam(self::resourceParam($deployment), $deployment->getAttribute('resourceId')) ->setParam('deploymentId', $deploymentId) ->setPayload($deployment->getArrayCopy()) ->trigger(); @@ -137,7 +151,7 @@ class Jobs extends Action }, self::LOCK_TIMEOUT); } - private function onLog(Database $dbForProject, Document $deployment, array $data): Document + protected function onLog(Database $dbForProject, Database $dbForPlatform, Document $project, Document $deployment, array $data, VcsFactory $vcsFactory, array $platform): Document { $lines = $data['lines'] ?? []; $chunk = \is_array($lines) ? \implode("\n", $lines) : (string) $lines; @@ -157,19 +171,51 @@ class Jobs extends Action $update['buildStartedAt'] = DateTime::now(); } - return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($update)); + // Guarded like finalize: a concurrent cancel appends its own closing + // log line, which a blind write here would clobber. + $dbForProject->updateDocuments('deployments', new Document($update), [ + Query::equal('$id', [$deployment->getId()]), + Query::notEqual('status', 'canceled'), + ]); + $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); + + if (($update['status'] ?? '') === 'building' && $deployment->getAttribute('status') === 'building') { + $this->gitAction('processing', $deployment, $project, $dbForProject, $dbForPlatform, $vcsFactory, $platform); + } + + return $deployment; } /** - * Record a reported artifact size. Remote-source builds (templates / VCS) - * fetch the source in the sidecar, so Appwrite can't size it the way the - * uploaded-tarball path does — the job stats the downloaded archive and the - * orchestrator reports its size here, which becomes the deployment's - * sourceSize. This arrives before the exit callback, so finalize folds it - * into totalSize. + * Record a reported artifact: 'sourceSize' (remote-source builds) becomes + * the deployment's sourceSize; 'manifest' (site builds) is the output file + * listing for adapter detection, saved as a marker that joins readiness. */ - private function onArtifact(Database $dbForProject, Document $deployment, array $data): Document - { + protected function onArtifact( + Database $dbForProject, + Database $dbForPlatform, + Document $project, + Document $deployment, + array $data, + UsageContext $usage, + UsagePublisher $publisherForUsage, + ScreenshotPublisher $publisherForScreenshots, + Device $deviceForBuilds, + VcsFactory $vcsFactory, + Cache $cache, + array $platform, + array $plan, + ): Document { + if (($data['artifactId'] ?? '') === 'manifest') { + // A failed manifest degrades to an empty listing (detection + // skipped), never a failed build. + $manifest = ($data['status'] ?? '') === 'success' ? ($data['content'] ?? null) : null; + $files = \is_array($manifest) ? (array) ($manifest['files'] ?? []) : []; + $cache->save('jobs-manifest-' . $deployment->getId(), ['files' => \array_values($files)]); + + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan); + } + if (($data['artifactId'] ?? '') !== 'sourceSize' || ($data['status'] ?? '') !== 'success') { return $deployment; } @@ -180,9 +226,6 @@ class Jobs extends Action return $deployment; } - // Recompute totalSize too: this callback can land either side of exit - // (they arrive out of order), and finalize likewise derives totalSize - // from the current sourceSize — so whichever runs last, the two agree. return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ 'sourceSize' => $size, 'totalSize' => $size + (int) $deployment->getAttribute('buildSize', 0), @@ -190,15 +233,13 @@ class Jobs extends Action } /** - * A successful build needs both terminal callbacks before it can ready: - * exit carries the code but fires before post-job artifacts are - * delivered; complete confirms delivery but carries no code. Failures - * short-circuit at exit — no output is needed to fail. The success join - * is order-independent: each of exit(0)/complete leaves a marker, and - * whichever finds the other's marker finalizes (markers live in cache, - * applied under the per-deployment lock). + * Failures short-circuit here — no output is needed to fail. A success + * needs every terminal callback: exit carries the code but fires before + * post-job artifacts, complete confirms delivery, and site builds also + * need the manifest. Each leaves a cache marker and re-attempts the join + * via ready(), so whichever lands last finalizes. */ - private function onExit( + protected function onExit( Database $dbForProject, Database $dbForPlatform, Document $project, @@ -206,80 +247,137 @@ class Jobs extends Action int $exitCode, UsageContext $usage, UsagePublisher $publisherForUsage, + ScreenshotPublisher $publisherForScreenshots, Device $deviceForBuilds, VcsFactory $vcsFactory, Cache $cache, + array $platform, + array $plan, ): Document { if ($exitCode !== 0) { - return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $vcsFactory); + return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform); } $cache->save('jobs-exit-' . $deployment->getId(), true); - if ($cache->load('jobs-complete-' . $deployment->getId(), self::DEDUPE_TTL) !== false) { - return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $deviceForBuilds, $vcsFactory); - } - return $deployment; + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan); } /** - * The delivery half of the success join — see onExit. Emitted by the - * jobs-service once the job's post-job artifacts have run, so on every - * storage strategy the output is already where Appwrite reads it. + * The delivery half of the success join — see onExit. Fires once post-job + * artifacts have run, so the output is already where Appwrite reads it. */ - private function onComplete( + protected function onComplete( Database $dbForProject, Database $dbForPlatform, Document $project, Document $deployment, UsageContext $usage, UsagePublisher $publisherForUsage, + ScreenshotPublisher $publisherForScreenshots, Device $deviceForBuilds, VcsFactory $vcsFactory, Cache $cache, + array $platform, + array $plan, ): Document { - if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'], true)) { - return $deployment; // already finalized (a non-zero exit fails at the exit callback) - } - - if ($cache->load('jobs-exit-' . $deployment->getId(), self::DEDUPE_TTL) !== false) { - return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $deviceForBuilds, $vcsFactory); - } - $cache->save('jobs-complete-' . $deployment->getId(), true); - return $deployment; + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan); } /** - * Finalize a successful build: enforce the size limit against the stored - * output, then mark ready. + * Attempt the success join (see onExit); a no-op until every marker is in. + * Once joined: adapter detection for sites, size limit, then ready. */ - private function ready( + protected function ready( Database $dbForProject, Database $dbForPlatform, Document $project, Document $deployment, UsageContext $usage, UsagePublisher $publisherForUsage, + ScreenshotPublisher $publisherForScreenshots, Device $deviceForBuilds, VcsFactory $vcsFactory, + Cache $cache, + array $platform, + array $plan, ): Document { + if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'], true)) { + return $deployment; // already finalized + } + + $deploymentId = $deployment->getId(); + $isSite = $deployment->getAttribute('resourceType') === 'sites'; + + if ($cache->load('jobs-exit-' . $deploymentId, self::DEDUPE_TTL) === false || $cache->load('jobs-complete-' . $deploymentId, self::DEDUPE_TTL) === false) { + return $deployment; + } + + $manifest = $isSite ? $cache->load('jobs-manifest-' . $deploymentId, self::DEDUPE_TTL) : null; + if ($isSite && $manifest === false) { + return $deployment; + } + + if ($isSite) { + [$deployment, $mismatch] = $this->detect($dbForProject, $deployment, (array) ($manifest['files'] ?? [])); + if ($mismatch !== null) { + return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, $mismatch, $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform); + } + } + $path = (string) $deployment->getAttribute('buildPath', ''); if ($path === '' || ! $deviceForBuilds->exists($path)) { - return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build produced no output artifact.', $usage, $publisherForUsage, $vcsFactory); + return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build produced no output artifact.', $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform); } $size = $deviceForBuilds->getFileSize($path); - $limit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000'); + $limit = isset($plan['buildSize']) + ? (int) $plan['buildSize'] * 1000 * 1000 + : (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000'); if ($limit !== 0 && $size > $limit) { $deviceForBuilds->delete($path); - return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build size should be less than ' . \number_format($limit / (1000 * 1000), 2) . ' MBs.', $usage, $publisherForUsage, $vcsFactory); + return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build size should be less than ' . \number_format($limit / (1000 * 1000), 2) . ' MBs.', $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform); } - return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, true, '', $usage, $publisherForUsage, $vcsFactory, $size); + return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, true, '', $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $size); + } + + /** + * Adapter detection over the site's build-manifest file listing: a first + * successful build pins adapter + fallbackFile on the site and deployment; + * a site pinned to 'ssr' that built static returns the failure message. + * + * @return array{Document, ?string} + */ + protected function detect(Database $dbForProject, Document $deployment, array $files): array + { + $site = empty($files) ? new Document() : $dbForProject->getDocument('sites', $deployment->getAttribute('resourceId')); + if ($site->isEmpty()) { + return [$deployment, null]; + } + + $detection = Detection::rendering($site->getAttribute('framework', ''), $files); + + $adapter = $site->getAttribute('adapter', ''); + if (empty($adapter)) { + $update = [ + 'adapter' => $detection->getName(), + 'fallbackFile' => $detection->getFallbackFile() ?? '', + ]; + $dbForProject->updateDocument('sites', $site->getId(), new Document($update)); + + return [$dbForProject->updateDocument('deployments', $deployment->getId(), new Document($update)), null]; + } + + if ($adapter === 'ssr' && $detection->getName() === 'static') { + return [$deployment, 'Adapter mismatch. Detected: ' . $detection->getName() . ' does not match with the set adapter: ' . $adapter]; + } + + return [$deployment, null]; } /** @@ -287,7 +385,7 @@ class Jobs extends Action * activation, usage and the latestDeployment pointer, mirroring the executor * Builds worker. */ - private function finalize( + protected function finalize( Database $dbForProject, Database $dbForPlatform, Document $project, @@ -296,10 +394,13 @@ class Jobs extends Action string $message, UsageContext $usage, UsagePublisher $publisherForUsage, + ScreenshotPublisher $publisherForScreenshots, VcsFactory $vcsFactory, + array $platform, int $buildSize = 0, ): Document { - $function = $dbForProject->getDocument('functions', $deployment->getAttribute('resourceId')); + $collection = $deployment->getAttribute('resourceType', 'functions'); + $resource = $dbForProject->getDocument($collection, $deployment->getAttribute('resourceId')); $logs = $deployment->getAttribute('buildLogs', ''); $trailer = $success @@ -325,30 +426,38 @@ class Jobs extends Action ]); $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); - if ($applied > 0 && $success && $deployment->getAttribute('activate') === true && ! $function->isEmpty()) { - $this->activate($dbForProject, $dbForPlatform, $project, $function, $deployment); + if ($applied > 0 && $success && $deployment->getAttribute('activate') === true && ! $resource->isEmpty()) { + $this->activate($dbForProject, $dbForPlatform, $project, $resource, $deployment); + } + + if ($applied > 0 && $success && $collection === 'sites' && ! $resource->isEmpty()) { + // Every successful site build, activated or not, repoints the + // branch preview rule and refreshes the console screenshots. + Base::activateBranchPreviewRule($project, $resource, $deployment, $dbForPlatform, $platform['sitesDomain']); + $publisherForScreenshots->enqueue(new \Appwrite\Event\Message\Screenshot( + project: $project, + deploymentId: $deployment->getId(), + )); } // Count the build for usage/billing once it reached a terminal outcome // (mirrors the executor Builds worker); never for a concurrently-canceled // build (the guard above left it 'canceled'). - if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'], true) && ! $function->isEmpty()) { - BuildUsage::publish($usage, $function, $deployment, $project, $publisherForUsage); + if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'], true) && ! $resource->isEmpty()) { + BuildUsage::publish($usage, $resource, $deployment, $project, $publisherForUsage); } - // Keep the function's "latest deployment" pointer + status current, and - // (re)activate its schedule so the scheduler enqueues cron executions — - // both mirror the executor Builds worker on build completion. - if (! $function->isEmpty()) { - $this->updateLatestDeployment($dbForProject, $function); - $this->schedule($dbForProject, $dbForPlatform, $function); + // Keep the resource's "latest deployment" pointer + status current, and + // (re)activate its schedule so the scheduler enqueues cron executions + // (sites have no scheduleId, so schedule() no-ops for them). + if (! $resource->isEmpty()) { + $this->updateLatestDeployment($dbForProject, $resource); + $this->schedule($dbForProject, $dbForPlatform, $resource); } - // Report the terminal outcome as a VCS commit status (jobs-built VCS - // deployments only; best-effort). "pending" at build start is a follow-up. $status = $deployment->getAttribute('status'); - if (\in_array($status, ['ready', 'failed'], true) && ! $function->isEmpty()) { - $this->gitStatus($vcsFactory, $dbForPlatform, $project, $function, $deployment, $status === 'ready' ? 'success' : 'failure'); + if (\in_array($status, ['ready', 'failed'], true) && ! $resource->isEmpty()) { + $this->gitAction($status, $deployment, $project, $dbForProject, $dbForPlatform, $vcsFactory, $platform); } return $deployment; @@ -378,52 +487,46 @@ class Jobs extends Action } /** - * Post a VCS commit status for a VCS deployment (no-op for non-VCS builds - * or silent mode). Best-effort — a failed status update never fails the build. + * Report a build state to the VCS provider for a VCS deployment + * (best-effort; no-op for non-VCS builds). */ - private function gitStatus(VcsFactory $vcsFactory, Database $dbForPlatform, Document $project, Document $function, Document $deployment, string $state): void + protected function gitAction(string $status, Document $deployment, Document $project, Database $dbForProject, Database $dbForPlatform, VcsFactory $vcsFactory, array $platform): void { - $commitHash = $deployment->getAttribute('providerCommitHash', ''); - if ($commitHash === '' || $function->getAttribute('providerSilentMode', false) === true) { + if ($deployment->getAttribute('providerCommitHash', '') === '' && $deployment->getAttribute('providerCommentId', '') === '') { return; } try { - $installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId', '')); - $providerInstallationId = $installation->getAttribute('providerInstallationId', ''); - if ($providerInstallationId === '') { + $resource = $dbForProject->getDocument($deployment->getAttribute('resourceType', 'functions'), $deployment->getAttribute('resourceId')); + $installation = $dbForPlatform->getDocument('installations', $resource->getAttribute('installationId', '')); + if ($resource->isEmpty() || $installation->getAttribute('providerInstallationId', '') === '') { return; } - $vcs = $vcsFactory->fromInstallation($installation); - $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https'; - $hostname = System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN', '')); - $region = $project->getAttribute('region', 'default'); - $targetUrl = "{$protocol}://{$hostname}/console/project-{$region}-{$project->getId()}/functions/function-{$function->getId()}"; - $message = $state === 'success' ? 'Build succeeded.' : 'Build failed.'; - $name = $function->getAttribute('name', '') . ' (' . $project->getAttribute('name', '') . ')'; - - $vcs->updateCommitStatus( - $deployment->getAttribute('providerRepositoryName', ''), - $commitHash, + GitAction::run( + $status, + $vcsFactory->fromInstallation($installation), + $deployment->getAttribute('providerCommitHash', ''), $deployment->getAttribute('providerRepositoryOwner', ''), - $state, - $message, - $targetUrl, - $name, + $deployment->getAttribute('providerRepositoryName', ''), + $project, + $resource, + $deployment, + $dbForPlatform, + $platform, ); } catch (\Throwable) { - // Best-effort; the build outcome stands regardless of the status update. + // Best-effort — never fails the build. } } /** - * Point the function at this deployment (auto-activate). Mirrors the - * essential function activation from the Builds worker. + * Point the resource at this deployment (auto-activate). Mirrors the + * essential activation from the Builds worker. */ - private function activate(Database $dbForProject, Database $dbForPlatform, Document $project, Document $function, Document $deployment): void + protected function activate(Database $dbForProject, Database $dbForPlatform, Document $project, Document $resource, Document $deployment): void { - $function = $dbForProject->updateDocument('functions', $function->getId(), new Document([ + $resource = $dbForProject->updateDocument($resource->getCollection(), $resource->getId(), new Document([ 'live' => true, 'deploymentId' => $deployment->getId(), 'deploymentInternalId' => $deployment->getSequence(), @@ -438,23 +541,23 @@ class Jobs extends Action }, [ Query::equal('projectInternalId', [$project->getSequence()]), Query::equal('type', ['deployment']), - Query::equal('deploymentResourceInternalId', [$function->getSequence()]), - Query::equal('deploymentResourceType', ['function']), + Query::equal('deploymentResourceInternalId', [$resource->getSequence()]), + Query::equal('deploymentResourceType', [$resource->getCollection() === 'sites' ? 'site' : 'function']), Query::equal('trigger', ['manual']), Query::equal('deploymentVcsProviderBranch', ['']), ]); } /** - * Refresh the function's latestDeployment* fields from its newest + * Refresh the resource's latestDeployment* fields from its newest * deployment. Mirrors the Builds worker so the console reflects the current * build status. */ - private function updateLatestDeployment(Database $dbForProject, Document $function): void + protected function updateLatestDeployment(Database $dbForProject, Document $resource): void { $latest = $dbForProject->findOne('deployments', [ - Query::equal('resourceType', ['functions']), - Query::equal('resourceInternalId', [$function->getSequence()]), + Query::equal('resourceType', [$resource->getCollection()]), + Query::equal('resourceInternalId', [$resource->getSequence()]), Query::orderDesc('$createdAt'), ]); @@ -462,7 +565,7 @@ class Jobs extends Action return; } - $dbForProject->updateDocument('functions', $function->getId(), new Document([ + $dbForProject->updateDocument($resource->getCollection(), $resource->getId(), new Document([ 'latestDeploymentId' => $latest->getId(), 'latestDeploymentInternalId' => $latest->getSequence(), 'latestDeploymentCreatedAt' => $latest->getCreatedAt(), @@ -471,20 +574,20 @@ class Jobs extends Action } /** - * (Re)activate the function's schedule document so the scheduler enqueues + * (Re)activate the resource's schedule document so the scheduler enqueues * cron executions. Mirrors the executor Builds worker: a schedule is active - * only when the function has both a cron expression and an active - * deployment. Re-reads the function so it sees a deploymentId just set by + * only when the resource has both a cron expression and an active + * deployment. Re-reads the resource so it sees a deploymentId just set by * activate(). */ - private function schedule(Database $dbForProject, Database $dbForPlatform, Document $function): void + protected function schedule(Database $dbForProject, Database $dbForPlatform, Document $resource): void { - $scheduleId = $function->getAttribute('scheduleId', ''); + $scheduleId = $resource->getAttribute('scheduleId', ''); if ($scheduleId === '') { return; } - $function = $dbForProject->getDocument('functions', $function->getId()); + $resource = $dbForProject->getDocument($resource->getCollection(), $resource->getId()); $schedule = $dbForPlatform->getDocument('schedules', $scheduleId); if ($schedule->isEmpty()) { return; @@ -492,8 +595,8 @@ class Jobs extends Action $dbForPlatform->updateDocument('schedules', $schedule->getId(), new Document([ 'resourceUpdatedAt' => DateTime::now(), - 'schedule' => $function->getAttribute('schedule', ''), - 'active' => ! empty($function->getAttribute('schedule')) && ! empty($function->getAttribute('deploymentId')), + 'schedule' => $resource->getAttribute('schedule', ''), + 'active' => ! empty($resource->getAttribute('schedule')) && ! empty($resource->getAttribute('deploymentId')), ])); } @@ -502,7 +605,7 @@ class Jobs extends Action * update, mirroring the executor Builds worker's fan-out. queueForEvents * only builds the event; realtime is triggered separately in action(). */ - private function dispatchUpdate( + protected function dispatchUpdate( Event $queueForEvents, Webhook $queueForWebhooks, FunctionPublisher $publisherForFunctions, @@ -512,8 +615,8 @@ class Jobs extends Action $model = new Deployment(); $update = $queueForEvents ->setProject($project) - ->setEvent('functions.[functionId].deployments.[deploymentId].update') - ->setParam('functionId', $deployment->getAttribute('resourceId')) + ->setEvent(self::event($deployment)) + ->setParam(self::resourceParam($deployment), $deployment->getAttribute('resourceId')) ->setParam('deploymentId', $deployment->getId()) ->setPayload($deployment->getArrayCopy(\array_keys($model->getRules()))); @@ -530,7 +633,19 @@ class Jobs extends Action )); } - private function truncate(string $logs): string + private static function event(Document $deployment): string + { + $param = self::resourceParam($deployment); + + return "{$deployment->getAttribute('resourceType', 'functions')}.[{$param}].deployments.[deploymentId].update"; + } + + private static function resourceParam(Document $deployment): string + { + return $deployment->getAttribute('resourceType') === 'sites' ? 'siteId' : 'functionId'; + } + + protected function truncate(string $logs): string { $limit = APP_LOG_LENGTH_LIMIT; if (\strlen($logs) <= $limit) { diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php index 40bdbe8e8e..73ef6e6328 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Download; +use Appwrite\Deployment\Token; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; use Appwrite\SDK\AuthType; @@ -9,13 +10,16 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\MethodType; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response; use Utopia\Database\Database; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Http\Adapter\Swoole\Request; use Utopia\Platform\Enum; use Utopia\Platform\Scope\HTTP; use Utopia\Storage\Device; +use Utopia\Validator\Text; use Utopia\Validator\WhiteList; class Get extends Action @@ -34,7 +38,10 @@ class Get extends Action ->setHttpPath('/v1/sites/:siteId/deployments/:deploymentId/download') ->desc('Get deployment download') ->groups(['api', 'sites']) - ->label('scope', 'sites.read') + // Reachable by privileged callers (console/CLI via sites.read) + // and by guests holding a valid presigned token (public); the + // action enforces which one actually applies. + ->label('scope', ['public', 'sites.read']) ->label('usage.resource', 'site/{request.siteId}') ->label('resourceType', RESOURCE_TYPE_SITES) ->label('sdk', new Method( @@ -58,11 +65,14 @@ class Get extends Action ->param('siteId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Site ID.', false, ['dbForProject']) ->param('deploymentId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Deployment ID.', false, ['dbForProject']) ->param('type', 'source', new WhiteList(['source', 'output']), 'Deployment file to download. Can be: "source", "output".', true, enum: new Enum(name: 'DeploymentDownloadType')) + ->param('token', '', new Text(2048), 'Presigned source-download token for accessing this deployment without a session (jobs-service).', true) ->inject('response') ->inject('request') ->inject('dbForProject') ->inject('deviceForSites') ->inject('deviceForBuilds') + ->inject('user') + ->inject('authorization') ->callback($this->action(...)); } @@ -70,18 +80,32 @@ class Get extends Action string $siteId, string $deploymentId, string $type, + string $token, Response $response, Request $request, Database $dbForProject, Device $deviceForSites, - Device $deviceForBuilds + Device $deviceForBuilds, + User $user, + Authorization $authorization, ) { - $site = $dbForProject->getDocument('sites', $siteId); + // Access is granted either to a privileged caller (console/CLI via + // session or API key) or to a valid presigned token bound to this + // deployment + artifact type (used by the jobs-service). + $isPrivileged = $user->isPrivileged($authorization->getRoles()) || $user->isKey($authorization->getRoles()); + if (! $isPrivileged && ! Token::verify($token, $deploymentId, $type)) { + throw new Exception(Exception::USER_UNAUTHORIZED); + } + + // Access has been authorized at the endpoint level (privileged caller or + // presigned token), so read the artifact documents without re-applying + // document-level permissions (a token holder is not an authed user). + $site = $authorization->skip(fn () => $dbForProject->getDocument('sites', $siteId)); if ($site->isEmpty()) { throw new Exception(Exception::SITE_NOT_FOUND); } - $deployment = $dbForProject->getDocument('deployments', $deploymentId); + $deployment = $authorization->skip(fn () => $dbForProject->getDocument('deployments', $deploymentId)); if ($deployment->isEmpty()) { throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php index b0eb8d343c..905764bd92 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php @@ -2,14 +2,14 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Duplicate; +use Appwrite\Deployment\Backend; use Appwrite\Event\Event; -use Appwrite\Event\Message\Build as BuildMessage; -use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response; +use Appwrite\Vcs\Factory as VcsFactory; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; @@ -20,6 +20,7 @@ use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; use Utopia\Storage\Device; use Utopia\System\System; +use Utopia\VCS\Adapter\Git\GitHub; class Create extends Action { @@ -65,8 +66,9 @@ class Create extends Action ->inject('dbForProject') ->inject('dbForPlatform') ->inject('queueForEvents') - ->inject('publisherForBuilds') + ->inject('deployments') ->inject('deviceForSites') + ->inject('vcsFactory') ->inject('authorization') ->inject('platform') ->callback($this->action(...)); @@ -81,8 +83,9 @@ class Create extends Action Database $dbForProject, Database $dbForPlatform, Event $queueForEvents, - BuildPublisher $publisherForBuilds, + Backend $deployments, Device $deviceForSites, + VcsFactory $vcsFactory, Authorization $authorization, array $platform ) { @@ -97,15 +100,27 @@ class Create extends Action throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } + // Remote-source deployments (templates / VCS) on the jobs-service + // backend never store a source tarball — the build sidecar fetches + // it — so a duplicate re-fetches the same source from the + // coordinates persisted on the deployment. $path = $deployment->getAttribute('sourcePath'); - if (empty($path) || !$deviceForSites->exists($path)) { + $hasSource = !empty($path) && $deviceForSites->exists($path); + $installationId = $deployment->getAttribute('installationId', ''); + $owner = $deployment->getAttribute('providerRepositoryOwner', ''); + $repository = $deployment->getAttribute('providerRepositoryName', ''); + + if (!$hasSource && ($owner === '' || $repository === '')) { throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } $deploymentId = ID::unique(); - $destination = $deviceForSites->getPath($deploymentId . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); - $deviceForSites->transfer($path, $destination, $deviceForSites); + $destination = ''; + if ($hasSource) { + $destination = $deviceForSites->getPath($deploymentId . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); + $deviceForSites->transfer($path, $destination, $deviceForSites); + } $commands = []; @@ -116,12 +131,16 @@ class Create extends Action $commands[] = $site->getAttribute('buildCommand', ''); } + // Cloning the source deployment's attributes onto the new one, with + // its own $id and no $sequence, tells the service to create it fresh + // rather than update the deployment being duplicated. A re-fetched + // source starts unsized; its stat artifact reports the size. $deployment->removeAttribute('$sequence'); - - $deployment = $dbForProject->createDocument('deployments', $deployment->setAttributes([ + $deployment->setAttributes([ '$id' => $deploymentId, 'sourcePath' => $destination, - 'totalSize' => $deployment->getAttribute('sourceSize', 0), + 'sourceSize' => $hasSource ? $deployment->getAttribute('sourceSize', 0) : 0, + 'totalSize' => $hasSource ? $deployment->getAttribute('sourceSize', 0) : 0, 'buildCommands' => \implode(' && ', $commands), 'startCommand' => $site->getAttribute('startCommand', ''), 'buildOutput' => $site->getAttribute('outputDirectory', ''), @@ -133,11 +152,44 @@ class Create extends Action 'buildEndedAt' => null, 'buildDuration' => 0, 'buildSize' => 0, - 'status' => 'waiting', 'buildPath' => '', 'buildLogs' => '', - 'type' => $request->getHeaderLine('x-sdk-language') === 'cli' ? 'cli' : 'manual' - ])); + 'type' => $request->getHeaderLine('x-sdk-language') === 'cli' ? 'cli' : 'manual', + // Not inherited: a redeploy always goes live, and the source's own + // flag is unset by deactivateOthers() once anything newer builds. + 'activate' => true, + ]); + + if ($hasSource) { + $deployment = $deployments->createFromUpload($site, $deployment); + } elseif ($installationId !== '') { + $installation = $dbForPlatform->getDocument('installations', $installationId); + if ($installation->isEmpty()) { + throw new Exception(Exception::INSTALLATION_NOT_FOUND); + } + + $vcs = $vcsFactory->fromInstallation($installation); + + $ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch'); + $deployment = $deployments->createFromUrl( + $site, + $deployment, + $vcs->getRepositoryPresignedUrl($owner, $repository, $ref), + $deployment->getAttribute('providerRootDirectory', ''), + ); + } else { + // Public template repo: providerBranch holds the resolved ref, + // fetched commit-style since a branch-type clone breaks on tags. + $deployment = $deployments->createFromRef( + $site, + $deployment, + $owner, + $repository, + GitHub::CLONE_TYPE_COMMIT, + $deployment->getAttribute('providerBranch', ''), + $deployment->getAttribute('providerRootDirectory', ''), + ); + } // Preview deployments for sites $sitesDomain = $platform['sitesDomain']; @@ -155,8 +207,8 @@ class Create extends Action 'domain' => $domain, 'type' => 'deployment', 'trigger' => 'deployment', - 'deploymentId' => $deployment->isEmpty() ? '' : $deployment->getId(), - 'deploymentInternalId' => $deployment->isEmpty() ? '' : $deployment->getSequence(), + 'deploymentId' => $deployment->getId(), + 'deploymentInternalId' => $deployment->getSequence(), 'deploymentResourceType' => 'site', 'deploymentResourceId' => $site->getId(), 'deploymentResourceInternalId' => $site->getSequence(), @@ -167,14 +219,6 @@ class Create extends Action ])) ); - $publisherForBuilds->enqueue(new BuildMessage( - project: $project, - resource: $site, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - platform: $platform, - )); - $queueForEvents ->setParam('siteId', $site->getId()) ->setParam('deploymentId', $deployment->getId()); diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php index c698a2d5f6..0d562258f5 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php @@ -2,18 +2,20 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Status; +use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Backend\Orchestrator; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response; -use Executor\Executor; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception\Transaction as TransactionException; use Utopia\Database\Validator\UID; +use Utopia\Lock\Exception\Contention; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; @@ -56,9 +58,9 @@ class Update extends Action ->param('deploymentId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Deployment ID.', false, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('project') ->inject('queueForEvents') - ->inject('executor') + ->inject('deployments') + ->inject('locks') ->callback($this->action(...)); } @@ -67,9 +69,9 @@ class Update extends Action string $deploymentId, Response $response, Database $dbForProject, - Document $project, Event $queueForEvents, - Executor $executor + Backend $deployments, + callable $locks ) { $site = $dbForProject->getDocument('sites', $siteId); @@ -91,36 +93,41 @@ class Update extends Action $endTime = new \DateTime('now'); $duration = $endTime->getTimestamp() - $startTime->getTimestamp(); + // Write under the Jobs worker's per-deployment lock: its handlers + // read-modify-write buildLogs, and an unserialized cancel write here + // loses the closing log line to an in-flight append. + $cancel = function () use ($dbForProject, $deployment, $duration, $deployments) { + try { + return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); + } catch (TransactionException) { + $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); + + if ($deployment->isEmpty()) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + + if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'])) { + throw new Exception(Exception::BUILD_ALREADY_COMPLETED); + } + + if ($deployment->getAttribute('status') !== 'canceled') { + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); + } + + return $deployment; + } + }; + try { - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'buildEndedAt' => DateTime::now(), - 'buildDuration' => $duration, - 'status' => 'canceled' - ])); - } catch (TransactionException) { - $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); - - if ($deployment->isEmpty()) { - throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); - } - - if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'])) { - throw new Exception(Exception::BUILD_ALREADY_COMPLETED); - } - - if ($deployment->getAttribute('status') !== 'canceled') { - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'buildEndedAt' => DateTime::now(), - 'buildDuration' => $duration, - 'status' => 'canceled' - ])); - } + $deployment = $locks('jobs-deployment:' . $deploymentId, 30, $cancel, 10.0); + } catch (Contention) { + $deployment = $cancel(); } + // Best-effort cleanup — the deployment is already marked 'canceled'. try { - $executor->deleteRuntime($project->getId(), $deploymentId . "-build"); + $deployments->cancel($deploymentId); } catch (\Throwable) { - // Best-effort cleanup — deployment status is already 'canceled' } $queueForEvents @@ -129,4 +136,27 @@ class Update extends Action $response->dynamic($deployment, Response::MODEL_DEPLOYMENT); } + + /** + * The sparse update marking a build canceled. Jobs-backed builds have no + * cancel worker to write the closing log line the executor's Builds worker + * adds, so it is appended here; executor deployments get it from their worker. + * + * @return array + */ + private function cancel(Document $deployment, int $duration, bool $appendLog): array + { + $update = [ + 'buildEndedAt' => DateTime::now(), + 'buildDuration' => $duration, + 'status' => 'canceled', + ]; + + if ($appendLog) { + $logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n"; + $update['buildLogs'] = \substr($logs, -APP_LOG_LENGTH_LIMIT); + } + + return $update; + } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php index 772fd88720..93cc7270b3 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php @@ -2,8 +2,8 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Template; +use Appwrite\Deployment\Backend; use Appwrite\Event\Event; -use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; @@ -84,6 +84,7 @@ class Create extends Base ->inject('publisherForBuilds') ->inject('vcsFactory') ->inject('authorization') + ->inject('deployments') ->inject('platform') ->callback($this->action(...)); } @@ -105,6 +106,7 @@ class Create extends Base BuildPublisher $publisherForBuilds, VcsFactory $vcsFactory, Authorization $authorization, + Backend $deployments, array $platform ) { $site = $dbForProject->getDocument('sites', $siteId); @@ -139,6 +141,7 @@ class Create extends Base vcs: $vcsFactory->fromInstallation($installation), activate: $activate, authorization: $authorization, + deployments: $deployments, platform: $platform ); @@ -161,6 +164,18 @@ class Create extends Base $commands[] = $site->getAttribute('buildCommand', ''); } + // Templates can pin a version range (e.g. "0.3.*"); codeload only takes + // a concrete ref, so resolve it to the highest matching tag via GitHub. + $ref = $reference; + if ($type === Git::CLONE_TYPE_TAG && \str_contains($reference, '*')) { + try { + $tags = $vcsFactory->fromProvider('github')->listTags($owner, $repository, $reference); + $ref = \end($tags) ?: $reference; + } catch (\Throwable) { + // Fall back to the raw reference; the build surfaces a bad ref. + } + } + $deploymentId = ID::unique(); $deployment = $dbForProject->createDocument('deployments', new Document([ '$id' => $deploymentId, @@ -179,7 +194,9 @@ class Create extends Base 'providerRepositoryOwner' => $owner, 'providerRepositoryUrl' => $repositoryUrl, 'providerBranchUrl' => $branchUrl, - 'providerBranch' => $type == Git::CLONE_TYPE_BRANCH ? $reference : '', + // The resolved concrete ref, so a duplicate can re-fetch the source. + 'providerBranch' => $ref, + 'providerRootDirectory' => $rootDirectory, 'adapter' => $site->getAttribute('adapter', ''), 'fallbackFile' => $site->getAttribute('fallbackFile', ''), 'type' => 'vcs', @@ -215,14 +232,18 @@ class Create extends Base $this->updateEmptyManualRule($project, $site, $deployment, $dbForPlatform, $authorization); - $publisherForBuilds->enqueue(new BuildMessage( - project: $project, - resource: $site, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - template: $template, - platform: $platform, - )); + // Public template: pull the source straight from GitHub's public repo + // (codeload tarball on the jobs backend, a plain git clone on the + // executor); unarchive/checkout strips down to the rootDirectory. + $deployment = $deployments->createFromRef( + $site, + $deployment, + $owner, + $repository, + $type, + $ref, + $rootDirectory, + ); $queueForEvents ->setParam('siteId', $site->getId()) diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Vcs/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Vcs/Create.php index 6e79b8fa81..4e10ed7822 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Vcs/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Vcs/Create.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments\Vcs; +use Appwrite\Deployment\Backend; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -76,6 +77,7 @@ class Create extends Base ->inject('publisherForBuilds') ->inject('vcsFactory') ->inject('authorization') + ->inject('deployments') ->inject('platform') ->callback($this->action(...)); } @@ -94,6 +96,7 @@ class Create extends Base BuildPublisher $publisherForBuilds, VcsFactory $vcsFactory, Authorization $authorization, + Backend $deployments, array $platform ) { $site = $dbForProject->getDocument('sites', $siteId); @@ -122,6 +125,7 @@ class Create extends Base vcs: $vcsFactory->fromInstallation($installation), activate: $activate, authorization: $authorization, + deployments: $deployments, reference: $reference, referenceType: $type, platform: $platform diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php index 6ae47e5a6e..216c861a82 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Sites; +use Appwrite\Deployment\Backend; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -19,6 +20,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Http\Adapter\Swoole\Request; use Utopia\Platform\Action; @@ -113,6 +115,8 @@ class Update extends Base ->inject('vcsFactory') ->inject('repositoryWebhooks') ->inject('executor') + ->inject('authorization') + ->inject('deployments') ->inject('platform') ->callback($this->action(...)); } @@ -151,6 +155,8 @@ class Update extends Base VcsFactory $vcsFactory, RepositoryWebhooks $repositoryWebhooks, Executor $executor, + Authorization $authorization, + Backend $deployments, array $platform ) { if (!empty($adapter)) { @@ -331,7 +337,7 @@ class Update extends Base // Redeploy logic if (!$isConnected && !empty($providerRepositoryId)) { - $this->redeployVcsFunction($request, $site, $project, $installation, $dbForProject, $publisherForBuilds, new Document(), $vcsFactory->fromInstallation($installation), true, $platform); + $this->redeployVcsSite($request, $site, $project, $installation, $dbForProject, $dbForPlatform, $publisherForBuilds, new Document(), $vcsFactory->fromInstallation($installation), true, $authorization, $deployments, $platform); } $queueForEvents->setParam('siteId', $site->getId()); diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Authorize/External/Update.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Authorize/External/Update.php index df629138e4..7d30ac8d45 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Authorize/External/Update.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Authorize/External/Update.php @@ -2,8 +2,6 @@ namespace Appwrite\Platform\Modules\VCS\Http\GitHub\Authorize\External; -use Appwrite\Deployment\Backend; -use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; use Appwrite\Platform\Modules\VCS\Http\GitHub\Deployment; @@ -60,8 +58,7 @@ class Update extends Action ->inject('dbForPlatform') ->inject('authorization') ->inject('getProjectDB') - ->inject('publisherForBuilds') - ->inject('deployments') + ->inject('deploymentsFactory') ->inject('platform') ->callback($this->action(...)); } @@ -76,8 +73,7 @@ class Update extends Action Database $dbForPlatform, Authorization $authorization, callable $getProjectDB, - BuildPublisher $publisherForBuilds, - Backend $deployments, + callable $deploymentsFactory, array $platform ) { $installation = $dbForPlatform->getDocument('installations', $installationId); @@ -137,7 +133,7 @@ class Update extends Action ...array_filter(array_column($prFiles, 'previous_filename')) ]; - $this->createGitDeployments($vcs, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, true, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments); + $this->createGitDeployments($vcs, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, true, $dbForPlatform, $authorization, $getProjectDB, $platform, $deploymentsFactory); $response->noContent(); } diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php index 1f38eee712..a4194292b9 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php @@ -2,9 +2,6 @@ namespace Appwrite\Platform\Modules\VCS\Http\GitHub; -use Appwrite\Deployment\Backend; -use Appwrite\Event\Message\Build as BuildMessage; -use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\Filter\BranchDomain as BranchDomainFilter; use Appwrite\Vcs\Comment; @@ -47,10 +44,9 @@ trait Deployment bool $external, Database $dbForPlatform, Authorization $authorization, - BuildPublisher $publisherForBuilds, callable $getProjectDB, array $platform, - ?Backend $deployments = null, + callable $deploymentsFactory, ) { $errors = []; $provider = $vcs->getName(); @@ -393,37 +389,15 @@ trait Deployment 'activate' => $activate, ]); - // Build function deployments through $deployments (executor - // or jobs-service, decided by _APP_BUILDS_BACKEND) when the - // caller opts in. Sites always stay on the executor. - if ($deployments !== null && $resourceCollection === 'functions') { - $deployment = $authorization->skip(fn () => $deployments - ->forProject($dbForProject, $project) - ->createFromUrl( - $resource, - $deployment, - $vcs->getRepositoryPresignedUrl($providerRepositoryOwner, $providerRepositoryName, $providerCommitHash), - $resource->getAttribute('providerRootDirectory', ''), - )); - } else { - $deployment = $authorization->skip(fn () => $dbForProject->createDocument('deployments', new Document([ - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - ...$deployment->getArrayCopy(), - 'status' => 'waiting', - ]))); - - $publisherForBuilds->enqueue(new BuildMessage( - project: $project, - resource: $resource, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - platform: $platform, + // The Backend is built per repository: a webhook fans out to + // many tenant projects, each with its own database. + $deployment = $authorization->skip(fn () => $deploymentsFactory($dbForProject, $project) + ->createFromUrl( + $resource, + $deployment, + $vcs->getRepositoryPresignedUrl($providerRepositoryOwner, $providerRepositoryName, $providerCommitHash), + $resource->getAttribute('providerRootDirectory', ''), )); - } if ($resource->getCollection() === 'sites') { $projectId = $project->getId(); diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Events/Create.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Events/Create.php index 7bd23d285d..7010584e84 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Events/Create.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Events/Create.php @@ -2,8 +2,6 @@ namespace Appwrite\Platform\Modules\VCS\Http\GitHub\Events; -use Appwrite\Deployment\Backend; -use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; use Appwrite\Platform\Modules\VCS\Http\GitHub\Deployment; @@ -44,8 +42,7 @@ class Create extends Action ->inject('dbForPlatform') ->inject('authorization') ->inject('getProjectDB') - ->inject('publisherForBuilds') - ->inject('deployments') + ->inject('deploymentsFactory') ->inject('platform') ->callback($this->action(...)); } @@ -58,8 +55,7 @@ class Create extends Action Database $dbForPlatform, Authorization $authorization, callable $getProjectDB, - BuildPublisher $publisherForBuilds, - Backend $deployments, + callable $deploymentsFactory, array $platform ) { $this->preprocessEvent($request); @@ -84,8 +80,8 @@ class Create extends Action match ($event) { GitHub::EVENT_INSTALLATION => $this->handleInstallationEvent($parsedPayload, $dbForPlatform, $authorization, $getProjectDB), - GitHub::EVENT_PUSH => $this->handlePushEvent($parsedPayload, $vcsFactory, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments), - GitHub::EVENT_PULL_REQUEST => $this->handlePullRequestEvent($parsedPayload, $vcsFactory, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments), + GitHub::EVENT_PUSH => $this->handlePushEvent($parsedPayload, $vcsFactory, $dbForPlatform, $authorization, $getProjectDB, $platform, $deploymentsFactory), + GitHub::EVENT_PULL_REQUEST => $this->handlePullRequestEvent($parsedPayload, $vcsFactory, $dbForPlatform, $authorization, $getProjectDB, $platform, $deploymentsFactory), default => null, }; @@ -188,10 +184,9 @@ class Create extends Action VcsFactory $vcsFactory, Database $dbForPlatform, Authorization $authorization, - BuildPublisher $publisherForBuilds, callable $getProjectDB, array $platform, - ?Backend $deployments = null, + callable $deploymentsFactory, ) { $providerBranchDeleted = $parsedPayload["branchDeleted"] ?? false; $providerBranch = $parsedPayload["branch"] ?? ''; @@ -227,7 +222,7 @@ class Create extends Action // Create new deployment only on push (not committed by us) and not when branch is deleted if ($providerCommitAuthorEmail !== APP_VCS_GITHUB_EMAIL && !$providerBranchDeleted) { $providerAffectedFiles = $parsedPayload['affectedFiles'] ?? []; - $this->createGitDeployments($vcs, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthorName, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', $providerAffectedFiles, false, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments); + $this->createGitDeployments($vcs, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthorName, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', $providerAffectedFiles, false, $dbForPlatform, $authorization, $getProjectDB, $platform, $deploymentsFactory); } } @@ -236,10 +231,9 @@ class Create extends Action VcsFactory $vcsFactory, Database $dbForPlatform, Authorization $authorization, - BuildPublisher $publisherForBuilds, callable $getProjectDB, array $platform, - ?Backend $deployments = null, + callable $deploymentsFactory, ) { $action = $parsedPayload["action"] ?? ''; @@ -293,7 +287,7 @@ class Create extends Action Query::orderDesc('$createdAt') ])); - $this->createGitDeployments($vcs, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, $external, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments); + $this->createGitDeployments($vcs, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, $external, $dbForPlatform, $authorization, $getProjectDB, $platform, $deploymentsFactory); } elseif ($action == "closed") { // Allowed external contributions cleanup diff --git a/src/Appwrite/Platform/Modules/VCS/Http/Gitea/Events/Create.php b/src/Appwrite/Platform/Modules/VCS/Http/Gitea/Events/Create.php index 10ec1bbcb6..04316862ee 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/Gitea/Events/Create.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/Gitea/Events/Create.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\VCS\Http\Gitea\Events; use Appwrite\Auth\OAuth2\Gitea as OAuth2Gitea; -use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; use Appwrite\Platform\Modules\VCS\Http\GitHub\Deployment; @@ -47,7 +46,7 @@ class Create extends Action ->inject('dbForPlatform') ->inject('authorization') ->inject('getProjectDB') - ->inject('publisherForBuilds') + ->inject('deploymentsFactory') ->inject('platform') ->callback($this->action(...)); } @@ -61,7 +60,7 @@ class Create extends Action Database $dbForPlatform, Authorization $authorization, callable $getProjectDB, - BuildPublisher $publisherForBuilds, + callable $deploymentsFactory, array $platform ) { $vcs = $vcsFactory->fromProvider('gitea'); @@ -83,8 +82,8 @@ class Create extends Action $parsedPayload = $vcs->getEvent($event, $payload); match ($event) { - 'push' => $this->handlePushEvent($parsedPayload, $vcsFactory, $installationTokens, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform), - 'pull_request' => $this->handlePullRequestEvent($parsedPayload, $vcsFactory, $installationTokens, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform), + 'push' => $this->handlePushEvent($parsedPayload, $vcsFactory, $installationTokens, $dbForPlatform, $authorization, $getProjectDB, $platform, $deploymentsFactory), + 'pull_request' => $this->handlePullRequestEvent($parsedPayload, $vcsFactory, $installationTokens, $dbForPlatform, $authorization, $getProjectDB, $platform, $deploymentsFactory), default => null, }; @@ -144,9 +143,9 @@ class Create extends Action InstallationTokens $installationTokens, Database $dbForPlatform, Authorization $authorization, - BuildPublisher $publisherForBuilds, callable $getProjectDB, array $platform, + callable $deploymentsFactory, ) { $providerBranchDeleted = $parsedPayload['branchDeleted'] ?? false; $providerBranch = $parsedPayload['branch'] ?? ''; @@ -187,7 +186,7 @@ class Create extends Action $providerInstallationId = $repository->getAttribute('installationId', ''); - $this->createGitDeployments($adapter, $providerInstallationId, [$repository], $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthorName, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', $providerAffectedFiles, false, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform); + $this->createGitDeployments($adapter, $providerInstallationId, [$repository], $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthorName, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', $providerAffectedFiles, false, $dbForPlatform, $authorization, $getProjectDB, $platform, $deploymentsFactory); } if (!empty($errors)) { @@ -201,9 +200,9 @@ class Create extends Action InstallationTokens $installationTokens, Database $dbForPlatform, Authorization $authorization, - BuildPublisher $publisherForBuilds, callable $getProjectDB, array $platform, + callable $deploymentsFactory, ) { $action = $parsedPayload['action'] ?? ''; @@ -260,7 +259,7 @@ class Create extends Action ...array_filter(array_column($prFiles, 'previous_filename')), ]; - $this->createGitDeployments($adapter, $providerInstallationId, [$repository], $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, $external, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform); + $this->createGitDeployments($adapter, $providerInstallationId, [$repository], $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, $external, $dbForPlatform, $authorization, $getProjectDB, $platform, $deploymentsFactory); } if (!empty($errors)) { diff --git a/tests/e2e/Services/Project/VariablesCustomServerTest.php b/tests/e2e/Services/Project/VariablesCustomServerTest.php index 6c710fecf2..0bd2de59ae 100644 --- a/tests/e2e/Services/Project/VariablesCustomServerTest.php +++ b/tests/e2e/Services/Project/VariablesCustomServerTest.php @@ -94,14 +94,18 @@ final class VariablesCustomServerTest extends Scope $this->assertSame($deploymentId, $function['body']['deploymentId'] ?? ''); }, 120000, 500); - // 5. Verify the project variable was available during build - $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/deployments/' . $deploymentId, [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - 'x-appwrite-key' => $apiKey, - ]); - $this->assertSame(200, $deployment['headers']['status-code']); - $this->assertStringContainsString('Project Variable Value', (string) $deployment['body']['buildLogs']); + // 5. Verify the project variable was available during build. Log + // callbacks stream asynchronously, so the echoed line can land in + // buildLogs shortly after the deployment turns ready. + $this->assertEventually(function () use ($projectId, $apiKey, $functionId, $deploymentId) { + $deployment = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/deployments/' . $deploymentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $apiKey, + ]); + $this->assertSame(200, $deployment['headers']['status-code']); + $this->assertStringContainsString('Project Variable Value', (string) $deployment['body']['buildLogs']); + }, 30000, 500); // 6. Execute the function and verify the project variable is in runtime output $execution = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/executions', array_merge([ diff --git a/tests/e2e/Services/Sites/SitesCustomServerTest.php b/tests/e2e/Services/Sites/SitesCustomServerTest.php index 10fe086f37..c4f4144b2a 100644 --- a/tests/e2e/Services/Sites/SitesCustomServerTest.php +++ b/tests/e2e/Services/Sites/SitesCustomServerTest.php @@ -1308,6 +1308,11 @@ final class SitesCustomServerTest extends Scope $this->assertEquals('ready', $deployment['body']['status']); }, 120000, 500); + // Adapter detection pinned the site on its first successful build. + $site = $this->getSite($siteId); + $this->assertEquals('static', $site['body']['adapter']); + $this->assertEquals('main.html', $site['body']['fallbackFile']); + $deployment = $this->createDeployment($siteId, [ 'code' => $this->packageSite('static-single-file'), 'activate' => 'false' @@ -1336,6 +1341,76 @@ final class SitesCustomServerTest extends Scope } #[Retry(count: 3)] + public function testAdapterDetectionSSR(): void + { + $siteId = $this->setupSite([ + 'siteId' => ID::unique(), + 'name' => 'SSR detection site', + 'framework' => 'astro', + 'buildRuntime' => 'node-22', + 'outputDirectory' => './dist', + 'buildCommand' => 'npm run build', + 'installCommand' => 'npm ci', + 'fallbackFile' => '', + ]); + + $this->setupDeployment($siteId, [ + 'code' => $this->packageSite('astro'), + 'activate' => 'true', + ]); + + $site = $this->getSite($siteId); + $this->assertEquals('ssr', $site['body']['adapter']); + $this->assertEmpty($site['body']['fallbackFile']); + + // A second build of the same site reuses the package-manager cache. + $deployment = $this->createDeployment($siteId, [ + 'code' => $this->packageSite('astro'), + 'activate' => 'false', + ]); + $this->assertEquals(202, $deployment['headers']['status-code']); + $deploymentId = $deployment['body']['$id']; + $this->waitDeploymentReady($siteId, $deploymentId); + + $deployment = $this->getDeployment($siteId, $deploymentId); + $this->assertStringContainsString('Build cache hit.', (string) $deployment['body']['buildLogs']); + + $this->cleanupSite($siteId); + } + + public function testAdapterMismatchFailsBuild(): void + { + $siteId = $this->setupSite([ + 'siteId' => ID::unique(), + 'name' => 'Adapter mismatch site', + 'framework' => 'astro', + 'adapter' => 'ssr', + 'buildRuntime' => 'node-22', + 'outputDirectory' => './dist', + 'buildCommand' => 'npm run build', + 'installCommand' => 'npm ci', + 'fallbackFile' => '', + ]); + + $deployment = $this->createDeployment($siteId, [ + 'code' => $this->packageSite('astro-static'), + 'activate' => 'true', + ]); + $this->assertEquals(202, $deployment['headers']['status-code']); + $deploymentId = $deployment['body']['$id']; + + $this->assertEventually(function () use ($siteId, $deploymentId) { + $deployment = $this->getDeployment($siteId, $deploymentId); + + $this->assertEquals('failed', $deployment['body']['status']); + }, 120000, 500); + + $deployment = $this->getDeployment($siteId, $deploymentId); + $this->assertStringContainsString('Adapter mismatch', (string) $deployment['body']['buildLogs']); + + $this->cleanupSite($siteId); + } + public function testCancelDeploymentBuild(): void { $siteId = $this->setupSite([ diff --git a/tests/unit/Deployment/BackendTest.php b/tests/unit/Deployment/BackendTest.php new file mode 100644 index 0000000000..db024a4ef0 --- /dev/null +++ b/tests/unit/Deployment/BackendTest.php @@ -0,0 +1,44 @@ + [ + 'envCommand' => 'cp .env.example .env', + 'bundleCommand' => 'npm run bundle', + ], + ]); + + $resource = new Document([ + '$collection' => 'sites', + 'framework' => 'astro', + ]); + $deployment = new Document([ + 'buildCommands' => 'npm run build', + ]); + + $this->assertSame( + 'cp .env.example .env && npm run build && npm run bundle', + Backend::command($resource, $deployment) + ); + } + + public function testFunctionCommandIsDeploymentBuildCommands(): void + { + $resource = new Document(['$collection' => 'functions']); + $deployment = new Document(['buildCommands' => 'npm install']); + + $this->assertSame('npm install', Backend::command($resource, $deployment)); + } +} diff --git a/tests/unit/Deployment/DetectionTest.php b/tests/unit/Deployment/DetectionTest.php new file mode 100644 index 0000000000..42ec656830 --- /dev/null +++ b/tests/unit/Deployment/DetectionTest.php @@ -0,0 +1,27 @@ +assertSame('static', $detection->getName()); + $this->assertSame('main.html', $detection->getFallbackFile()); + } + + public function testRenderingFindsAstroSSR(): void + { + $detection = Detection::rendering('astro', ['./client/index.html', './server/entry.mjs']); + + $this->assertSame('ssr', $detection->getName()); + $this->assertNull($detection->getFallbackFile()); + } +} diff --git a/tests/unit/Functions/BuildsTest.php b/tests/unit/Functions/BuildsTest.php deleted file mode 100644 index a14a8137b3..0000000000 --- a/tests/unit/Functions/BuildsTest.php +++ /dev/null @@ -1,171 +0,0 @@ -builds = new Builds(); - } - - public function testSiteCommandIncludesFrameworkAndDeploymentCommands(): void - { - Config::setParam('frameworks', [ - 'astro' => [ - 'envCommand' => 'cp .env.example .env', - 'bundleCommand' => 'npm run bundle', - ], - ]); - - $resource = new Document([ - '$collection' => 'sites', - 'framework' => 'astro', - ]); - $deployment = new Document([ - 'buildCommands' => 'npm run build', - ]); - - $this->assertSame( - 'cp .env.example .env && npm run build && npm run bundle', - $this->callBuilds('getCommand', $resource, $deployment) - ); - } - - public function testSiteBuildCommandAppendsDetectionFileListing(): void - { - $command = $this->callBuilds('prepareSiteBuildCommand', 'npm run build', './dist folder', 'astro'); - - $this->assertStringStartsWith('npm run build && echo "{APPWRITE_DETECTION_SEPARATOR_START}"', $command); - $this->assertStringContainsString('cd /usr/local/build && cd \'./dist folder\'', (string) $command); - $this->assertStringContainsString('( [ -e \'server/entry.mjs\' ] && echo \'server/entry.mjs\' || true )', (string) $command); - $this->assertStringContainsString('find . -name \'node_modules\' -prune -o -type f -name \'*.html\' -print | head -n 2', (string) $command); - $this->assertStringEndsWith('echo "{APPWRITE_DETECTION_SEPARATOR_END}"', $command); - } - - public function testSiteBuildCommandCanBeOnlyDetectionFileListing(): void - { - $command = $this->callBuilds('prepareSiteBuildCommand', '', '', 'sveltekit'); - - $this->assertStringStartsWith('echo "{APPWRITE_DETECTION_SEPARATOR_START}"', $command); - $this->assertStringContainsString('cd /usr/local/build && ( [ -e \'handler.js\' ] && echo \'handler.js\' || true )', (string) $command); - $this->assertStringContainsString('find . -name \'node_modules\' -prune -o -type f -name \'*.html\' -print | head -n 2', (string) $command); - } - - public function testSplitSiteDetectionLogsRemovesDetectionBlock(): void - { - $result = $this->callBuilds( - 'splitSiteDetectionLogs', - "before\n{APPWRITE_DETECTION_SEPARATOR_START}\n./index.html\n./server/entry.mjs\n{APPWRITE_DETECTION_SEPARATOR_END}\nafter\n" - ); - - $this->assertSame("before\n\nafter\n", $result['logs']); - $this->assertSame("\n./index.html\n./server/entry.mjs\n", $result['detectionLogs']); - } - - public function testDetectSiteRenderingFindsStaticFallbackFile(): void - { - $detection = $this->callBuilds('detectSiteRendering', 'other', "./main.html\n"); - - $this->assertSame('static', $detection->getName()); - $this->assertSame('main.html', $detection->getFallbackFile()); - } - - public function testDetectSiteRenderingFindsAstroSSR(): void - { - $detection = $this->callBuilds('detectSiteRendering', 'astro', "./client/index.html\n./server/entry.mjs\n"); - - $this->assertSame('ssr', $detection->getName()); - $this->assertNull($detection->getFallbackFile()); - } - - public function testDisconnectVcsClearsAllFieldsAndDeletesRepository(): void - { - $resource = new Document([ - '$id' => 'func123', - '$collection' => 'functions', - 'repositoryId' => 'repo456', - 'installationId' => 'install789', - 'installationInternalId' => 'install789internal', - 'providerRepositoryId' => 'gh123', - 'providerBranch' => 'main', - 'providerRootDirectory' => 'functions/my-func', - 'providerSilentMode' => true, - 'repositoryInternalId' => 'repo456internal', - ]); - - $dbForProject = $this->createMock(Database::class); - $dbForPlatform = $this->createMock(Database::class); - - $dbForPlatform->expects($this->once()) - ->method('deleteDocument') - ->with('repositories', 'repo456'); - - $dbForProject->expects($this->once()) - ->method('updateDocument') - ->with('functions', 'func123', $this->callback(function (Document $doc) { - $this->assertSame('', $doc->getAttribute('installationId')); - $this->assertSame('', $doc->getAttribute('installationInternalId')); - $this->assertSame('', $doc->getAttribute('providerRepositoryId')); - $this->assertSame('', $doc->getAttribute('providerBranch')); - $this->assertSame('', $doc->getAttribute('providerRootDirectory')); - $this->assertFalse($doc->getAttribute('providerSilentMode')); - $this->assertSame('', $doc->getAttribute('repositoryId')); - $this->assertSame('', $doc->getAttribute('repositoryInternalId')); - return true; - })); - - $this->callBuilds('disconnectVcs', $resource, $dbForProject, $dbForPlatform); - } - - public function testDisconnectVcsSkipsRepositoryDeletionWhenRepositoryIdEmpty(): void - { - $resource = new Document([ - '$id' => 'func123', - '$collection' => 'functions', - 'repositoryId' => '', - ]); - - $dbForProject = $this->createMock(Database::class); - $dbForPlatform = $this->createMock(Database::class); - - $dbForPlatform->expects($this->never())->method('deleteDocument'); - $dbForProject->expects($this->once())->method('updateDocument'); - - $this->callBuilds('disconnectVcs', $resource, $dbForProject, $dbForPlatform); - } - - public function testTruncateBuildLogsKeepsShortLogsUnchanged(): void - { - $logs = "Build finished.\n"; - - $this->assertSame($logs, $this->callBuilds('truncateBuildLogs', $logs)); - } - - public function testTruncateBuildLogsTrimsOversizedLogsToLimitKeepingTail(): void - { - $oversized = \str_repeat('a', APP_LOG_LENGTH_LIMIT + 5000) . 'TAIL_MARKER'; - - $result = (string) $this->callBuilds('truncateBuildLogs', $oversized); - - $this->assertLessThanOrEqual(APP_LOG_LENGTH_LIMIT, \strlen($result)); - $this->assertStringContainsString('[WARNING] Logs truncated.', $result); - $this->assertStringEndsWith('TAIL_MARKER', $result); - } - - private function callBuilds(string $method, mixed ...$arguments): mixed - { - return (new ReflectionMethod($this->builds, $method))->invoke($this->builds, ...$arguments); - } -} diff --git a/tests/unit/Functions/JobsTest.php b/tests/unit/Functions/JobsTest.php deleted file mode 100644 index 6f66bcacba..0000000000 --- a/tests/unit/Functions/JobsTest.php +++ /dev/null @@ -1,66 +0,0 @@ -jobs = new Jobs(); - } - - public function testDurationRoundsSubSecondBuildUpToOneSecond(): void - { - $deployment = new Document([ - 'buildStartedAt' => (new \DateTimeImmutable('-500 milliseconds'))->format('Y-m-d H:i:s.v'), - ]); - - $this->assertSame(1, $this->callJobs('duration', $deployment)); - } - - public function testDurationFallsBackToCreationTimeWhenStartMissing(): void - { - $deployment = new Document([ - '$createdAt' => (new \DateTimeImmutable('-4500 milliseconds'))->format('Y-m-d H:i:s.v'), - ]); - - $this->assertSame(5, $this->callJobs('duration', $deployment)); - } - - public function testDurationIsZeroWithoutAnyTimestamp(): void - { - $this->assertSame(0, $this->callJobs('duration', new Document([]))); - } - - public function testDurationClampsFutureStartToZero(): void - { - $deployment = new Document([ - 'buildStartedAt' => (new \DateTimeImmutable('+10 seconds'))->format('Y-m-d H:i:s.v'), - ]); - - $this->assertSame(0, $this->callJobs('duration', $deployment)); - } - - public function testDurationIsZeroForUnparseableStart(): void - { - $deployment = new Document([ - 'buildStartedAt' => 'not-a-date', - ]); - - $this->assertSame(0, $this->callJobs('duration', $deployment)); - } - - private function callJobs(string $method, mixed ...$arguments): mixed - { - return (new ReflectionMethod($this->jobs, $method))->invoke($this->jobs, ...$arguments); - } -} diff --git a/tests/unit/Vcs/GitHubInstallationEventTest.php b/tests/unit/Vcs/GitHubInstallationEventTest.php deleted file mode 100644 index 045ac5d9d6..0000000000 --- a/tests/unit/Vcs/GitHubInstallationEventTest.php +++ /dev/null @@ -1,160 +0,0 @@ -handler = new Create(); - } - - public function testHandleInstallationEventIgnoresNonDeletedActions(): void - { - $dbForPlatform = $this->createMock(Database::class); - $authorization = $this->createStub(Authorization::class); - - $dbForPlatform->expects($this->never())->method('find'); - $dbForPlatform->expects($this->never())->method('deleteDocument'); - - $result = $this->callHandler('handleInstallationEvent', [ - 'action' => 'created', - 'installationId' => '123', - ], $dbForPlatform, $authorization, fn () => null); - - $this->assertNull($result); - } - - public function testHandleInstallationEventDisconnectsFunctionsAndSites(): void - { - $installation = new Document([ - '$id' => 'install1', - '$sequence' => 'install1seq', - 'projectId' => 'project1', - 'providerInstallationId' => 'gh-install-123', - ]); - - $function = new Document([ - '$id' => 'func1', - '$collection' => 'functions', - 'installationId' => 'install1', - 'installationInternalId' => 'install1seq', - 'providerRepositoryId' => 'repo123', - 'providerBranch' => 'main', - 'providerRootDirectory' => 'functions/my-func', - 'providerSilentMode' => true, - 'repositoryId' => 'repo1', - 'repositoryInternalId' => 'repo1seq', - ]); - - $project = new Document(['$id' => 'project1']); - - $dbForProject = $this->createMock(Database::class); - $dbForPlatform = $this->createMock(Database::class); - $authorization = $this->createMock(Authorization::class); - - $dbForPlatform->expects($this->exactly(2)) - ->method('find') - ->willReturnCallback(fn (string $collection): array => match ($collection) { - 'installations' => [$installation], - 'repositories' => [], - default => [], - }); - - $dbForPlatform->expects($this->once()) - ->method('getDocument') - ->with('projects', 'project1') - ->willReturn($project); - - $dbForProject->expects($this->exactly(2)) - ->method('find') - ->willReturnCallback(fn (string $collection): array => $collection === 'functions' ? [$function] : []); - - $dbForProject->expects($this->once()) - ->method('updateDocument') - ->with('functions', 'func1', $this->callback(function (Document $doc) { - $this->assertSame('', $doc->getAttribute('installationId')); - $this->assertSame('', $doc->getAttribute('installationInternalId')); - $this->assertSame('', $doc->getAttribute('providerRepositoryId')); - $this->assertSame('', $doc->getAttribute('providerBranch')); - $this->assertSame('', $doc->getAttribute('providerRootDirectory')); - $this->assertFalse($doc->getAttribute('providerSilentMode')); - $this->assertSame('', $doc->getAttribute('repositoryId')); - $this->assertSame('', $doc->getAttribute('repositoryInternalId')); - return true; - })); - - // 1: find(installations), 2: getDocument(projects), 3: find(functions), 4: updateDocument(func1), - // 5: find(sites), 6: find(repositories), 7: deleteDocument(installations) - $authorization->expects($this->exactly(7)) - ->method('skip') - ->willReturnCallback(fn (callable $fn): mixed => $fn()); - - $dbForPlatform->expects($this->once()) - ->method('deleteDocument') - ->with('installations', 'install1'); - - $getProjectDB = fn () => $dbForProject; - - $this->callHandler('handleInstallationEvent', [ - 'action' => 'deleted', - 'installationId' => 'gh-install-123', - ], $dbForPlatform, $authorization, $getProjectDB); - } - - public function testHandleInstallationEventSkipsDisconnectWhenProjectNotFound(): void - { - $installation = new Document([ - '$id' => 'install1', - '$sequence' => 'install1seq', - 'projectId' => 'project1', - 'providerInstallationId' => 'gh-install-123', - ]); - - $dbForPlatform = $this->createMock(Database::class); - $authorization = $this->createMock(Authorization::class); - - $dbForPlatform->expects($this->exactly(2)) - ->method('find') - ->willReturnCallback(fn (string $collection): array => match ($collection) { - 'installations' => [$installation], - 'repositories' => [], - default => [], - }); - - $dbForPlatform->expects($this->once()) - ->method('getDocument') - ->with('projects', 'project1') - ->willReturn(new Document()); - - // 1: find(installations), 2: getDocument(projects), 3: find(repositories), 4: deleteDocument(installations) - $authorization->expects($this->exactly(4)) - ->method('skip') - ->willReturnCallback(fn (callable $fn): mixed => $fn()); - - $dbForPlatform->expects($this->once()) - ->method('deleteDocument') - ->with('installations', 'install1'); - - $this->callHandler('handleInstallationEvent', [ - 'action' => 'deleted', - 'installationId' => 'gh-install-123', - ], $dbForPlatform, $authorization, fn () => null); - } - - private function callHandler(string $method, mixed ...$arguments): mixed - { - return (new ReflectionMethod($this->handler, $method))->invoke($this->handler, ...$arguments); - } -} diff --git a/tests/unit/Vcs/GiteaClosedPullRequestEventTest.php b/tests/unit/Vcs/GiteaClosedPullRequestEventTest.php deleted file mode 100644 index 93000c3339..0000000000 --- a/tests/unit/Vcs/GiteaClosedPullRequestEventTest.php +++ /dev/null @@ -1,84 +0,0 @@ -handler = new Create(); - } - - public function testClosedEventOnlyUpdatesGiteaRepositories(): void - { - // A GitHub repo and a Gitea repo can share the same numeric - // providerRepositoryId since each is scoped to its own server. - $giteaRepository = new Document([ - '$id' => 'repo-gitea', - 'installationId' => 'install-gitea', - 'providerRepositoryId' => '5', - 'providerPullRequestIds' => [5], - ]); - - $githubRepository = new Document([ - '$id' => 'repo-github', - 'installationId' => 'install-github', - 'providerRepositoryId' => '5', - 'providerPullRequestIds' => [5], - ]); - - $giteaInstallation = new Document(['$id' => 'install-gitea', 'provider' => 'gitea']); - $githubInstallation = new Document(['$id' => 'install-github', 'provider' => 'github']); - - $dbForPlatform = $this->createMock(Database::class); - $authorization = $this->createStub(Authorization::class); - - $dbForPlatform->method('find')->willReturn([$giteaRepository, $githubRepository]); - - $dbForPlatform->method('getDocument') - ->willReturnCallback(fn (string $collection, string $id): Document => match ($id) { - 'install-gitea' => $giteaInstallation, - 'install-github' => $githubInstallation, - default => new Document(), - }); - - $authorization->method('skip')->willReturnCallback(fn (callable $fn): mixed => $fn()); - - $dbForPlatform->expects($this->once()) - ->method('updateDocument') - ->with('repositories', 'repo-gitea', $this->callback(function (Document $doc) { - $this->assertSame([], array_values($doc->getAttribute('providerPullRequestIds'))); - return true; - })); - - $publisherForBuilds = $this->createStub(BuildPublisher::class); - $vcsFactory = $this->createStub(VcsFactory::class); - $installationTokens = $this->createStub(InstallationTokens::class); - - $this->callHandler('handlePullRequestEvent', [ - 'action' => 'closed', - 'repositoryId' => '5', - 'pullRequestNumber' => 5, - 'external' => true, - ], $vcsFactory, $installationTokens, $dbForPlatform, $authorization, $publisherForBuilds, fn () => null, []); - } - - private function callHandler(string $method, mixed ...$arguments): mixed - { - return (new ReflectionMethod($this->handler, $method))->invoke($this->handler, ...$arguments); - } -}