From 98dd8d98f352eb34d3599b317c51f2aa946544a8 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 2 Sep 2026 13:46:41 +0100 Subject: [PATCH] fix: let the storage strategy bring build output to buildPath, and cover S3 in CI Moving the artifact off the builds volume is part of the volume storage strategy, not of the Jobs worker: Deployments::store() now does it, next to storage() which decides where build.sh writes. A strategy whose artifacts already land on the device (Cloud uploads them from the sidecar) overrides it with a no-op instead of relying on the volume file being absent. The executor's storage DSN example carried the bucket in its path. Executor 0.29 appends that bucket to the endpoint and treats keys as bucket-relative, while Appwrite keys objects under the bucket already, so functions could not start ("File not found"). The example now leaves the bucket out for S3-compatible endpoints. CI gains an "S3 Storage" e2e config: a MinIO service behind the s3 compose profile, every storage device and the executor pointed at it, and a Functions test that deploys, executes and downloads a build through the configured device. The test fails on main (the deployment never becomes ready) and passes with the fix; it also runs in the regular Functions suite against the local device. --- .github/workflows/ci.yml | 20 +++++++ app/config/variables.php | 2 +- docker-compose.override.yml | 29 ++++++++++ src/Appwrite/Deployment/Deployments.php | 29 ++++++++++ .../Modules/Functions/Workers/Jobs.php | 40 +++++++------ .../Functions/FunctionsCustomServerTest.php | 56 +++++++++++++++++++ 6 files changed, 154 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a56b7fc26..d550cbde3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -593,6 +593,14 @@ jobs: storage_antivirus: enabled compose_profiles: antivirus wait: clamav + - label: S3 Storage + group: s3 + path: /usr/src/code/tests/e2e/Services + options_abuse: disabled + storage_antivirus: disabled + compose_profiles: s3 + wait: executor + storage_device: s3 env: # Shared tables mode connects every pool at boot, including the MongoDB-only DocumentsDB. COMPOSE_PROFILES: ${{ matrix.mode == 'shared' && format('mongodb,{0}', matrix.config.compose_profiles) || matrix.config.compose_profiles }} @@ -637,6 +645,14 @@ jobs: env: _APP_OPTIONS_ABUSE: ${{ matrix.config.options_abuse }} _APP_STORAGE_ANTIVIRUS: ${{ matrix.config.storage_antivirus }} + # The s3 config points every storage device and the executor at the MinIO profile service. + _APP_STORAGE_DEVICE: ${{ matrix.config.storage_device || 'local' }} + _APP_STORAGE_S3_ACCESS_KEY: minio + _APP_STORAGE_S3_SECRET: minio12345 + _APP_STORAGE_S3_BUCKET: appwrite + _APP_STORAGE_S3_REGION: us-east-1 + _APP_STORAGE_S3_ENDPOINT: http://minio:9000 + _APP_EXECUTOR_CONNECTION_STORAGE: ${{ matrix.config.storage_device == 's3' && 's3://minio:minio12345@localhost?region=us-east-1&url=http%3A%2F%2Fminio%3A9000' || 'local://localhost' }} _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} _APP_DATABASE_DOCUMENTSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'documentsdb_db_main' || '' }} _APP_DATABASE_VECTORSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'vectorsdb_db_main' || '' }} @@ -696,6 +712,10 @@ jobs: echo "=== ClamAV Logs ===" docker compose logs --timestamps clamav || true fi + if [ "${{ matrix.config.storage_device }}" = "s3" ]; then + echo "=== MinIO Logs ===" + docker compose logs --timestamps minio || true + fi benchmark: name: Benchmark diff --git a/app/config/variables.php b/app/config/variables.php index e21b086f92..2086d4110c 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -1242,7 +1242,7 @@ return [ ], [ 'name' => '_APP_EXECUTOR_CONNECTION_STORAGE', - 'description' => "DSN for Open Runtimes executor storage. When `_APP_STORAGE_DEVICE` is not local, point this at the same backend so the executor can read deployment artifacts. Defaults to `local://localhost`.\n\nExamples:\n- Local: `local://localhost`\n- AWS S3: `s3://ACCESS_KEY:SECRET@BUCKET.s3.REGION.amazonaws.com?region=REGION`\n- S3-compatible: `s3://ACCESS_KEY:SECRET@localhost/BUCKET?region=REGION&url=http%3A%2F%2Fminio%3A9000`", + 'description' => "DSN for Open Runtimes executor storage. When `_APP_STORAGE_DEVICE` is not local, point this at the same backend so the executor can read deployment artifacts. Defaults to `local://localhost`.\n\nExamples:\n- Local: `local://localhost`\n- AWS S3: `s3://ACCESS_KEY:SECRET@BUCKET.s3.REGION.amazonaws.com?region=REGION`\n- S3-compatible (`_APP_STORAGE_S3_ENDPOINT` set): `s3://ACCESS_KEY:SECRET@localhost?region=REGION&url=http%3A%2F%2Fminio%3A9000` — leave the bucket out of the DSN path; Appwrite keys objects under `BUCKET/` already, so the executor must not add it to the endpoint", 'introduction' => '1.9.5', 'default' => 'local://localhost', 'required' => false, diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 99dea6460b..5245a1c333 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -560,6 +560,35 @@ services: restart: unless-stopped networks: - appwrite + # S3-compatible storage for the s3 e2e group. Creates the bucket the + # _APP_STORAGE_S3_* variables point at once the server is up. + minio: + profiles: + - s3 + image: minio/minio:RELEASE.2025-09-07T16-13-09Z + container_name: appwrite-minio + restart: unless-stopped + networks: + - appwrite + environment: + - MINIO_ROOT_USER=${_APP_STORAGE_S3_ACCESS_KEY:-minio} + - MINIO_ROOT_PASSWORD=${_APP_STORAGE_S3_SECRET:-minio12345} + - MINIO_BUCKET=${_APP_STORAGE_S3_BUCKET:-appwrite} + entrypoint: + - sh + - -c + - | + minio server /data & + until mc alias set local http://localhost:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do sleep 1; done + mc mb -p "local/$$MINIO_BUCKET" + wait + healthcheck: + test: + - CMD-SHELL + - mc ls "local/$$MINIO_BUCKET" >/dev/null + interval: 5s + timeout: 5s + retries: 20 configs: redisinsight-connections.json: content: | diff --git a/src/Appwrite/Deployment/Deployments.php b/src/Appwrite/Deployment/Deployments.php index 978666cdfe..f1f6a280bb 100644 --- a/src/Appwrite/Deployment/Deployments.php +++ b/src/Appwrite/Deployment/Deployments.php @@ -23,6 +23,8 @@ use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Storage\Device; +use Utopia\Storage\Device\Local; +use Utopia\Storage\DeviceType; use Utopia\System\System; use Utopia\VCS\Adapter\Git; @@ -460,6 +462,33 @@ readonly class Deployments ]; } + /** + * Bring a finished build's artifact to the deployment's buildPath, once + * the job has exited and delivered its artifacts. With the volume strategy + * (see storage()) build.sh wrote it onto the builds volume, which on the + * local device already is the buildPath; on a remote device (S3 and + * friends) it is moved off the volume onto the device here. A strategy + * whose artifacts already land on the device — Cloud uploads them from + * the sidecar — overrides this with a no-op. + */ + public function store(Document $deployment): void + { + if ($this->deviceForBuilds->getType() === DeviceType::Local) { + return; + } + + $source = static::buildPath($this->project->getId(), $deployment->getId()); + $target = (string) $deployment->getAttribute('buildPath', ''); + $local = new Local(); + if ($target === '' || ! $local->exists($source)) { + return; + } + + if ($local->copy($source, $target, $this->deviceForBuilds)) { + $local->delete($source); + } + } + /** * The jobs-service job id for a deployment build (used to submit and cancel). */ diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index ab3d43684d..d5782bb1e9 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -28,8 +28,6 @@ use Utopia\Database\Query; use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\Storage\Device; -use Utopia\Storage\Device\Local; -use Utopia\Storage\DeviceType; use Utopia\System\System; /** @@ -90,6 +88,7 @@ class Jobs extends Action ->inject('platform') ->inject('plan') ->inject('bus') + ->inject('deployments') ->callback($this->action(...)); } @@ -112,6 +111,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): void { $event = JobsMessage::fromArray($message->getPayload()); @@ -120,7 +120,7 @@ class Jobs extends Action return; } - $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForScreenshots, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $deploymentId, $bus): void { + $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForScreenshots, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $deploymentId, $bus, $deployments): void { if ($event->id !== '') { $key = 'jobs-event-' . $event->id; if ($cache->load($key, self::DEDUPE_TTL) !== false) { @@ -138,10 +138,10 @@ class Jobs extends Action $deployment = match ($event->event) { 'orchestrator.job.log' => $this->onLog($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $vcsFactory, $platform), - 'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), - 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), - 'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), - default => $this->onCallback($event->event, $dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), + 'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), + 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), + 'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), + default => $this->onCallback($event->event, $dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), }; // Console realtime on every callback (log stream + status). @@ -229,6 +229,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { return $deployment; } @@ -278,6 +279,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { if (($data['artifactId'] ?? '') === 'manifest') { // A failed manifest degrades to an empty listing (detection @@ -286,7 +288,7 @@ class Jobs extends Action $files = \is_array($manifest) ? (array) ($manifest['files'] ?? []) : []; $cache->save('jobs-manifest-' . $deployment->getId(), ['files' => \array_values($files)]); - return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus); + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments); } if (($data['artifactId'] ?? '') !== 'sourceSize' || ($data['status'] ?? '') !== 'success') { @@ -327,6 +329,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { if ($exitCode !== 0) { return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus); @@ -334,7 +337,7 @@ class Jobs extends Action $cache->save('jobs-exit-' . $deployment->getId(), true); - return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus); + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments); } /** @@ -355,10 +358,11 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { $cache->save('jobs-complete-' . $deployment->getId(), true); - return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus); + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments); } /** @@ -379,6 +383,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'], true)) { return $deployment; // already finalized @@ -403,18 +408,11 @@ class Jobs extends Action } } - // The build job wrote its artifact onto the shared builds volume (see - // Deployments::storage()). On a remote device (S3 and friends) the - // executor and the download endpoint read buildPath through that - // device, so move the artifact off the volume onto it first. + // The build job wrote its artifact where the storage strategy put it; + // bring it to buildPath before checking for it. + $deployments->store($deployment); + $path = (string) $deployment->getAttribute('buildPath', ''); - if ($path !== '' && $deviceForBuilds->getType() !== DeviceType::Local) { - $local = new Local(); - $source = Deployments::buildPath($project->getId(), $deploymentId); - if ($local->exists($source) && $local->copy($source, $path, $deviceForBuilds)) { - $local->delete($source); - } - } if ($path === '' || ! $deviceForBuilds->exists($path)) { return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build produced no output artifact.', $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus); } diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 84b3269e21..5102b241c6 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -7,6 +7,7 @@ namespace Tests\E2E\Services\Functions; use Appwrite\Platform\Modules\Compute\Specification; use Appwrite\Tests\Retry; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\Group; use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; @@ -1030,6 +1031,61 @@ final class FunctionsCustomServerTest extends Scope $this->assertEquals(204, $deployment['headers']['status-code']); } + /** + * The build job writes onto the builds volume while the executor and the + * download endpoint read the deployment through the configured storage + * device. Runs in the `s3` CI group against MinIO, where the artifact has + * to be moved off the volume and keyed under the bucket; on the default + * local device the two are the same path. + */ + #[Group('s3')] + public function testDeploymentBuildOutputIsServedFromTheBuildsDevice(): void + { + $functionId = $this->setupFunction([ + 'functionId' => ID::unique(), + 'name' => 'Build output', + 'runtime' => 'node-22', + 'entrypoint' => 'index.js', + 'execute' => [Role::any()->toString()], + ]); + + $deployment = $this->createDeployment($functionId, [ + 'code' => $this->packageFunction('basic'), + 'activate' => true, + ]); + $this->assertEquals(202, $deployment['headers']['status-code']); + $deploymentId = $deployment['body']['$id']; + + $this->assertEventually(function () use ($functionId, $deploymentId) { + $deployment = $this->getDeployment($functionId, $deploymentId); + $this->assertEquals('ready', $deployment['body']['status'], $deployment['body']['buildLogs'] ?? ''); + }, 100000, 500); + + /** + * Test for SUCCESS + */ + $deployment = $this->getDeployment($functionId, $deploymentId); + $this->assertGreaterThan(0, $deployment['body']['buildSize']); + + // The executor fetches the artifact through its own storage connection. + $execution = $this->createExecution($functionId); + $this->assertEquals(201, $execution['headers']['status-code']); + $this->assertEquals('completed', $execution['body']['status'], $execution['body']['errors'] ?? ''); + $this->assertEquals(200, $execution['body']['responseStatusCode']); + $this->assertSame($deploymentId, \json_decode($execution['body']['responseBody'], true)['APPWRITE_FUNCTION_DEPLOYMENT']); + + // The download endpoint reads buildPath through the builds device. + $output = $this->getDeploymentDownload($functionId, $deploymentId, 'output'); + $this->assertEquals(200, $output['headers']['status-code']); + $this->assertStringStartsWith("\x1f\x8b", $output['body']); + + $source = $this->getDeploymentDownload($functionId, $deploymentId, 'source'); + $this->assertEquals(200, $source['headers']['status-code']); + $this->assertStringStartsWith("\x1f\x8b", $source['body']); + + $this->cleanupFunction($functionId); + } + #[Retry(count: 3)] public function testCancelDeploymentBuild(): void {