From 3e1b1ad578ca6b416b7bb8c110bab0ad9a90c4b9 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 1 Sep 2026 20:41:34 +0100 Subject: [PATCH 1/6] fix: publish build output to the builds device when storage is not local The jobs-based build system writes the artifact onto the shared builds volume, but the Jobs worker, the executor and the download endpoint all read the deployment's buildPath through the configured storage device. With _APP_STORAGE_DEVICE=s3 the artifact never reached S3, so every build failed with "Build produced no output artifact". With an S3-compatible endpoint (MinIO, etc.) the stored buildPath was also wrong: the device folds the bucket into its root, so the plain volume path pointed at a bucket named "storage". The deployment now stores the device path as its buildPath (unchanged for the local device), and the Jobs worker moves the artifact off the volume onto a remote device before checking readiness. The separate-profile jobs worker also receives the storage variables it needs to build that device. --- app/init/resources/request.php | 12 ++++-- app/init/worker/message.php | 7 ++-- docker-compose.yml | 22 +++++++++++ src/Appwrite/Deployment/Deployments.php | 19 ++++++++- .../Modules/Functions/Workers/Jobs.php | 29 ++++++++++++++ tests/unit/Deployment/DeploymentsTest.php | 39 +++++++++++++++++++ 6 files changed, 120 insertions(+), 8 deletions(-) diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 8039571243..af80952ef2 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -213,9 +213,15 @@ return function (Container $context): void { ), ['publisher']); // Builds a Deployments bound to a given project — webhook handlers resolve // their tenant projects mid-request, after this container is initialized. - $context->set('deploymentsFactory', function (Jobs $jobs, array $platform) { - return fn (Database $dbForProject, Document $project): Deployments => new Deployments($jobs, $dbForProject, $project, $platform); - }, ['jobs', 'platform']); + $context->set('deploymentsFactory', function (Jobs $jobs, array $platform, Telemetry $telemetry) { + return fn (Database $dbForProject, Document $project): Deployments => new Deployments( + $jobs, + $dbForProject, + $project, + $platform, + new Device\Telemetry($telemetry, getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId())), + ); + }, ['jobs', 'platform', 'telemetry']); $context->set('deployments', fn (callable $deploymentsFactory, Database $dbForProject, Document $project) => $deploymentsFactory($dbForProject, $project), ['deploymentsFactory', 'dbForProject', 'project']); $context->set('eventProcessor', fn () => new EventProcessor(), []); $context->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory( diff --git a/app/init/worker/message.php b/app/init/worker/message.php index 6f6fc39eef..4ba9d415da 100644 --- a/app/init/worker/message.php +++ b/app/init/worker/message.php @@ -24,6 +24,7 @@ use Utopia\Queue\Publisher; use Utopia\Queue\Queue; use Utopia\Registry\Registry; use Utopia\Span\Span; +use Utopia\Storage\Device; use Utopia\Storage\Device\Telemetry as TelemetryDevice; use Utopia\System\System; use Utopia\Telemetry\Adapter as Telemetry; @@ -190,9 +191,9 @@ return function (Container $container): void { // Only the Builds worker uses this, handing template-into-repo pushes to // the jobs-service. - $container->set('deployments', function (Jobs $jobs, Database $dbForProject, Document $project, array $platform) { - return new Deployments($jobs, $dbForProject, $project, $platform); - }, ['jobs', 'dbForProject', 'project', 'platform']); + $container->set('deployments', function (Jobs $jobs, Database $dbForProject, Document $project, array $platform, Device $deviceForBuilds) { + return new Deployments($jobs, $dbForProject, $project, $platform, $deviceForBuilds); + }, ['jobs', 'dbForProject', 'project', 'platform', 'deviceForBuilds']); $container->set('logError', function (Registry $register, Document $project) { return function (Throwable $error, string $namespace, string $action, ?array $extras = null) use ($register, $project) { diff --git a/docker-compose.yml b/docker-compose.yml index 9d5793dd77..7e15696bb3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -902,6 +902,28 @@ services: - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 + - _APP_STORAGE_DEVICE + - _APP_STORAGE_S3_ACCESS_KEY + - _APP_STORAGE_S3_SECRET + - _APP_STORAGE_S3_REGION + - _APP_STORAGE_S3_BUCKET + - _APP_STORAGE_S3_ENDPOINT + - _APP_STORAGE_DO_SPACES_ACCESS_KEY + - _APP_STORAGE_DO_SPACES_SECRET + - _APP_STORAGE_DO_SPACES_REGION + - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_BACKBLAZE_ACCESS_KEY + - _APP_STORAGE_BACKBLAZE_SECRET + - _APP_STORAGE_BACKBLAZE_REGION + - _APP_STORAGE_BACKBLAZE_BUCKET + - _APP_STORAGE_LINODE_ACCESS_KEY + - _APP_STORAGE_LINODE_SECRET + - _APP_STORAGE_LINODE_REGION + - _APP_STORAGE_LINODE_BUCKET + - _APP_STORAGE_WASABI_ACCESS_KEY + - _APP_STORAGE_WASABI_SECRET + - _APP_STORAGE_WASABI_REGION + - _APP_STORAGE_WASABI_BUCKET - _APP_JOBS_HOST - _APP_JOBS_SECRET - _APP_REDIS_HOST diff --git a/src/Appwrite/Deployment/Deployments.php b/src/Appwrite/Deployment/Deployments.php index 000e7de5d3..bf1554277a 100644 --- a/src/Appwrite/Deployment/Deployments.php +++ b/src/Appwrite/Deployment/Deployments.php @@ -22,6 +22,7 @@ use Utopia\Database\Document; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; +use Utopia\Storage\Device; use Utopia\System\System; use Utopia\VCS\Adapter\Git; @@ -53,6 +54,7 @@ readonly class Deployments protected Database $dbForProject, protected Document $project, private array $platform, + private Device $deviceForBuilds, ) { } @@ -182,7 +184,7 @@ readonly class Deployments $queued = $this->dbForProject->updateDocuments('deployments', new Document([ 'status' => 'waiting', - 'buildPath' => static::buildPath($this->project->getId(), $deployment->getId()), + 'buildPath' => $this->devicePath($deployment), ]), [ Query::equal('$id', [$deployment->getId()]), Query::notEqual('status', 'canceled'), @@ -476,7 +478,20 @@ readonly class Deployments } /** - * The build output path on the builds volume, declared at submission. + * The deployment's buildPath: where the artifact lives on the builds + * device, which is what the executor, the download endpoint and the + * deletes worker read. Matches buildPath() on the local device; a remote + * device (S3 and friends) keys it under its own root, bucket included. + */ + public function devicePath(Document $deployment): string + { + return $this->deviceForBuilds->getPath($deployment->getId() . '/' . static::artifact()); + } + + /** + * The build output path on the builds volume. On the local device this is + * also the deployment's buildPath; on a remote device (S3 and friends) the + * buildPath is a device path and the Jobs worker moves the artifact there. */ public static function buildPath(string $projectId, string $deploymentId): string { diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index 4b9c32533d..ee0ca36835 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Functions\Workers; use Appwrite\Bus\Events\RuleUpdated; +use Appwrite\Deployment\Deployments; use Appwrite\Deployment\Detection; use Appwrite\Deployment\GitAction; use Appwrite\Event\Event; @@ -27,6 +28,8 @@ use Utopia\Database\Query; use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\Storage\Device; +use Utopia\Storage\Device\Local; +use Utopia\Storage\DeviceType; use Utopia\System\System; /** @@ -401,6 +404,9 @@ class Jobs extends Action } $path = (string) $deployment->getAttribute('buildPath', ''); + if ($path !== '') { + $this->publish($deviceForBuilds, Deployments::buildPath($project->getId(), $deploymentId), $path); + } if ($path === '' || ! $deviceForBuilds->exists($path)) { return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build produced no output artifact.', $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus); } @@ -429,6 +435,29 @@ class Jobs extends Action return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, true, '', $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus, $size); } + /** + * The build job writes its artifact onto the shared builds volume (see + * Deployments::storage()). When the builds device is remote (S3 and + * friends), the executor and the download endpoint read the deployment's + * buildPath through that device, so move the artifact off the volume onto + * it. A no-op on the local device, where the volume already is the device. + */ + private function publish(Device $deviceForBuilds, string $source, string $target): void + { + if ($deviceForBuilds->getType() === DeviceType::Local) { + return; + } + + $local = new Local(); + if (! $local->exists($source)) { + return; + } + + if ($local->copy($source, $target, $deviceForBuilds)) { + $local->delete($source); + } + } + /** * Adapter detection over the site's build-manifest file listing: a first * successful build pins adapter + fallbackFile on the site and deployment; diff --git a/tests/unit/Deployment/DeploymentsTest.php b/tests/unit/Deployment/DeploymentsTest.php index de41f435d3..5fe08ab1d8 100644 --- a/tests/unit/Deployment/DeploymentsTest.php +++ b/tests/unit/Deployment/DeploymentsTest.php @@ -6,9 +6,14 @@ namespace Tests\Unit\Deployment; use Appwrite\Deployment\Deployments; use Appwrite\Extend\Exception; +use OpenRuntimes\Orchestrator\Jobs; use PHPUnit\Framework\TestCase; use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Storage\Device; +use Utopia\Storage\Device\Local; +use Utopia\Storage\Device\S3; final class DeploymentsTest extends TestCase { @@ -128,6 +133,40 @@ final class DeploymentsTest extends TestCase $this->assertSame(['users.read'], Deployments::scopes($site)); } + private function deployments(Device $deviceForBuilds): Deployments + { + return new Deployments( + new Jobs(), + $this->createStub(Database::class), + new Document(['$id' => 'project1']), + ['apiHostname' => 'localhost'], + $deviceForBuilds, + ); + } + + public function testDevicePathOnLocalDeviceIsTheVolumePath(): void + { + $deployments = $this->deployments(new Local(APP_STORAGE_BUILDS . '/app-project1')); + + $this->assertSame( + Deployments::buildPath('project1', 'deployment1'), + $deployments->devicePath(new Document(['$id' => 'deployment1'])) + ); + } + + public function testDevicePathOnRemoteDeviceIsKeyedUnderTheDeviceRoot(): void + { + // An S3-compatible endpoint folds the bucket into the device root (see + // getDevice()), so the artifact key is not the volume path. + $device = new S3('appwrite' . APP_STORAGE_BUILDS . '/app-project1', 'key', 'secret', 'http://minio:9000', 'us-east-1'); + $deployments = $this->deployments($device); + + $path = $deployments->devicePath(new Document(['$id' => 'deployment1'])); + + $this->assertSame('appwrite' . APP_STORAGE_BUILDS . '/app-project1/deployment1/' . Deployments::artifact(), $path); + $this->assertNotSame(Deployments::buildPath('project1', 'deployment1'), $path); + } + private function buildPayload(array $vars): array { // Presigned-URL and ephemeral-key signing both run before the From 3f9dad26de8fcb1a41fe7dd3ab3ee3f9e3463f93 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 2 Sep 2026 13:17:47 +0100 Subject: [PATCH 2/6] refactor: inline the buildPath device lookup --- src/Appwrite/Deployment/Deployments.php | 13 +------- tests/unit/Deployment/DeploymentsTest.php | 39 ----------------------- 2 files changed, 1 insertion(+), 51 deletions(-) diff --git a/src/Appwrite/Deployment/Deployments.php b/src/Appwrite/Deployment/Deployments.php index bf1554277a..978666cdfe 100644 --- a/src/Appwrite/Deployment/Deployments.php +++ b/src/Appwrite/Deployment/Deployments.php @@ -184,7 +184,7 @@ readonly class Deployments $queued = $this->dbForProject->updateDocuments('deployments', new Document([ 'status' => 'waiting', - 'buildPath' => $this->devicePath($deployment), + 'buildPath' => $this->deviceForBuilds->getPath($deployment->getId() . '/' . static::artifact()), ]), [ Query::equal('$id', [$deployment->getId()]), Query::notEqual('status', 'canceled'), @@ -477,17 +477,6 @@ readonly class Deployments return APP_STORAGE_BUILDS . "/app-{$projectId}/{$deploymentId}"; } - /** - * The deployment's buildPath: where the artifact lives on the builds - * device, which is what the executor, the download endpoint and the - * deletes worker read. Matches buildPath() on the local device; a remote - * device (S3 and friends) keys it under its own root, bucket included. - */ - public function devicePath(Document $deployment): string - { - return $this->deviceForBuilds->getPath($deployment->getId() . '/' . static::artifact()); - } - /** * The build output path on the builds volume. On the local device this is * also the deployment's buildPath; on a remote device (S3 and friends) the diff --git a/tests/unit/Deployment/DeploymentsTest.php b/tests/unit/Deployment/DeploymentsTest.php index 5fe08ab1d8..de41f435d3 100644 --- a/tests/unit/Deployment/DeploymentsTest.php +++ b/tests/unit/Deployment/DeploymentsTest.php @@ -6,14 +6,9 @@ namespace Tests\Unit\Deployment; use Appwrite\Deployment\Deployments; use Appwrite\Extend\Exception; -use OpenRuntimes\Orchestrator\Jobs; use PHPUnit\Framework\TestCase; use Utopia\Config\Config; -use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Storage\Device; -use Utopia\Storage\Device\Local; -use Utopia\Storage\Device\S3; final class DeploymentsTest extends TestCase { @@ -133,40 +128,6 @@ final class DeploymentsTest extends TestCase $this->assertSame(['users.read'], Deployments::scopes($site)); } - private function deployments(Device $deviceForBuilds): Deployments - { - return new Deployments( - new Jobs(), - $this->createStub(Database::class), - new Document(['$id' => 'project1']), - ['apiHostname' => 'localhost'], - $deviceForBuilds, - ); - } - - public function testDevicePathOnLocalDeviceIsTheVolumePath(): void - { - $deployments = $this->deployments(new Local(APP_STORAGE_BUILDS . '/app-project1')); - - $this->assertSame( - Deployments::buildPath('project1', 'deployment1'), - $deployments->devicePath(new Document(['$id' => 'deployment1'])) - ); - } - - public function testDevicePathOnRemoteDeviceIsKeyedUnderTheDeviceRoot(): void - { - // An S3-compatible endpoint folds the bucket into the device root (see - // getDevice()), so the artifact key is not the volume path. - $device = new S3('appwrite' . APP_STORAGE_BUILDS . '/app-project1', 'key', 'secret', 'http://minio:9000', 'us-east-1'); - $deployments = $this->deployments($device); - - $path = $deployments->devicePath(new Document(['$id' => 'deployment1'])); - - $this->assertSame('appwrite' . APP_STORAGE_BUILDS . '/app-project1/deployment1/' . Deployments::artifact(), $path); - $this->assertNotSame(Deployments::buildPath('project1', 'deployment1'), $path); - } - private function buildPayload(array $vars): array { // Presigned-URL and ephemeral-key signing both run before the From 09f80fdffe770950220091f7bbb3bed30a8fe2cf Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 2 Sep 2026 13:19:08 +0100 Subject: [PATCH 3/6] refactor: inline the build artifact move into ready() --- .../Modules/Functions/Workers/Jobs.php | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index ee0ca36835..ab3d43684d 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -403,9 +403,17 @@ class Jobs extends Action } } + // The build job wrote its artifact onto the shared builds volume (see + // Deployments::storage()). On a remote device (S3 and friends) the + // executor and the download endpoint read buildPath through that + // device, so move the artifact off the volume onto it first. $path = (string) $deployment->getAttribute('buildPath', ''); - if ($path !== '') { - $this->publish($deviceForBuilds, Deployments::buildPath($project->getId(), $deploymentId), $path); + if ($path !== '' && $deviceForBuilds->getType() !== DeviceType::Local) { + $local = new Local(); + $source = Deployments::buildPath($project->getId(), $deploymentId); + if ($local->exists($source) && $local->copy($source, $path, $deviceForBuilds)) { + $local->delete($source); + } } if ($path === '' || ! $deviceForBuilds->exists($path)) { return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build produced no output artifact.', $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus); @@ -435,29 +443,6 @@ class Jobs extends Action return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, true, '', $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus, $size); } - /** - * The build job writes its artifact onto the shared builds volume (see - * Deployments::storage()). When the builds device is remote (S3 and - * friends), the executor and the download endpoint read the deployment's - * buildPath through that device, so move the artifact off the volume onto - * it. A no-op on the local device, where the volume already is the device. - */ - private function publish(Device $deviceForBuilds, string $source, string $target): void - { - if ($deviceForBuilds->getType() === DeviceType::Local) { - return; - } - - $local = new Local(); - if (! $local->exists($source)) { - return; - } - - if ($local->copy($source, $target, $deviceForBuilds)) { - $local->delete($source); - } - } - /** * Adapter detection over the site's build-manifest file listing: a first * successful build pins adapter + fallbackFile on the site and deployment; From 98dd8d98f352eb34d3599b317c51f2aa946544a8 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 2 Sep 2026 13:46:41 +0100 Subject: [PATCH 4/6] fix: let the storage strategy bring build output to buildPath, and cover S3 in CI Moving the artifact off the builds volume is part of the volume storage strategy, not of the Jobs worker: Deployments::store() now does it, next to storage() which decides where build.sh writes. A strategy whose artifacts already land on the device (Cloud uploads them from the sidecar) overrides it with a no-op instead of relying on the volume file being absent. The executor's storage DSN example carried the bucket in its path. Executor 0.29 appends that bucket to the endpoint and treats keys as bucket-relative, while Appwrite keys objects under the bucket already, so functions could not start ("File not found"). The example now leaves the bucket out for S3-compatible endpoints. CI gains an "S3 Storage" e2e config: a MinIO service behind the s3 compose profile, every storage device and the executor pointed at it, and a Functions test that deploys, executes and downloads a build through the configured device. The test fails on main (the deployment never becomes ready) and passes with the fix; it also runs in the regular Functions suite against the local device. --- .github/workflows/ci.yml | 20 +++++++ app/config/variables.php | 2 +- docker-compose.override.yml | 29 ++++++++++ src/Appwrite/Deployment/Deployments.php | 29 ++++++++++ .../Modules/Functions/Workers/Jobs.php | 40 +++++++------ .../Functions/FunctionsCustomServerTest.php | 56 +++++++++++++++++++ 6 files changed, 154 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a56b7fc26..d550cbde3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -593,6 +593,14 @@ jobs: storage_antivirus: enabled compose_profiles: antivirus wait: clamav + - label: S3 Storage + group: s3 + path: /usr/src/code/tests/e2e/Services + options_abuse: disabled + storage_antivirus: disabled + compose_profiles: s3 + wait: executor + storage_device: s3 env: # Shared tables mode connects every pool at boot, including the MongoDB-only DocumentsDB. COMPOSE_PROFILES: ${{ matrix.mode == 'shared' && format('mongodb,{0}', matrix.config.compose_profiles) || matrix.config.compose_profiles }} @@ -637,6 +645,14 @@ jobs: env: _APP_OPTIONS_ABUSE: ${{ matrix.config.options_abuse }} _APP_STORAGE_ANTIVIRUS: ${{ matrix.config.storage_antivirus }} + # The s3 config points every storage device and the executor at the MinIO profile service. + _APP_STORAGE_DEVICE: ${{ matrix.config.storage_device || 'local' }} + _APP_STORAGE_S3_ACCESS_KEY: minio + _APP_STORAGE_S3_SECRET: minio12345 + _APP_STORAGE_S3_BUCKET: appwrite + _APP_STORAGE_S3_REGION: us-east-1 + _APP_STORAGE_S3_ENDPOINT: http://minio:9000 + _APP_EXECUTOR_CONNECTION_STORAGE: ${{ matrix.config.storage_device == 's3' && 's3://minio:minio12345@localhost?region=us-east-1&url=http%3A%2F%2Fminio%3A9000' || 'local://localhost' }} _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} _APP_DATABASE_DOCUMENTSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'documentsdb_db_main' || '' }} _APP_DATABASE_VECTORSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'vectorsdb_db_main' || '' }} @@ -696,6 +712,10 @@ jobs: echo "=== ClamAV Logs ===" docker compose logs --timestamps clamav || true fi + if [ "${{ matrix.config.storage_device }}" = "s3" ]; then + echo "=== MinIO Logs ===" + docker compose logs --timestamps minio || true + fi benchmark: name: Benchmark diff --git a/app/config/variables.php b/app/config/variables.php index e21b086f92..2086d4110c 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -1242,7 +1242,7 @@ return [ ], [ 'name' => '_APP_EXECUTOR_CONNECTION_STORAGE', - 'description' => "DSN for Open Runtimes executor storage. When `_APP_STORAGE_DEVICE` is not local, point this at the same backend so the executor can read deployment artifacts. Defaults to `local://localhost`.\n\nExamples:\n- Local: `local://localhost`\n- AWS S3: `s3://ACCESS_KEY:SECRET@BUCKET.s3.REGION.amazonaws.com?region=REGION`\n- S3-compatible: `s3://ACCESS_KEY:SECRET@localhost/BUCKET?region=REGION&url=http%3A%2F%2Fminio%3A9000`", + 'description' => "DSN for Open Runtimes executor storage. When `_APP_STORAGE_DEVICE` is not local, point this at the same backend so the executor can read deployment artifacts. Defaults to `local://localhost`.\n\nExamples:\n- Local: `local://localhost`\n- AWS S3: `s3://ACCESS_KEY:SECRET@BUCKET.s3.REGION.amazonaws.com?region=REGION`\n- S3-compatible (`_APP_STORAGE_S3_ENDPOINT` set): `s3://ACCESS_KEY:SECRET@localhost?region=REGION&url=http%3A%2F%2Fminio%3A9000` — leave the bucket out of the DSN path; Appwrite keys objects under `BUCKET/` already, so the executor must not add it to the endpoint", 'introduction' => '1.9.5', 'default' => 'local://localhost', 'required' => false, diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 99dea6460b..5245a1c333 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -560,6 +560,35 @@ services: restart: unless-stopped networks: - appwrite + # S3-compatible storage for the s3 e2e group. Creates the bucket the + # _APP_STORAGE_S3_* variables point at once the server is up. + minio: + profiles: + - s3 + image: minio/minio:RELEASE.2025-09-07T16-13-09Z + container_name: appwrite-minio + restart: unless-stopped + networks: + - appwrite + environment: + - MINIO_ROOT_USER=${_APP_STORAGE_S3_ACCESS_KEY:-minio} + - MINIO_ROOT_PASSWORD=${_APP_STORAGE_S3_SECRET:-minio12345} + - MINIO_BUCKET=${_APP_STORAGE_S3_BUCKET:-appwrite} + entrypoint: + - sh + - -c + - | + minio server /data & + until mc alias set local http://localhost:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do sleep 1; done + mc mb -p "local/$$MINIO_BUCKET" + wait + healthcheck: + test: + - CMD-SHELL + - mc ls "local/$$MINIO_BUCKET" >/dev/null + interval: 5s + timeout: 5s + retries: 20 configs: redisinsight-connections.json: content: | diff --git a/src/Appwrite/Deployment/Deployments.php b/src/Appwrite/Deployment/Deployments.php index 978666cdfe..f1f6a280bb 100644 --- a/src/Appwrite/Deployment/Deployments.php +++ b/src/Appwrite/Deployment/Deployments.php @@ -23,6 +23,8 @@ use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Storage\Device; +use Utopia\Storage\Device\Local; +use Utopia\Storage\DeviceType; use Utopia\System\System; use Utopia\VCS\Adapter\Git; @@ -460,6 +462,33 @@ readonly class Deployments ]; } + /** + * Bring a finished build's artifact to the deployment's buildPath, once + * the job has exited and delivered its artifacts. With the volume strategy + * (see storage()) build.sh wrote it onto the builds volume, which on the + * local device already is the buildPath; on a remote device (S3 and + * friends) it is moved off the volume onto the device here. A strategy + * whose artifacts already land on the device — Cloud uploads them from + * the sidecar — overrides this with a no-op. + */ + public function store(Document $deployment): void + { + if ($this->deviceForBuilds->getType() === DeviceType::Local) { + return; + } + + $source = static::buildPath($this->project->getId(), $deployment->getId()); + $target = (string) $deployment->getAttribute('buildPath', ''); + $local = new Local(); + if ($target === '' || ! $local->exists($source)) { + return; + } + + if ($local->copy($source, $target, $this->deviceForBuilds)) { + $local->delete($source); + } + } + /** * The jobs-service job id for a deployment build (used to submit and cancel). */ diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index ab3d43684d..d5782bb1e9 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -28,8 +28,6 @@ use Utopia\Database\Query; use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\Storage\Device; -use Utopia\Storage\Device\Local; -use Utopia\Storage\DeviceType; use Utopia\System\System; /** @@ -90,6 +88,7 @@ class Jobs extends Action ->inject('platform') ->inject('plan') ->inject('bus') + ->inject('deployments') ->callback($this->action(...)); } @@ -112,6 +111,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): void { $event = JobsMessage::fromArray($message->getPayload()); @@ -120,7 +120,7 @@ class Jobs extends Action return; } - $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForScreenshots, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $deploymentId, $bus): void { + $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForScreenshots, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $deploymentId, $bus, $deployments): void { if ($event->id !== '') { $key = 'jobs-event-' . $event->id; if ($cache->load($key, self::DEDUPE_TTL) !== false) { @@ -138,10 +138,10 @@ class Jobs extends Action $deployment = match ($event->event) { 'orchestrator.job.log' => $this->onLog($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $vcsFactory, $platform), - 'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), - 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), - 'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), - default => $this->onCallback($event->event, $dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), + 'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), + 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), + 'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), + default => $this->onCallback($event->event, $dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), }; // Console realtime on every callback (log stream + status). @@ -229,6 +229,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { return $deployment; } @@ -278,6 +279,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { if (($data['artifactId'] ?? '') === 'manifest') { // A failed manifest degrades to an empty listing (detection @@ -286,7 +288,7 @@ class Jobs extends Action $files = \is_array($manifest) ? (array) ($manifest['files'] ?? []) : []; $cache->save('jobs-manifest-' . $deployment->getId(), ['files' => \array_values($files)]); - return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus); + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments); } if (($data['artifactId'] ?? '') !== 'sourceSize' || ($data['status'] ?? '') !== 'success') { @@ -327,6 +329,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { if ($exitCode !== 0) { return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus); @@ -334,7 +337,7 @@ class Jobs extends Action $cache->save('jobs-exit-' . $deployment->getId(), true); - return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus); + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments); } /** @@ -355,10 +358,11 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { $cache->save('jobs-complete-' . $deployment->getId(), true); - return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus); + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments); } /** @@ -379,6 +383,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'], true)) { return $deployment; // already finalized @@ -403,18 +408,11 @@ class Jobs extends Action } } - // The build job wrote its artifact onto the shared builds volume (see - // Deployments::storage()). On a remote device (S3 and friends) the - // executor and the download endpoint read buildPath through that - // device, so move the artifact off the volume onto it first. + // The build job wrote its artifact where the storage strategy put it; + // bring it to buildPath before checking for it. + $deployments->store($deployment); + $path = (string) $deployment->getAttribute('buildPath', ''); - if ($path !== '' && $deviceForBuilds->getType() !== DeviceType::Local) { - $local = new Local(); - $source = Deployments::buildPath($project->getId(), $deploymentId); - if ($local->exists($source) && $local->copy($source, $path, $deviceForBuilds)) { - $local->delete($source); - } - } if ($path === '' || ! $deviceForBuilds->exists($path)) { return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build produced no output artifact.', $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus); } diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 84b3269e21..5102b241c6 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -7,6 +7,7 @@ namespace Tests\E2E\Services\Functions; use Appwrite\Platform\Modules\Compute\Specification; use Appwrite\Tests\Retry; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\Group; use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; @@ -1030,6 +1031,61 @@ final class FunctionsCustomServerTest extends Scope $this->assertEquals(204, $deployment['headers']['status-code']); } + /** + * The build job writes onto the builds volume while the executor and the + * download endpoint read the deployment through the configured storage + * device. Runs in the `s3` CI group against MinIO, where the artifact has + * to be moved off the volume and keyed under the bucket; on the default + * local device the two are the same path. + */ + #[Group('s3')] + public function testDeploymentBuildOutputIsServedFromTheBuildsDevice(): void + { + $functionId = $this->setupFunction([ + 'functionId' => ID::unique(), + 'name' => 'Build output', + 'runtime' => 'node-22', + 'entrypoint' => 'index.js', + 'execute' => [Role::any()->toString()], + ]); + + $deployment = $this->createDeployment($functionId, [ + 'code' => $this->packageFunction('basic'), + 'activate' => true, + ]); + $this->assertEquals(202, $deployment['headers']['status-code']); + $deploymentId = $deployment['body']['$id']; + + $this->assertEventually(function () use ($functionId, $deploymentId) { + $deployment = $this->getDeployment($functionId, $deploymentId); + $this->assertEquals('ready', $deployment['body']['status'], $deployment['body']['buildLogs'] ?? ''); + }, 100000, 500); + + /** + * Test for SUCCESS + */ + $deployment = $this->getDeployment($functionId, $deploymentId); + $this->assertGreaterThan(0, $deployment['body']['buildSize']); + + // The executor fetches the artifact through its own storage connection. + $execution = $this->createExecution($functionId); + $this->assertEquals(201, $execution['headers']['status-code']); + $this->assertEquals('completed', $execution['body']['status'], $execution['body']['errors'] ?? ''); + $this->assertEquals(200, $execution['body']['responseStatusCode']); + $this->assertSame($deploymentId, \json_decode($execution['body']['responseBody'], true)['APPWRITE_FUNCTION_DEPLOYMENT']); + + // The download endpoint reads buildPath through the builds device. + $output = $this->getDeploymentDownload($functionId, $deploymentId, 'output'); + $this->assertEquals(200, $output['headers']['status-code']); + $this->assertStringStartsWith("\x1f\x8b", $output['body']); + + $source = $this->getDeploymentDownload($functionId, $deploymentId, 'source'); + $this->assertEquals(200, $source['headers']['status-code']); + $this->assertStringStartsWith("\x1f\x8b", $source['body']); + + $this->cleanupFunction($functionId); + } + #[Retry(count: 3)] public function testCancelDeploymentBuild(): void { From 771744fa2fb0becf4180aa07d62dd7a4b42d7e27 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 2 Sep 2026 13:49:49 +0100 Subject: [PATCH 5/6] test: compare the build download to buildSize instead of assuming a gzip artifact --- tests/e2e/Services/Functions/FunctionsCustomServerTest.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 5102b241c6..742997eda8 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -1074,10 +1074,11 @@ final class FunctionsCustomServerTest extends Scope $this->assertEquals(200, $execution['body']['responseStatusCode']); $this->assertSame($deploymentId, \json_decode($execution['body']['responseBody'], true)['APPWRITE_FUNCTION_DEPLOYMENT']); - // The download endpoint reads buildPath through the builds device. + // The download endpoint reads buildPath through the builds device. The + // artifact format depends on the storage strategy, so compare sizes. $output = $this->getDeploymentDownload($functionId, $deploymentId, 'output'); $this->assertEquals(200, $output['headers']['status-code']); - $this->assertStringStartsWith("\x1f\x8b", $output['body']); + $this->assertSame($deployment['body']['buildSize'], \strlen($output['body'])); $source = $this->getDeploymentDownload($functionId, $deploymentId, 'source'); $this->assertEquals(200, $source['headers']['status-code']); From 4a8815487d9525caeaa42e88617b63fe7d4bc1fb Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 2 Sep 2026 16:26:36 +0100 Subject: [PATCH 6/6] docs: keep the store() docblock generic --- src/Appwrite/Deployment/Deployments.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Appwrite/Deployment/Deployments.php b/src/Appwrite/Deployment/Deployments.php index f1f6a280bb..16359424bb 100644 --- a/src/Appwrite/Deployment/Deployments.php +++ b/src/Appwrite/Deployment/Deployments.php @@ -468,8 +468,7 @@ readonly class Deployments * (see storage()) build.sh wrote it onto the builds volume, which on the * local device already is the buildPath; on a remote device (S3 and * friends) it is moved off the volume onto the device here. A strategy - * whose artifacts already land on the device — Cloud uploads them from - * the sidecar — overrides this with a no-op. + * whose artifacts already land on the device overrides this with a no-op. */ public function store(Document $deployment): void {