diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a56b7fc26..d550cbde3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -593,6 +593,14 @@ jobs: storage_antivirus: enabled compose_profiles: antivirus wait: clamav + - label: S3 Storage + group: s3 + path: /usr/src/code/tests/e2e/Services + options_abuse: disabled + storage_antivirus: disabled + compose_profiles: s3 + wait: executor + storage_device: s3 env: # Shared tables mode connects every pool at boot, including the MongoDB-only DocumentsDB. COMPOSE_PROFILES: ${{ matrix.mode == 'shared' && format('mongodb,{0}', matrix.config.compose_profiles) || matrix.config.compose_profiles }} @@ -637,6 +645,14 @@ jobs: env: _APP_OPTIONS_ABUSE: ${{ matrix.config.options_abuse }} _APP_STORAGE_ANTIVIRUS: ${{ matrix.config.storage_antivirus }} + # The s3 config points every storage device and the executor at the MinIO profile service. + _APP_STORAGE_DEVICE: ${{ matrix.config.storage_device || 'local' }} + _APP_STORAGE_S3_ACCESS_KEY: minio + _APP_STORAGE_S3_SECRET: minio12345 + _APP_STORAGE_S3_BUCKET: appwrite + _APP_STORAGE_S3_REGION: us-east-1 + _APP_STORAGE_S3_ENDPOINT: http://minio:9000 + _APP_EXECUTOR_CONNECTION_STORAGE: ${{ matrix.config.storage_device == 's3' && 's3://minio:minio12345@localhost?region=us-east-1&url=http%3A%2F%2Fminio%3A9000' || 'local://localhost' }} _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} _APP_DATABASE_DOCUMENTSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'documentsdb_db_main' || '' }} _APP_DATABASE_VECTORSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'vectorsdb_db_main' || '' }} @@ -696,6 +712,10 @@ jobs: echo "=== ClamAV Logs ===" docker compose logs --timestamps clamav || true fi + if [ "${{ matrix.config.storage_device }}" = "s3" ]; then + echo "=== MinIO Logs ===" + docker compose logs --timestamps minio || true + fi benchmark: name: Benchmark diff --git a/app/config/variables.php b/app/config/variables.php index e21b086f92..2086d4110c 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -1242,7 +1242,7 @@ return [ ], [ 'name' => '_APP_EXECUTOR_CONNECTION_STORAGE', - 'description' => "DSN for Open Runtimes executor storage. When `_APP_STORAGE_DEVICE` is not local, point this at the same backend so the executor can read deployment artifacts. Defaults to `local://localhost`.\n\nExamples:\n- Local: `local://localhost`\n- AWS S3: `s3://ACCESS_KEY:SECRET@BUCKET.s3.REGION.amazonaws.com?region=REGION`\n- S3-compatible: `s3://ACCESS_KEY:SECRET@localhost/BUCKET?region=REGION&url=http%3A%2F%2Fminio%3A9000`", + 'description' => "DSN for Open Runtimes executor storage. When `_APP_STORAGE_DEVICE` is not local, point this at the same backend so the executor can read deployment artifacts. Defaults to `local://localhost`.\n\nExamples:\n- Local: `local://localhost`\n- AWS S3: `s3://ACCESS_KEY:SECRET@BUCKET.s3.REGION.amazonaws.com?region=REGION`\n- S3-compatible (`_APP_STORAGE_S3_ENDPOINT` set): `s3://ACCESS_KEY:SECRET@localhost?region=REGION&url=http%3A%2F%2Fminio%3A9000` — leave the bucket out of the DSN path; Appwrite keys objects under `BUCKET/` already, so the executor must not add it to the endpoint", 'introduction' => '1.9.5', 'default' => 'local://localhost', 'required' => false, diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 99dea6460b..5245a1c333 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -560,6 +560,35 @@ services: restart: unless-stopped networks: - appwrite + # S3-compatible storage for the s3 e2e group. Creates the bucket the + # _APP_STORAGE_S3_* variables point at once the server is up. + minio: + profiles: + - s3 + image: minio/minio:RELEASE.2025-09-07T16-13-09Z + container_name: appwrite-minio + restart: unless-stopped + networks: + - appwrite + environment: + - MINIO_ROOT_USER=${_APP_STORAGE_S3_ACCESS_KEY:-minio} + - MINIO_ROOT_PASSWORD=${_APP_STORAGE_S3_SECRET:-minio12345} + - MINIO_BUCKET=${_APP_STORAGE_S3_BUCKET:-appwrite} + entrypoint: + - sh + - -c + - | + minio server /data & + until mc alias set local http://localhost:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do sleep 1; done + mc mb -p "local/$$MINIO_BUCKET" + wait + healthcheck: + test: + - CMD-SHELL + - mc ls "local/$$MINIO_BUCKET" >/dev/null + interval: 5s + timeout: 5s + retries: 20 configs: redisinsight-connections.json: content: | diff --git a/src/Appwrite/Deployment/Deployments.php b/src/Appwrite/Deployment/Deployments.php index 978666cdfe..f1f6a280bb 100644 --- a/src/Appwrite/Deployment/Deployments.php +++ b/src/Appwrite/Deployment/Deployments.php @@ -23,6 +23,8 @@ use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Storage\Device; +use Utopia\Storage\Device\Local; +use Utopia\Storage\DeviceType; use Utopia\System\System; use Utopia\VCS\Adapter\Git; @@ -460,6 +462,33 @@ readonly class Deployments ]; } + /** + * Bring a finished build's artifact to the deployment's buildPath, once + * the job has exited and delivered its artifacts. With the volume strategy + * (see storage()) build.sh wrote it onto the builds volume, which on the + * local device already is the buildPath; on a remote device (S3 and + * friends) it is moved off the volume onto the device here. A strategy + * whose artifacts already land on the device — Cloud uploads them from + * the sidecar — overrides this with a no-op. + */ + public function store(Document $deployment): void + { + if ($this->deviceForBuilds->getType() === DeviceType::Local) { + return; + } + + $source = static::buildPath($this->project->getId(), $deployment->getId()); + $target = (string) $deployment->getAttribute('buildPath', ''); + $local = new Local(); + if ($target === '' || ! $local->exists($source)) { + return; + } + + if ($local->copy($source, $target, $this->deviceForBuilds)) { + $local->delete($source); + } + } + /** * The jobs-service job id for a deployment build (used to submit and cancel). */ diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index ab3d43684d..d5782bb1e9 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -28,8 +28,6 @@ use Utopia\Database\Query; use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\Storage\Device; -use Utopia\Storage\Device\Local; -use Utopia\Storage\DeviceType; use Utopia\System\System; /** @@ -90,6 +88,7 @@ class Jobs extends Action ->inject('platform') ->inject('plan') ->inject('bus') + ->inject('deployments') ->callback($this->action(...)); } @@ -112,6 +111,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): void { $event = JobsMessage::fromArray($message->getPayload()); @@ -120,7 +120,7 @@ class Jobs extends Action return; } - $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForScreenshots, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $deploymentId, $bus): void { + $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForScreenshots, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $deploymentId, $bus, $deployments): void { if ($event->id !== '') { $key = 'jobs-event-' . $event->id; if ($cache->load($key, self::DEDUPE_TTL) !== false) { @@ -138,10 +138,10 @@ class Jobs extends Action $deployment = match ($event->event) { 'orchestrator.job.log' => $this->onLog($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $vcsFactory, $platform), - 'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), - 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), - 'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), - default => $this->onCallback($event->event, $dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), + 'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), + 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), + 'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), + default => $this->onCallback($event->event, $dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), }; // Console realtime on every callback (log stream + status). @@ -229,6 +229,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { return $deployment; } @@ -278,6 +279,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { if (($data['artifactId'] ?? '') === 'manifest') { // A failed manifest degrades to an empty listing (detection @@ -286,7 +288,7 @@ class Jobs extends Action $files = \is_array($manifest) ? (array) ($manifest['files'] ?? []) : []; $cache->save('jobs-manifest-' . $deployment->getId(), ['files' => \array_values($files)]); - return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus); + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments); } if (($data['artifactId'] ?? '') !== 'sourceSize' || ($data['status'] ?? '') !== 'success') { @@ -327,6 +329,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { if ($exitCode !== 0) { return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus); @@ -334,7 +337,7 @@ class Jobs extends Action $cache->save('jobs-exit-' . $deployment->getId(), true); - return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus); + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments); } /** @@ -355,10 +358,11 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { $cache->save('jobs-complete-' . $deployment->getId(), true); - return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus); + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments); } /** @@ -379,6 +383,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'], true)) { return $deployment; // already finalized @@ -403,18 +408,11 @@ class Jobs extends Action } } - // The build job wrote its artifact onto the shared builds volume (see - // Deployments::storage()). On a remote device (S3 and friends) the - // executor and the download endpoint read buildPath through that - // device, so move the artifact off the volume onto it first. + // The build job wrote its artifact where the storage strategy put it; + // bring it to buildPath before checking for it. + $deployments->store($deployment); + $path = (string) $deployment->getAttribute('buildPath', ''); - if ($path !== '' && $deviceForBuilds->getType() !== DeviceType::Local) { - $local = new Local(); - $source = Deployments::buildPath($project->getId(), $deploymentId); - if ($local->exists($source) && $local->copy($source, $path, $deviceForBuilds)) { - $local->delete($source); - } - } if ($path === '' || ! $deviceForBuilds->exists($path)) { return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build produced no output artifact.', $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus); } diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 84b3269e21..5102b241c6 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -7,6 +7,7 @@ namespace Tests\E2E\Services\Functions; use Appwrite\Platform\Modules\Compute\Specification; use Appwrite\Tests\Retry; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\Group; use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; @@ -1030,6 +1031,61 @@ final class FunctionsCustomServerTest extends Scope $this->assertEquals(204, $deployment['headers']['status-code']); } + /** + * The build job writes onto the builds volume while the executor and the + * download endpoint read the deployment through the configured storage + * device. Runs in the `s3` CI group against MinIO, where the artifact has + * to be moved off the volume and keyed under the bucket; on the default + * local device the two are the same path. + */ + #[Group('s3')] + public function testDeploymentBuildOutputIsServedFromTheBuildsDevice(): void + { + $functionId = $this->setupFunction([ + 'functionId' => ID::unique(), + 'name' => 'Build output', + 'runtime' => 'node-22', + 'entrypoint' => 'index.js', + 'execute' => [Role::any()->toString()], + ]); + + $deployment = $this->createDeployment($functionId, [ + 'code' => $this->packageFunction('basic'), + 'activate' => true, + ]); + $this->assertEquals(202, $deployment['headers']['status-code']); + $deploymentId = $deployment['body']['$id']; + + $this->assertEventually(function () use ($functionId, $deploymentId) { + $deployment = $this->getDeployment($functionId, $deploymentId); + $this->assertEquals('ready', $deployment['body']['status'], $deployment['body']['buildLogs'] ?? ''); + }, 100000, 500); + + /** + * Test for SUCCESS + */ + $deployment = $this->getDeployment($functionId, $deploymentId); + $this->assertGreaterThan(0, $deployment['body']['buildSize']); + + // The executor fetches the artifact through its own storage connection. + $execution = $this->createExecution($functionId); + $this->assertEquals(201, $execution['headers']['status-code']); + $this->assertEquals('completed', $execution['body']['status'], $execution['body']['errors'] ?? ''); + $this->assertEquals(200, $execution['body']['responseStatusCode']); + $this->assertSame($deploymentId, \json_decode($execution['body']['responseBody'], true)['APPWRITE_FUNCTION_DEPLOYMENT']); + + // The download endpoint reads buildPath through the builds device. + $output = $this->getDeploymentDownload($functionId, $deploymentId, 'output'); + $this->assertEquals(200, $output['headers']['status-code']); + $this->assertStringStartsWith("\x1f\x8b", $output['body']); + + $source = $this->getDeploymentDownload($functionId, $deploymentId, 'source'); + $this->assertEquals(200, $source['headers']['status-code']); + $this->assertStringStartsWith("\x1f\x8b", $source['body']); + + $this->cleanupFunction($functionId); + } + #[Retry(count: 3)] public function testCancelDeploymentBuild(): void {