diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a56b7fc26..d550cbde3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -593,6 +593,14 @@ jobs: storage_antivirus: enabled compose_profiles: antivirus wait: clamav + - label: S3 Storage + group: s3 + path: /usr/src/code/tests/e2e/Services + options_abuse: disabled + storage_antivirus: disabled + compose_profiles: s3 + wait: executor + storage_device: s3 env: # Shared tables mode connects every pool at boot, including the MongoDB-only DocumentsDB. COMPOSE_PROFILES: ${{ matrix.mode == 'shared' && format('mongodb,{0}', matrix.config.compose_profiles) || matrix.config.compose_profiles }} @@ -637,6 +645,14 @@ jobs: env: _APP_OPTIONS_ABUSE: ${{ matrix.config.options_abuse }} _APP_STORAGE_ANTIVIRUS: ${{ matrix.config.storage_antivirus }} + # The s3 config points every storage device and the executor at the MinIO profile service. + _APP_STORAGE_DEVICE: ${{ matrix.config.storage_device || 'local' }} + _APP_STORAGE_S3_ACCESS_KEY: minio + _APP_STORAGE_S3_SECRET: minio12345 + _APP_STORAGE_S3_BUCKET: appwrite + _APP_STORAGE_S3_REGION: us-east-1 + _APP_STORAGE_S3_ENDPOINT: http://minio:9000 + _APP_EXECUTOR_CONNECTION_STORAGE: ${{ matrix.config.storage_device == 's3' && 's3://minio:minio12345@localhost?region=us-east-1&url=http%3A%2F%2Fminio%3A9000' || 'local://localhost' }} _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} _APP_DATABASE_DOCUMENTSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'documentsdb_db_main' || '' }} _APP_DATABASE_VECTORSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'vectorsdb_db_main' || '' }} @@ -696,6 +712,10 @@ jobs: echo "=== ClamAV Logs ===" docker compose logs --timestamps clamav || true fi + if [ "${{ matrix.config.storage_device }}" = "s3" ]; then + echo "=== MinIO Logs ===" + docker compose logs --timestamps minio || true + fi benchmark: name: Benchmark diff --git a/app/config/variables.php b/app/config/variables.php index 4b69212e75..0776c4f7ba 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -1251,7 +1251,7 @@ return [ ], [ 'name' => '_APP_EXECUTOR_CONNECTION_STORAGE', - 'description' => "DSN for Open Runtimes executor storage. When `_APP_STORAGE_DEVICE` is not local, point this at the same backend so the executor can read deployment artifacts. Defaults to `local://localhost`.\n\nExamples:\n- Local: `local://localhost`\n- AWS S3: `s3://ACCESS_KEY:SECRET@BUCKET.s3.REGION.amazonaws.com?region=REGION`\n- S3-compatible: `s3://ACCESS_KEY:SECRET@localhost/BUCKET?region=REGION&url=http%3A%2F%2Fminio%3A9000`", + 'description' => "DSN for Open Runtimes executor storage. When `_APP_STORAGE_DEVICE` is not local, point this at the same backend so the executor can read deployment artifacts. Defaults to `local://localhost`.\n\nExamples:\n- Local: `local://localhost`\n- AWS S3: `s3://ACCESS_KEY:SECRET@BUCKET.s3.REGION.amazonaws.com?region=REGION`\n- S3-compatible (`_APP_STORAGE_S3_ENDPOINT` set): `s3://ACCESS_KEY:SECRET@localhost?region=REGION&url=http%3A%2F%2Fminio%3A9000` — leave the bucket out of the DSN path; Appwrite keys objects under `BUCKET/` already, so the executor must not add it to the endpoint", 'introduction' => '1.9.5', 'default' => 'local://localhost', 'required' => false, diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 8aa7f7cf70..ce5d6f6653 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -213,9 +213,15 @@ return function (Container $context): void { ), ['publisher']); // Builds a Deployments bound to a given project — webhook handlers resolve // their tenant projects mid-request, after this container is initialized. - $context->set('deploymentsFactory', function (Jobs $jobs, array $platform) { - return fn (Database $dbForProject, Document $project): Deployments => new Deployments($jobs, $dbForProject, $project, $platform); - }, ['jobs', 'platform']); + $context->set('deploymentsFactory', function (Jobs $jobs, array $platform, Telemetry $telemetry) { + return fn (Database $dbForProject, Document $project): Deployments => new Deployments( + $jobs, + $dbForProject, + $project, + $platform, + new Device\Telemetry($telemetry, getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId())), + ); + }, ['jobs', 'platform', 'telemetry']); $context->set('deployments', fn (callable $deploymentsFactory, Database $dbForProject, Document $project) => $deploymentsFactory($dbForProject, $project), ['deploymentsFactory', 'dbForProject', 'project']); $context->set('eventProcessor', fn () => new EventProcessor(), []); $context->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory( diff --git a/app/init/worker/message.php b/app/init/worker/message.php index 6f6fc39eef..4ba9d415da 100644 --- a/app/init/worker/message.php +++ b/app/init/worker/message.php @@ -24,6 +24,7 @@ use Utopia\Queue\Publisher; use Utopia\Queue\Queue; use Utopia\Registry\Registry; use Utopia\Span\Span; +use Utopia\Storage\Device; use Utopia\Storage\Device\Telemetry as TelemetryDevice; use Utopia\System\System; use Utopia\Telemetry\Adapter as Telemetry; @@ -190,9 +191,9 @@ return function (Container $container): void { // Only the Builds worker uses this, handing template-into-repo pushes to // the jobs-service. - $container->set('deployments', function (Jobs $jobs, Database $dbForProject, Document $project, array $platform) { - return new Deployments($jobs, $dbForProject, $project, $platform); - }, ['jobs', 'dbForProject', 'project', 'platform']); + $container->set('deployments', function (Jobs $jobs, Database $dbForProject, Document $project, array $platform, Device $deviceForBuilds) { + return new Deployments($jobs, $dbForProject, $project, $platform, $deviceForBuilds); + }, ['jobs', 'dbForProject', 'project', 'platform', 'deviceForBuilds']); $container->set('logError', function (Registry $register, Document $project) { return function (Throwable $error, string $namespace, string $action, ?array $extras = null) use ($register, $project) { diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 99dea6460b..5245a1c333 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -560,6 +560,35 @@ services: restart: unless-stopped networks: - appwrite + # S3-compatible storage for the s3 e2e group. Creates the bucket the + # _APP_STORAGE_S3_* variables point at once the server is up. + minio: + profiles: + - s3 + image: minio/minio:RELEASE.2025-09-07T16-13-09Z + container_name: appwrite-minio + restart: unless-stopped + networks: + - appwrite + environment: + - MINIO_ROOT_USER=${_APP_STORAGE_S3_ACCESS_KEY:-minio} + - MINIO_ROOT_PASSWORD=${_APP_STORAGE_S3_SECRET:-minio12345} + - MINIO_BUCKET=${_APP_STORAGE_S3_BUCKET:-appwrite} + entrypoint: + - sh + - -c + - | + minio server /data & + until mc alias set local http://localhost:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do sleep 1; done + mc mb -p "local/$$MINIO_BUCKET" + wait + healthcheck: + test: + - CMD-SHELL + - mc ls "local/$$MINIO_BUCKET" >/dev/null + interval: 5s + timeout: 5s + retries: 20 configs: redisinsight-connections.json: content: | diff --git a/docker-compose.yml b/docker-compose.yml index 90c3538ea5..74414264a5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -906,6 +906,28 @@ services: - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 + - _APP_STORAGE_DEVICE + - _APP_STORAGE_S3_ACCESS_KEY + - _APP_STORAGE_S3_SECRET + - _APP_STORAGE_S3_REGION + - _APP_STORAGE_S3_BUCKET + - _APP_STORAGE_S3_ENDPOINT + - _APP_STORAGE_DO_SPACES_ACCESS_KEY + - _APP_STORAGE_DO_SPACES_SECRET + - _APP_STORAGE_DO_SPACES_REGION + - _APP_STORAGE_DO_SPACES_BUCKET + - _APP_STORAGE_BACKBLAZE_ACCESS_KEY + - _APP_STORAGE_BACKBLAZE_SECRET + - _APP_STORAGE_BACKBLAZE_REGION + - _APP_STORAGE_BACKBLAZE_BUCKET + - _APP_STORAGE_LINODE_ACCESS_KEY + - _APP_STORAGE_LINODE_SECRET + - _APP_STORAGE_LINODE_REGION + - _APP_STORAGE_LINODE_BUCKET + - _APP_STORAGE_WASABI_ACCESS_KEY + - _APP_STORAGE_WASABI_SECRET + - _APP_STORAGE_WASABI_REGION + - _APP_STORAGE_WASABI_BUCKET - _APP_JOBS_HOST - _APP_JOBS_SECRET - _APP_REDIS_HOST diff --git a/src/Appwrite/Deployment/Deployments.php b/src/Appwrite/Deployment/Deployments.php index 000e7de5d3..16359424bb 100644 --- a/src/Appwrite/Deployment/Deployments.php +++ b/src/Appwrite/Deployment/Deployments.php @@ -22,6 +22,9 @@ use Utopia\Database\Document; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; +use Utopia\Storage\Device; +use Utopia\Storage\Device\Local; +use Utopia\Storage\DeviceType; use Utopia\System\System; use Utopia\VCS\Adapter\Git; @@ -53,6 +56,7 @@ readonly class Deployments protected Database $dbForProject, protected Document $project, private array $platform, + private Device $deviceForBuilds, ) { } @@ -182,7 +186,7 @@ readonly class Deployments $queued = $this->dbForProject->updateDocuments('deployments', new Document([ 'status' => 'waiting', - 'buildPath' => static::buildPath($this->project->getId(), $deployment->getId()), + 'buildPath' => $this->deviceForBuilds->getPath($deployment->getId() . '/' . static::artifact()), ]), [ Query::equal('$id', [$deployment->getId()]), Query::notEqual('status', 'canceled'), @@ -458,6 +462,32 @@ readonly class Deployments ]; } + /** + * Bring a finished build's artifact to the deployment's buildPath, once + * the job has exited and delivered its artifacts. With the volume strategy + * (see storage()) build.sh wrote it onto the builds volume, which on the + * local device already is the buildPath; on a remote device (S3 and + * friends) it is moved off the volume onto the device here. A strategy + * whose artifacts already land on the device overrides this with a no-op. + */ + public function store(Document $deployment): void + { + if ($this->deviceForBuilds->getType() === DeviceType::Local) { + return; + } + + $source = static::buildPath($this->project->getId(), $deployment->getId()); + $target = (string) $deployment->getAttribute('buildPath', ''); + $local = new Local(); + if ($target === '' || ! $local->exists($source)) { + return; + } + + if ($local->copy($source, $target, $this->deviceForBuilds)) { + $local->delete($source); + } + } + /** * The jobs-service job id for a deployment build (used to submit and cancel). */ @@ -476,7 +506,9 @@ readonly class Deployments } /** - * The build output path on the builds volume, declared at submission. + * The build output path on the builds volume. On the local device this is + * also the deployment's buildPath; on a remote device (S3 and friends) the + * buildPath is a device path and the Jobs worker moves the artifact there. */ public static function buildPath(string $projectId, string $deploymentId): string { diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index 4b9c32533d..d5782bb1e9 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Functions\Workers; use Appwrite\Bus\Events\RuleUpdated; +use Appwrite\Deployment\Deployments; use Appwrite\Deployment\Detection; use Appwrite\Deployment\GitAction; use Appwrite\Event\Event; @@ -87,6 +88,7 @@ class Jobs extends Action ->inject('platform') ->inject('plan') ->inject('bus') + ->inject('deployments') ->callback($this->action(...)); } @@ -109,6 +111,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): void { $event = JobsMessage::fromArray($message->getPayload()); @@ -117,7 +120,7 @@ class Jobs extends Action return; } - $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForScreenshots, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $deploymentId, $bus): void { + $locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForScreenshots, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $deploymentId, $bus, $deployments): void { if ($event->id !== '') { $key = 'jobs-event-' . $event->id; if ($cache->load($key, self::DEDUPE_TTL) !== false) { @@ -135,10 +138,10 @@ class Jobs extends Action $deployment = match ($event->event) { 'orchestrator.job.log' => $this->onLog($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $vcsFactory, $platform), - 'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), - 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), - 'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), - default => $this->onCallback($event->event, $dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus), + 'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), + 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), + 'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), + default => $this->onCallback($event->event, $dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments), }; // Console realtime on every callback (log stream + status). @@ -226,6 +229,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { return $deployment; } @@ -275,6 +279,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { if (($data['artifactId'] ?? '') === 'manifest') { // A failed manifest degrades to an empty listing (detection @@ -283,7 +288,7 @@ class Jobs extends Action $files = \is_array($manifest) ? (array) ($manifest['files'] ?? []) : []; $cache->save('jobs-manifest-' . $deployment->getId(), ['files' => \array_values($files)]); - return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus); + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments); } if (($data['artifactId'] ?? '') !== 'sourceSize' || ($data['status'] ?? '') !== 'success') { @@ -324,6 +329,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { if ($exitCode !== 0) { return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus); @@ -331,7 +337,7 @@ class Jobs extends Action $cache->save('jobs-exit-' . $deployment->getId(), true); - return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus); + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments); } /** @@ -352,10 +358,11 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { $cache->save('jobs-complete-' . $deployment->getId(), true); - return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus); + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments); } /** @@ -376,6 +383,7 @@ class Jobs extends Action array $platform, array $plan, Bus $bus, + Deployments $deployments, ): Document { if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'], true)) { return $deployment; // already finalized @@ -400,6 +408,10 @@ class Jobs extends Action } } + // The build job wrote its artifact where the storage strategy put it; + // bring it to buildPath before checking for it. + $deployments->store($deployment); + $path = (string) $deployment->getAttribute('buildPath', ''); if ($path === '' || ! $deviceForBuilds->exists($path)) { return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build produced no output artifact.', $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus); diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 84b3269e21..742997eda8 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -7,6 +7,7 @@ namespace Tests\E2E\Services\Functions; use Appwrite\Platform\Modules\Compute\Specification; use Appwrite\Tests\Retry; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\Group; use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; @@ -1030,6 +1031,62 @@ final class FunctionsCustomServerTest extends Scope $this->assertEquals(204, $deployment['headers']['status-code']); } + /** + * The build job writes onto the builds volume while the executor and the + * download endpoint read the deployment through the configured storage + * device. Runs in the `s3` CI group against MinIO, where the artifact has + * to be moved off the volume and keyed under the bucket; on the default + * local device the two are the same path. + */ + #[Group('s3')] + public function testDeploymentBuildOutputIsServedFromTheBuildsDevice(): void + { + $functionId = $this->setupFunction([ + 'functionId' => ID::unique(), + 'name' => 'Build output', + 'runtime' => 'node-22', + 'entrypoint' => 'index.js', + 'execute' => [Role::any()->toString()], + ]); + + $deployment = $this->createDeployment($functionId, [ + 'code' => $this->packageFunction('basic'), + 'activate' => true, + ]); + $this->assertEquals(202, $deployment['headers']['status-code']); + $deploymentId = $deployment['body']['$id']; + + $this->assertEventually(function () use ($functionId, $deploymentId) { + $deployment = $this->getDeployment($functionId, $deploymentId); + $this->assertEquals('ready', $deployment['body']['status'], $deployment['body']['buildLogs'] ?? ''); + }, 100000, 500); + + /** + * Test for SUCCESS + */ + $deployment = $this->getDeployment($functionId, $deploymentId); + $this->assertGreaterThan(0, $deployment['body']['buildSize']); + + // The executor fetches the artifact through its own storage connection. + $execution = $this->createExecution($functionId); + $this->assertEquals(201, $execution['headers']['status-code']); + $this->assertEquals('completed', $execution['body']['status'], $execution['body']['errors'] ?? ''); + $this->assertEquals(200, $execution['body']['responseStatusCode']); + $this->assertSame($deploymentId, \json_decode($execution['body']['responseBody'], true)['APPWRITE_FUNCTION_DEPLOYMENT']); + + // The download endpoint reads buildPath through the builds device. The + // artifact format depends on the storage strategy, so compare sizes. + $output = $this->getDeploymentDownload($functionId, $deploymentId, 'output'); + $this->assertEquals(200, $output['headers']['status-code']); + $this->assertSame($deployment['body']['buildSize'], \strlen($output['body'])); + + $source = $this->getDeploymentDownload($functionId, $deploymentId, 'source'); + $this->assertEquals(200, $source['headers']['status-code']); + $this->assertStringStartsWith("\x1f\x8b", $source['body']); + + $this->cleanupFunction($functionId); + } + #[Retry(count: 3)] public function testCancelDeploymentBuild(): void {