fix: let the storage strategy bring build output to buildPath, and cover S3 in CI

Moving the artifact off the builds volume is part of the volume storage strategy, not of the Jobs worker: Deployments::store() now does it, next to storage() which decides where build.sh writes. A strategy whose artifacts already land on the device (Cloud uploads them from the sidecar) overrides it with a no-op instead of relying on the volume file being absent.

The executor's storage DSN example carried the bucket in its path. Executor 0.29 appends that bucket to the endpoint and treats keys as bucket-relative, while Appwrite keys objects under the bucket already, so functions could not start ("File not found"). The example now leaves the bucket out for S3-compatible endpoints.

CI gains an "S3 Storage" e2e config: a MinIO service behind the s3 compose profile, every storage device and the executor pointed at it, and a Functions test that deploys, executes and downloads a build through the configured device. The test fails on main (the deployment never becomes ready) and passes with the fix; it also runs in the regular Functions suite against the local device.
This commit is contained in:
Chirag Aggarwal committed 2026-09-02 13:46:41 +01:00
1 parent 09f80fdffe
commit 98dd8d98f3
6 files changed
+154 -22

No files matched your search

+20
View File
@@ -593,6 +593,14 @@ jobs:
storage_antivirus: enabled
compose_profiles: antivirus
wait: clamav
- label: S3 Storage
group: s3
path: /usr/src/code/tests/e2e/Services
options_abuse: disabled
storage_antivirus: disabled
compose_profiles: s3
wait: executor
storage_device: s3
env:
# Shared tables mode connects every pool at boot, including the MongoDB-only DocumentsDB.
COMPOSE_PROFILES: ${{ matrix.mode == 'shared' && format('mongodb,{0}', matrix.config.compose_profiles) || matrix.config.compose_profiles }}
@@ -637,6 +645,14 @@ jobs:
env:
_APP_OPTIONS_ABUSE: ${{ matrix.config.options_abuse }}
_APP_STORAGE_ANTIVIRUS: ${{ matrix.config.storage_antivirus }}
# The s3 config points every storage device and the executor at the MinIO profile service.
_APP_STORAGE_DEVICE: ${{ matrix.config.storage_device || 'local' }}
_APP_STORAGE_S3_ACCESS_KEY: minio
_APP_STORAGE_S3_SECRET: minio12345
_APP_STORAGE_S3_BUCKET: appwrite
_APP_STORAGE_S3_REGION: us-east-1
_APP_STORAGE_S3_ENDPOINT: http://minio:9000
_APP_EXECUTOR_CONNECTION_STORAGE: ${{ matrix.config.storage_device == 's3' && 's3://minio:minio12345@localhost?region=us-east-1&url=http%3A%2F%2Fminio%3A9000' || 'local://localhost' }}
_APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }}
_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'documentsdb_db_main' || '' }}
_APP_DATABASE_VECTORSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'vectorsdb_db_main' || '' }}
@@ -696,6 +712,10 @@ jobs:
echo "=== ClamAV Logs ==="
docker compose logs --timestamps clamav || true
fi
if [ "${{ matrix.config.storage_device }}" = "s3" ]; then
echo "=== MinIO Logs ==="
docker compose logs --timestamps minio || true
fi
benchmark:
name: Benchmark
+1 -1
View File
@@ -1242,7 +1242,7 @@ return [
],
[
'name' => '_APP_EXECUTOR_CONNECTION_STORAGE',
'description' => "DSN for Open Runtimes executor storage. When `_APP_STORAGE_DEVICE` is not local, point this at the same backend so the executor can read deployment artifacts. Defaults to `local://localhost`.\n\nExamples:\n- Local: `local://localhost`\n- AWS S3: `s3://ACCESS_KEY:SECRET@BUCKET.s3.REGION.amazonaws.com?region=REGION`\n- S3-compatible: `s3://ACCESS_KEY:SECRET@localhost/BUCKET?region=REGION&url=http%3A%2F%2Fminio%3A9000`",
'description' => "DSN for Open Runtimes executor storage. When `_APP_STORAGE_DEVICE` is not local, point this at the same backend so the executor can read deployment artifacts. Defaults to `local://localhost`.\n\nExamples:\n- Local: `local://localhost`\n- AWS S3: `s3://ACCESS_KEY:SECRET@BUCKET.s3.REGION.amazonaws.com?region=REGION`\n- S3-compatible (`_APP_STORAGE_S3_ENDPOINT` set): `s3://ACCESS_KEY:SECRET@localhost?region=REGION&url=http%3A%2F%2Fminio%3A9000` — leave the bucket out of the DSN path; Appwrite keys objects under `BUCKET/` already, so the executor must not add it to the endpoint",
'introduction' => '1.9.5',
'default' => 'local://localhost',
'required' => false,
+29
View File
@@ -560,6 +560,35 @@ services:
restart: unless-stopped
networks:
- appwrite
# S3-compatible storage for the s3 e2e group. Creates the bucket the
# _APP_STORAGE_S3_* variables point at once the server is up.
minio:
profiles:
- s3
image: minio/minio:RELEASE.2025-09-07T16-13-09Z
container_name: appwrite-minio
restart: unless-stopped
networks:
- appwrite
environment:
- MINIO_ROOT_USER=${_APP_STORAGE_S3_ACCESS_KEY:-minio}
- MINIO_ROOT_PASSWORD=${_APP_STORAGE_S3_SECRET:-minio12345}
- MINIO_BUCKET=${_APP_STORAGE_S3_BUCKET:-appwrite}
entrypoint:
- sh
- -c
- |
minio server /data &
until mc alias set local http://localhost:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do sleep 1; done
mc mb -p "local/$$MINIO_BUCKET"
wait
healthcheck:
test:
- CMD-SHELL
- mc ls "local/$$MINIO_BUCKET" >/dev/null
interval: 5s
timeout: 5s
retries: 20
configs:
redisinsight-connections.json:
content: |
+29
View File
@@ -23,6 +23,8 @@ use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Query;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
use Utopia\Storage\DeviceType;
use Utopia\System\System;
use Utopia\VCS\Adapter\Git;
@@ -460,6 +462,33 @@ readonly class Deployments
];
}
/**
* Bring a finished build's artifact to the deployment's buildPath, once
* the job has exited and delivered its artifacts. With the volume strategy
* (see storage()) build.sh wrote it onto the builds volume, which on the
* local device already is the buildPath; on a remote device (S3 and
* friends) it is moved off the volume onto the device here. A strategy
* whose artifacts already land on the device — Cloud uploads them from
* the sidecar — overrides this with a no-op.
*/
public function store(Document $deployment): void
{
if ($this->deviceForBuilds->getType() === DeviceType::Local) {
return;
}
$source = static::buildPath($this->project->getId(), $deployment->getId());
$target = (string) $deployment->getAttribute('buildPath', '');
$local = new Local();
if ($target === '' || ! $local->exists($source)) {
return;
}
if ($local->copy($source, $target, $this->deviceForBuilds)) {
$local->delete($source);
}
}
/**
* The jobs-service job id for a deployment build (used to submit and cancel).
*/
@@ -28,8 +28,6 @@ use Utopia\Database\Query;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
use Utopia\Storage\DeviceType;
use Utopia\System\System;
/**
@@ -90,6 +88,7 @@ class Jobs extends Action
->inject('platform')
->inject('plan')
->inject('bus')
->inject('deployments')
->callback($this->action(...));
}
@@ -112,6 +111,7 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): void {
$event = JobsMessage::fromArray($message->getPayload());
@@ -120,7 +120,7 @@ class Jobs extends Action
return;
}
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForScreenshots, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $deploymentId, $bus): void {
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForScreenshots, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $deploymentId, $bus, $deployments): void {
if ($event->id !== '') {
$key = 'jobs-event-' . $event->id;
if ($cache->load($key, self::DEDUPE_TTL) !== false) {
@@ -138,10 +138,10 @@ class Jobs extends Action
$deployment = match ($event->event) {
'orchestrator.job.log' => $this->onLog($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $vcsFactory, $platform),
'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus),
'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus),
'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus),
default => $this->onCallback($event->event, $dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus),
'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments),
'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments),
'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments),
default => $this->onCallback($event->event, $dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments),
};
// Console realtime on every callback (log stream + status).
@@ -229,6 +229,7 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): Document {
return $deployment;
}
@@ -278,6 +279,7 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): Document {
if (($data['artifactId'] ?? '') === 'manifest') {
// A failed manifest degrades to an empty listing (detection
@@ -286,7 +288,7 @@ class Jobs extends Action
$files = \is_array($manifest) ? (array) ($manifest['files'] ?? []) : [];
$cache->save('jobs-manifest-' . $deployment->getId(), ['files' => \array_values($files)]);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments);
}
if (($data['artifactId'] ?? '') !== 'sourceSize' || ($data['status'] ?? '') !== 'success') {
@@ -327,6 +329,7 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): Document {
if ($exitCode !== 0) {
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus);
@@ -334,7 +337,7 @@ class Jobs extends Action
$cache->save('jobs-exit-' . $deployment->getId(), true);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments);
}
/**
@@ -355,10 +358,11 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): Document {
$cache->save('jobs-complete-' . $deployment->getId(), true);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments);
}
/**
@@ -379,6 +383,7 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): Document {
if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'], true)) {
return $deployment; // already finalized
@@ -403,18 +408,11 @@ class Jobs extends Action
}
}
// The build job wrote its artifact onto the shared builds volume (see
// Deployments::storage()). On a remote device (S3 and friends) the
// executor and the download endpoint read buildPath through that
// device, so move the artifact off the volume onto it first.
// The build job wrote its artifact where the storage strategy put it;
// bring it to buildPath before checking for it.
$deployments->store($deployment);
$path = (string) $deployment->getAttribute('buildPath', '');
if ($path !== '' && $deviceForBuilds->getType() !== DeviceType::Local) {
$local = new Local();
$source = Deployments::buildPath($project->getId(), $deploymentId);
if ($local->exists($source) && $local->copy($source, $path, $deviceForBuilds)) {
$local->delete($source);
}
}
if ($path === '' || ! $deviceForBuilds->exists($path)) {
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build produced no output artifact.', $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus);
}
@@ -7,6 +7,7 @@ namespace Tests\E2E\Services\Functions;
use Appwrite\Platform\Modules\Compute\Specification;
use Appwrite\Tests\Retry;
use PHPUnit\Framework\Attributes\DataProvider;
use PHPUnit\Framework\Attributes\Group;
use Tests\E2E\Client;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\Scope;
@@ -1030,6 +1031,61 @@ final class FunctionsCustomServerTest extends Scope
$this->assertEquals(204, $deployment['headers']['status-code']);
}
/**
* The build job writes onto the builds volume while the executor and the
* download endpoint read the deployment through the configured storage
* device. Runs in the `s3` CI group against MinIO, where the artifact has
* to be moved off the volume and keyed under the bucket; on the default
* local device the two are the same path.
*/
#[Group('s3')]
public function testDeploymentBuildOutputIsServedFromTheBuildsDevice(): void
{
$functionId = $this->setupFunction([
'functionId' => ID::unique(),
'name' => 'Build output',
'runtime' => 'node-22',
'entrypoint' => 'index.js',
'execute' => [Role::any()->toString()],
]);
$deployment = $this->createDeployment($functionId, [
'code' => $this->packageFunction('basic'),
'activate' => true,
]);
$this->assertEquals(202, $deployment['headers']['status-code']);
$deploymentId = $deployment['body']['$id'];
$this->assertEventually(function () use ($functionId, $deploymentId) {
$deployment = $this->getDeployment($functionId, $deploymentId);
$this->assertEquals('ready', $deployment['body']['status'], $deployment['body']['buildLogs'] ?? '');
}, 100000, 500);
/**
* Test for SUCCESS
*/
$deployment = $this->getDeployment($functionId, $deploymentId);
$this->assertGreaterThan(0, $deployment['body']['buildSize']);
// The executor fetches the artifact through its own storage connection.
$execution = $this->createExecution($functionId);
$this->assertEquals(201, $execution['headers']['status-code']);
$this->assertEquals('completed', $execution['body']['status'], $execution['body']['errors'] ?? '');
$this->assertEquals(200, $execution['body']['responseStatusCode']);
$this->assertSame($deploymentId, \json_decode($execution['body']['responseBody'], true)['APPWRITE_FUNCTION_DEPLOYMENT']);
// The download endpoint reads buildPath through the builds device.
$output = $this->getDeploymentDownload($functionId, $deploymentId, 'output');
$this->assertEquals(200, $output['headers']['status-code']);
$this->assertStringStartsWith("\x1f\x8b", $output['body']);
$source = $this->getDeploymentDownload($functionId, $deploymentId, 'source');
$this->assertEquals(200, $source['headers']['status-code']);
$this->assertStringStartsWith("\x1f\x8b", $source['body']);
$this->cleanupFunction($functionId);
}
#[Retry(count: 3)]
public function testCancelDeploymentBuild(): void
{