Merge pull request #13443 from appwrite/fix/s3-build-output

fix: publish build output to the builds device when storage is not local
This commit is contained in:
Chirag Aggarwal authored and GitHub committed 2026-09-03 02:07:59 -07:00
commit 2c4b617b00
9 files changed
+196 -17

No files matched your search

+20
View File
@@ -593,6 +593,14 @@ jobs:
storage_antivirus: enabled
compose_profiles: antivirus
wait: clamav
- label: S3 Storage
group: s3
path: /usr/src/code/tests/e2e/Services
options_abuse: disabled
storage_antivirus: disabled
compose_profiles: s3
wait: executor
storage_device: s3
env:
# Shared tables mode connects every pool at boot, including the MongoDB-only DocumentsDB.
COMPOSE_PROFILES: ${{ matrix.mode == 'shared' && format('mongodb,{0}', matrix.config.compose_profiles) || matrix.config.compose_profiles }}
@@ -637,6 +645,14 @@ jobs:
env:
_APP_OPTIONS_ABUSE: ${{ matrix.config.options_abuse }}
_APP_STORAGE_ANTIVIRUS: ${{ matrix.config.storage_antivirus }}
# The s3 config points every storage device and the executor at the MinIO profile service.
_APP_STORAGE_DEVICE: ${{ matrix.config.storage_device || 'local' }}
_APP_STORAGE_S3_ACCESS_KEY: minio
_APP_STORAGE_S3_SECRET: minio12345
_APP_STORAGE_S3_BUCKET: appwrite
_APP_STORAGE_S3_REGION: us-east-1
_APP_STORAGE_S3_ENDPOINT: http://minio:9000
_APP_EXECUTOR_CONNECTION_STORAGE: ${{ matrix.config.storage_device == 's3' && 's3://minio:minio12345@localhost?region=us-east-1&url=http%3A%2F%2Fminio%3A9000' || 'local://localhost' }}
_APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }}
_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'documentsdb_db_main' || '' }}
_APP_DATABASE_VECTORSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'vectorsdb_db_main' || '' }}
@@ -696,6 +712,10 @@ jobs:
echo "=== ClamAV Logs ==="
docker compose logs --timestamps clamav || true
fi
if [ "${{ matrix.config.storage_device }}" = "s3" ]; then
echo "=== MinIO Logs ==="
docker compose logs --timestamps minio || true
fi
benchmark:
name: Benchmark
+1 -1
View File
@@ -1251,7 +1251,7 @@ return [
],
[
'name' => '_APP_EXECUTOR_CONNECTION_STORAGE',
'description' => "DSN for Open Runtimes executor storage. When `_APP_STORAGE_DEVICE` is not local, point this at the same backend so the executor can read deployment artifacts. Defaults to `local://localhost`.\n\nExamples:\n- Local: `local://localhost`\n- AWS S3: `s3://ACCESS_KEY:SECRET@BUCKET.s3.REGION.amazonaws.com?region=REGION`\n- S3-compatible: `s3://ACCESS_KEY:SECRET@localhost/BUCKET?region=REGION&url=http%3A%2F%2Fminio%3A9000`",
'description' => "DSN for Open Runtimes executor storage. When `_APP_STORAGE_DEVICE` is not local, point this at the same backend so the executor can read deployment artifacts. Defaults to `local://localhost`.\n\nExamples:\n- Local: `local://localhost`\n- AWS S3: `s3://ACCESS_KEY:SECRET@BUCKET.s3.REGION.amazonaws.com?region=REGION`\n- S3-compatible (`_APP_STORAGE_S3_ENDPOINT` set): `s3://ACCESS_KEY:SECRET@localhost?region=REGION&url=http%3A%2F%2Fminio%3A9000` — leave the bucket out of the DSN path; Appwrite keys objects under `BUCKET/` already, so the executor must not add it to the endpoint",
'introduction' => '1.9.5',
'default' => 'local://localhost',
'required' => false,
+9 -3
View File
@@ -213,9 +213,15 @@ return function (Container $context): void {
), ['publisher']);
// Builds a Deployments bound to a given project — webhook handlers resolve
// their tenant projects mid-request, after this container is initialized.
$context->set('deploymentsFactory', function (Jobs $jobs, array $platform) {
return fn (Database $dbForProject, Document $project): Deployments => new Deployments($jobs, $dbForProject, $project, $platform);
}, ['jobs', 'platform']);
$context->set('deploymentsFactory', function (Jobs $jobs, array $platform, Telemetry $telemetry) {
return fn (Database $dbForProject, Document $project): Deployments => new Deployments(
$jobs,
$dbForProject,
$project,
$platform,
new Device\Telemetry($telemetry, getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId())),
);
}, ['jobs', 'platform', 'telemetry']);
$context->set('deployments', fn (callable $deploymentsFactory, Database $dbForProject, Document $project) => $deploymentsFactory($dbForProject, $project), ['deploymentsFactory', 'dbForProject', 'project']);
$context->set('eventProcessor', fn () => new EventProcessor(), []);
$context->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory(
+4 -3
View File
@@ -24,6 +24,7 @@ use Utopia\Queue\Publisher;
use Utopia\Queue\Queue;
use Utopia\Registry\Registry;
use Utopia\Span\Span;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Telemetry as TelemetryDevice;
use Utopia\System\System;
use Utopia\Telemetry\Adapter as Telemetry;
@@ -190,9 +191,9 @@ return function (Container $container): void {
// Only the Builds worker uses this, handing template-into-repo pushes to
// the jobs-service.
$container->set('deployments', function (Jobs $jobs, Database $dbForProject, Document $project, array $platform) {
return new Deployments($jobs, $dbForProject, $project, $platform);
}, ['jobs', 'dbForProject', 'project', 'platform']);
$container->set('deployments', function (Jobs $jobs, Database $dbForProject, Document $project, array $platform, Device $deviceForBuilds) {
return new Deployments($jobs, $dbForProject, $project, $platform, $deviceForBuilds);
}, ['jobs', 'dbForProject', 'project', 'platform', 'deviceForBuilds']);
$container->set('logError', function (Registry $register, Document $project) {
return function (Throwable $error, string $namespace, string $action, ?array $extras = null) use ($register, $project) {
+29
View File
@@ -560,6 +560,35 @@ services:
restart: unless-stopped
networks:
- appwrite
# S3-compatible storage for the s3 e2e group. Creates the bucket the
# _APP_STORAGE_S3_* variables point at once the server is up.
minio:
profiles:
- s3
image: minio/minio:RELEASE.2025-09-07T16-13-09Z
container_name: appwrite-minio
restart: unless-stopped
networks:
- appwrite
environment:
- MINIO_ROOT_USER=${_APP_STORAGE_S3_ACCESS_KEY:-minio}
- MINIO_ROOT_PASSWORD=${_APP_STORAGE_S3_SECRET:-minio12345}
- MINIO_BUCKET=${_APP_STORAGE_S3_BUCKET:-appwrite}
entrypoint:
- sh
- -c
- |
minio server /data &
until mc alias set local http://localhost:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do sleep 1; done
mc mb -p "local/$$MINIO_BUCKET"
wait
healthcheck:
test:
- CMD-SHELL
- mc ls "local/$$MINIO_BUCKET" >/dev/null
interval: 5s
timeout: 5s
retries: 20
configs:
redisinsight-connections.json:
content: |
+22
View File
@@ -906,6 +906,28 @@ services:
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
- _APP_STORAGE_DEVICE
- _APP_STORAGE_S3_ACCESS_KEY
- _APP_STORAGE_S3_SECRET
- _APP_STORAGE_S3_REGION
- _APP_STORAGE_S3_BUCKET
- _APP_STORAGE_S3_ENDPOINT
- _APP_STORAGE_DO_SPACES_ACCESS_KEY
- _APP_STORAGE_DO_SPACES_SECRET
- _APP_STORAGE_DO_SPACES_REGION
- _APP_STORAGE_DO_SPACES_BUCKET
- _APP_STORAGE_BACKBLAZE_ACCESS_KEY
- _APP_STORAGE_BACKBLAZE_SECRET
- _APP_STORAGE_BACKBLAZE_REGION
- _APP_STORAGE_BACKBLAZE_BUCKET
- _APP_STORAGE_LINODE_ACCESS_KEY
- _APP_STORAGE_LINODE_SECRET
- _APP_STORAGE_LINODE_REGION
- _APP_STORAGE_LINODE_BUCKET
- _APP_STORAGE_WASABI_ACCESS_KEY
- _APP_STORAGE_WASABI_SECRET
- _APP_STORAGE_WASABI_REGION
- _APP_STORAGE_WASABI_BUCKET
- _APP_JOBS_HOST
- _APP_JOBS_SECRET
- _APP_REDIS_HOST
+34 -2
View File
@@ -22,6 +22,9 @@ use Utopia\Database\Document;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Query;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
use Utopia\Storage\DeviceType;
use Utopia\System\System;
use Utopia\VCS\Adapter\Git;
@@ -53,6 +56,7 @@ readonly class Deployments
protected Database $dbForProject,
protected Document $project,
private array $platform,
private Device $deviceForBuilds,
) {
}
@@ -182,7 +186,7 @@ readonly class Deployments
$queued = $this->dbForProject->updateDocuments('deployments', new Document([
'status' => 'waiting',
'buildPath' => static::buildPath($this->project->getId(), $deployment->getId()),
'buildPath' => $this->deviceForBuilds->getPath($deployment->getId() . '/' . static::artifact()),
]), [
Query::equal('$id', [$deployment->getId()]),
Query::notEqual('status', 'canceled'),
@@ -458,6 +462,32 @@ readonly class Deployments
];
}
/**
* Bring a finished build's artifact to the deployment's buildPath, once
* the job has exited and delivered its artifacts. With the volume strategy
* (see storage()) build.sh wrote it onto the builds volume, which on the
* local device already is the buildPath; on a remote device (S3 and
* friends) it is moved off the volume onto the device here. A strategy
* whose artifacts already land on the device overrides this with a no-op.
*/
public function store(Document $deployment): void
{
if ($this->deviceForBuilds->getType() === DeviceType::Local) {
return;
}
$source = static::buildPath($this->project->getId(), $deployment->getId());
$target = (string) $deployment->getAttribute('buildPath', '');
$local = new Local();
if ($target === '' || ! $local->exists($source)) {
return;
}
if ($local->copy($source, $target, $this->deviceForBuilds)) {
$local->delete($source);
}
}
/**
* The jobs-service job id for a deployment build (used to submit and cancel).
*/
@@ -476,7 +506,9 @@ readonly class Deployments
}
/**
* The build output path on the builds volume, declared at submission.
* The build output path on the builds volume. On the local device this is
* also the deployment's buildPath; on a remote device (S3 and friends) the
* buildPath is a device path and the Jobs worker moves the artifact there.
*/
public static function buildPath(string $projectId, string $deploymentId): string
{
@@ -3,6 +3,7 @@
namespace Appwrite\Platform\Modules\Functions\Workers;
use Appwrite\Bus\Events\RuleUpdated;
use Appwrite\Deployment\Deployments;
use Appwrite\Deployment\Detection;
use Appwrite\Deployment\GitAction;
use Appwrite\Event\Event;
@@ -87,6 +88,7 @@ class Jobs extends Action
->inject('platform')
->inject('plan')
->inject('bus')
->inject('deployments')
->callback($this->action(...));
}
@@ -109,6 +111,7 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): void {
$event = JobsMessage::fromArray($message->getPayload());
@@ -117,7 +120,7 @@ class Jobs extends Action
return;
}
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForScreenshots, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $deploymentId, $bus): void {
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForScreenshots, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $deploymentId, $bus, $deployments): void {
if ($event->id !== '') {
$key = 'jobs-event-' . $event->id;
if ($cache->load($key, self::DEDUPE_TTL) !== false) {
@@ -135,10 +138,10 @@ class Jobs extends Action
$deployment = match ($event->event) {
'orchestrator.job.log' => $this->onLog($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $vcsFactory, $platform),
'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus),
'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus),
'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus),
default => $this->onCallback($event->event, $dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus),
'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments),
'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments),
'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments),
default => $this->onCallback($event->event, $dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments),
};
// Console realtime on every callback (log stream + status).
@@ -226,6 +229,7 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): Document {
return $deployment;
}
@@ -275,6 +279,7 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): Document {
if (($data['artifactId'] ?? '') === 'manifest') {
// A failed manifest degrades to an empty listing (detection
@@ -283,7 +288,7 @@ class Jobs extends Action
$files = \is_array($manifest) ? (array) ($manifest['files'] ?? []) : [];
$cache->save('jobs-manifest-' . $deployment->getId(), ['files' => \array_values($files)]);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments);
}
if (($data['artifactId'] ?? '') !== 'sourceSize' || ($data['status'] ?? '') !== 'success') {
@@ -324,6 +329,7 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): Document {
if ($exitCode !== 0) {
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus);
@@ -331,7 +337,7 @@ class Jobs extends Action
$cache->save('jobs-exit-' . $deployment->getId(), true);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments);
}
/**
@@ -352,10 +358,11 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): Document {
$cache->save('jobs-complete-' . $deployment->getId(), true);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments);
}
/**
@@ -376,6 +383,7 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): Document {
if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'], true)) {
return $deployment; // already finalized
@@ -400,6 +408,10 @@ class Jobs extends Action
}
}
// The build job wrote its artifact where the storage strategy put it;
// bring it to buildPath before checking for it.
$deployments->store($deployment);
$path = (string) $deployment->getAttribute('buildPath', '');
if ($path === '' || ! $deviceForBuilds->exists($path)) {
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build produced no output artifact.', $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus);
@@ -7,6 +7,7 @@ namespace Tests\E2E\Services\Functions;
use Appwrite\Platform\Modules\Compute\Specification;
use Appwrite\Tests\Retry;
use PHPUnit\Framework\Attributes\DataProvider;
use PHPUnit\Framework\Attributes\Group;
use Tests\E2E\Client;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\Scope;
@@ -1030,6 +1031,62 @@ final class FunctionsCustomServerTest extends Scope
$this->assertEquals(204, $deployment['headers']['status-code']);
}
/**
* The build job writes onto the builds volume while the executor and the
* download endpoint read the deployment through the configured storage
* device. Runs in the `s3` CI group against MinIO, where the artifact has
* to be moved off the volume and keyed under the bucket; on the default
* local device the two are the same path.
*/
#[Group('s3')]
public function testDeploymentBuildOutputIsServedFromTheBuildsDevice(): void
{
$functionId = $this->setupFunction([
'functionId' => ID::unique(),
'name' => 'Build output',
'runtime' => 'node-22',
'entrypoint' => 'index.js',
'execute' => [Role::any()->toString()],
]);
$deployment = $this->createDeployment($functionId, [
'code' => $this->packageFunction('basic'),
'activate' => true,
]);
$this->assertEquals(202, $deployment['headers']['status-code']);
$deploymentId = $deployment['body']['$id'];
$this->assertEventually(function () use ($functionId, $deploymentId) {
$deployment = $this->getDeployment($functionId, $deploymentId);
$this->assertEquals('ready', $deployment['body']['status'], $deployment['body']['buildLogs'] ?? '');
}, 100000, 500);
/**
* Test for SUCCESS
*/
$deployment = $this->getDeployment($functionId, $deploymentId);
$this->assertGreaterThan(0, $deployment['body']['buildSize']);
// The executor fetches the artifact through its own storage connection.
$execution = $this->createExecution($functionId);
$this->assertEquals(201, $execution['headers']['status-code']);
$this->assertEquals('completed', $execution['body']['status'], $execution['body']['errors'] ?? '');
$this->assertEquals(200, $execution['body']['responseStatusCode']);
$this->assertSame($deploymentId, \json_decode($execution['body']['responseBody'], true)['APPWRITE_FUNCTION_DEPLOYMENT']);
// The download endpoint reads buildPath through the builds device. The
// artifact format depends on the storage strategy, so compare sizes.
$output = $this->getDeploymentDownload($functionId, $deploymentId, 'output');
$this->assertEquals(200, $output['headers']['status-code']);
$this->assertSame($deployment['body']['buildSize'], \strlen($output['body']));
$source = $this->getDeploymentDownload($functionId, $deploymentId, 'source');
$this->assertEquals(200, $source['headers']['status-code']);
$this->assertStringStartsWith("\x1f\x8b", $source['body']);
$this->cleanupFunction($functionId);
}
#[Retry(count: 3)]
public function testCancelDeploymentBuild(): void
{