fix: let the build job deliver its output to the builds device

#13443 fixed self-hosted builds on S3 by having the Jobs worker copy the
finished artifact off the builds volume onto the device in a new
Deployments::store(), which Cloud then had to override with a no-op: one
decision (where build output lands) split across two overridable methods
that had to agree.

Instead the storage strategy follows the builds device. Local keeps the
shared volume; a remote device gets the s3:// upload/download artifacts
Cloud already runs, signed by the orchestrator's S3_* credentials, so the
job delivers the artifact itself and the worker only checks buildPath.
buildPath()/cachePath() are the device's paths, which keys them under the
bucket on path-style stores such as MinIO and is unchanged on the local
and virtual-host devices. store() and the device constructor parameter go
away, so Cloud's Deployments and factories compile unchanged.
This commit is contained in:
Chirag Aggarwal committed 2026-09-03 10:43:31 +01:00
1 parent e6a468a11f
commit 5685834eed
6 files changed
+153 -100

No files matched your search

+3 -9
View File
@@ -213,15 +213,9 @@ return function (Container $context): void {
), ['publisher']);
// Builds a Deployments bound to a given project — webhook handlers resolve
// their tenant projects mid-request, after this container is initialized.
$context->set('deploymentsFactory', function (Jobs $jobs, array $platform, Telemetry $telemetry) {
return fn (Database $dbForProject, Document $project): Deployments => new Deployments(
$jobs,
$dbForProject,
$project,
$platform,
new Device\Telemetry($telemetry, getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId())),
);
}, ['jobs', 'platform', 'telemetry']);
$context->set('deploymentsFactory', function (Jobs $jobs, array $platform) {
return fn (Database $dbForProject, Document $project): Deployments => new Deployments($jobs, $dbForProject, $project, $platform);
}, ['jobs', 'platform']);
$context->set('deployments', fn (callable $deploymentsFactory, Database $dbForProject, Document $project) => $deploymentsFactory($dbForProject, $project), ['deploymentsFactory', 'dbForProject', 'project']);
$context->set('eventProcessor', fn () => new EventProcessor(), []);
$context->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory(
+3 -4
View File
@@ -24,7 +24,6 @@ use Utopia\Queue\Publisher;
use Utopia\Queue\Queue;
use Utopia\Registry\Registry;
use Utopia\Span\Span;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Telemetry as TelemetryDevice;
use Utopia\System\System;
use Utopia\Telemetry\Adapter as Telemetry;
@@ -191,9 +190,9 @@ return function (Container $container): void {
// Only the Builds worker uses this, handing template-into-repo pushes to
// the jobs-service.
$container->set('deployments', function (Jobs $jobs, Database $dbForProject, Document $project, array $platform, Device $deviceForBuilds) {
return new Deployments($jobs, $dbForProject, $project, $platform, $deviceForBuilds);
}, ['jobs', 'dbForProject', 'project', 'platform', 'deviceForBuilds']);
$container->set('deployments', function (Jobs $jobs, Database $dbForProject, Document $project, array $platform) {
return new Deployments($jobs, $dbForProject, $project, $platform);
}, ['jobs', 'dbForProject', 'project', 'platform']);
$container->set('logError', function (Registry $register, Document $project) {
return function (Throwable $error, string $namespace, string $action, ?array $extras = null) use ($register, $project) {
+7
View File
@@ -1707,6 +1707,13 @@ services:
- ORCHESTRATOR_NETWORK=appwrite
- DOCKER_NETWORK=appwrite
- ARTIFACT_ENDPOINT=http://orchestrator:8080
# Signs the s3:// build artifacts Appwrite submits when the builds
# device is not local (see Deployments::storage()); forwarded into
# the sidecar that runs them. Empty on the local device.
- S3_ENDPOINT=${_APP_STORAGE_S3_ENDPOINT:-}
- S3_REGION=${_APP_STORAGE_S3_REGION:-us-east-1}
- S3_ACCESS_KEY_ID=${_APP_STORAGE_S3_ACCESS_KEY:-}
- S3_SECRET_ACCESS_KEY=${_APP_STORAGE_S3_SECRET:-}
mariadb:
image: mariadb:10.11
+129 -64
View File
@@ -13,6 +13,7 @@ use OpenRuntimes\Orchestrator\Model\Artifact\DownloadArtifact;
use OpenRuntimes\Orchestrator\Model\Artifact\ReadArtifact;
use OpenRuntimes\Orchestrator\Model\Artifact\StatArtifact;
use OpenRuntimes\Orchestrator\Model\Artifact\UnarchiveArtifact;
use OpenRuntimes\Orchestrator\Model\Artifact\UploadArtifact;
use OpenRuntimes\Orchestrator\Model\Callback;
use OpenRuntimes\Orchestrator\Model\Volume;
use Utopia\Config\Config;
@@ -22,8 +23,8 @@ use Utopia\Database\Document;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Query;
use Utopia\DSN\DSN;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
use Utopia\Storage\DeviceType;
use Utopia\System\System;
use Utopia\VCS\Adapter\Git;
@@ -35,14 +36,16 @@ use Utopia\VCS\Adapter\Git;
*
* Source crosses the boundary via the artifacts system (presigned GET download
* + unarchive, run by the sidecar) — a GET has no request-body cap, so large
* sources are fine. The build output and package-manager cache, by default,
* go on a mounted volume: the builds storage volume is attached to the build
* worker at its Appwrite path, so build.sh writes its artifact + the cache
* squashfs straight onto the volume Appwrite already reads. That keeps the
* multi-hundred-MB output off the (capped) HTTP upload path and out of the
* Appwrite process. Deployments that need a different strategy (e.g. S3
* upload/download artifacts instead of a shared volume) override storage()
* — everything else about the payload stays the same.
* sources are fine. The build output and package-manager cache go wherever
* the builds device is (see storage()). On the local device the builds
* storage volume is attached to the build worker at its Appwrite path, so
* build.sh writes its artifact + the cache squashfs straight onto the volume
* Appwrite already reads. On a remote device (S3 and friends) no volume spans
* Appwrite and the build workers, so the sidecar moves them over s3://
* upload/download artifacts instead. Either way the multi-hundred-MB output
* stays off the (capped) HTTP upload path and out of the Appwrite process.
* Deployments that need yet another strategy override storage() — everything
* else about the payload stays the same.
*
* Covers function and site deployments whose source is a tarball: manual
* upload, duplicate/rebuild, VCS commits, and templates (public GitHub tarball
@@ -56,7 +59,6 @@ readonly class Deployments
protected Database $dbForProject,
protected Document $project,
private array $platform,
private Device $deviceForBuilds,
) {
}
@@ -186,7 +188,7 @@ readonly class Deployments
$queued = $this->dbForProject->updateDocuments('deployments', new Document([
'status' => 'waiting',
'buildPath' => $this->deviceForBuilds->getPath($deployment->getId() . '/' . static::artifact()),
'buildPath' => static::buildPath($this->project->getId(), $deployment->getId()),
]), [
Query::equal('$id', [$deployment->getId()]),
Query::notEqual('status', 'canceled'),
@@ -462,32 +464,6 @@ readonly class Deployments
];
}
/**
* Bring a finished build's artifact to the deployment's buildPath, once
* the job has exited and delivered its artifacts. With the volume strategy
* (see storage()) build.sh wrote it onto the builds volume, which on the
* local device already is the buildPath; on a remote device (S3 and
* friends) it is moved off the volume onto the device here. A strategy
* whose artifacts already land on the device overrides this with a no-op.
*/
public function store(Document $deployment): void
{
if ($this->deviceForBuilds->getType() === DeviceType::Local) {
return;
}
$source = static::buildPath($this->project->getId(), $deployment->getId());
$target = (string) $deployment->getAttribute('buildPath', '');
$local = new Local();
if ($target === '' || ! $local->exists($source)) {
return;
}
if ($local->copy($source, $target, $this->deviceForBuilds)) {
$local->delete($source);
}
}
/**
* The jobs-service job id for a deployment build (used to submit and cancel).
*/
@@ -497,8 +473,8 @@ readonly class Deployments
}
/**
* The build output directory on the builds volume. The produced artifact's
* complete path is discovered and persisted after the job finishes.
* The build output directory on the builds volume, where build.sh writes
* under the volume strategy (see storage()).
*/
public static function outputDirectory(string $projectId, string $deploymentId): string
{
@@ -506,13 +482,15 @@ readonly class Deployments
}
/**
* The build output path on the builds volume. On the local device this is
* also the deployment's buildPath; on a remote device (S3 and friends) the
* buildPath is a device path and the Jobs worker moves the artifact there.
* The build artifact's path on the builds device, declared at submission
* and read back through deviceForBuilds by the jobs worker, executions,
* downloads and deletes. On the local device it is outputDirectory() +
* the artifact, i.e. where build.sh writes; a path-style S3 device keys
* it under its bucket.
*/
public static function buildPath(string $projectId, string $deploymentId): string
{
return static::outputDirectory($projectId, $deploymentId) . '/' . static::artifact();
return static::device($projectId)->getPath("{$deploymentId}/" . static::artifact());
}
/**
@@ -541,24 +519,45 @@ readonly class Deployments
return \substr(\hash('sha256', "{$projectId}:{$resourceId}:{$image}"), 0, 48);
}
/**
* The package-manager cache's path on the builds device (see buildPath()).
*/
public static function cachePath(string $projectId, string $cacheKey): string
{
return APP_STORAGE_BUILDS . "/app-{$projectId}/cache/{$cacheKey}.sqfs";
return static::device($projectId)->getPath("cache/{$cacheKey}.sqfs");
}
/**
* The builds device for a project, as everything else reads it
* (deviceForBuilds).
*/
protected static function device(string $projectId): Device
{
return getDevice(APP_STORAGE_BUILDS . "/app-{$projectId}");
}
/**
* Where build.sh's output artifact and package-manager cache
* (a squashfs) land, and what the job needs to get them there. The
* default mounts the shared builds volume at outputDirectory()/cachePath();
* build.sh only cares that OPEN_RUNTIMES_BUILD_OUTPUT_DIR/_CACHE_ARTIFACT
* point somewhere on its local filesystem, volume-backed or not — so a
* strategy without a shared volume (e.g. S3) instead points them at a
* local tmp path and moves things in/out via 'artifacts':
* - cache pull, before the build: a plain DownloadArtifact (no
* `depends`, so it runs before the command) into the local cache path.
* - cache push and output upload, after the build: an UploadArtifact
* with `depends: 'job'` — 'job' is the orchestrator's sentinel id for
* "after the build command finishes", not an id of another artifact.
* (a squashfs) land, and what the job needs to get them there. build.sh
* only cares that OPEN_RUNTIMES_BUILD_OUTPUT_DIR/_CACHE_ARTIFACT point
* somewhere on its local filesystem, volume-backed or not, so the
* strategy follows the builds device:
* - local: the shared builds volume is mounted and build.sh writes to
* outputDirectory()/cachePath() on it, which already are the device's
* paths.
* - remote (S3 and friends): no volume spans Appwrite and the build
* workers, so build.sh writes into the job workspace and the sidecar
* moves things in/out via s3:// 'artifacts', signed with the S3_*
* credentials the orchestrator is configured with:
* - cache pull, before the build: a plain DownloadArtifact (no
* `depends`, so it runs before the command) into the local cache
* path.
* - cache push and output upload, after the build: an UploadArtifact
* with `depends: 'job'` — 'job' is the orchestrator's sentinel id
* for "after the build command finishes", not an id of another
* artifact.
* Object keys are the device's buildPath()/cachePath(), so everything
* that reads builds through deviceForBuilds works unchanged.
*
* @return array{volumes: array<Volume>, artifacts: array<mixed>, environment: array<string, string>}
*/
@@ -568,22 +567,88 @@ readonly class Deployments
$deploymentId = $deployment->getId();
$runtime = self::runtime($resource, self::version($resource));
$cacheKey = static::cacheKey($projectId, $resource->getId(), $runtime['image'] ?? '');
$cachePath = static::cachePath($projectId, $cacheKey);
$device = static::device($projectId);
if ($device->getType() === DeviceType::Local) {
return [
// Docker volume / K8s PVC named by _APP_BUILDS_VOLUME, attached
// to the worker at its Appwrite path so build.sh writes output +
// cache straight onto it.
'volumes' => [
new Volume(source: System::getEnv('_APP_BUILDS_VOLUME', 'appwrite-builds'), path: APP_STORAGE_BUILDS),
],
'artifacts' => [],
'environment' => [
'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => static::outputDirectory($projectId, $deploymentId),
'OPEN_RUNTIMES_BUILD_CACHE_ARTIFACT' => $cachePath,
],
];
}
// 'in' paths are workspace-relative.
$artifacts = [
new UploadArtifact(id: 'output', in: 'output/' . static::artifact(), out: static::objectUrl($device, static::buildPath($projectId, $deploymentId)), depends: 'job'),
new UploadArtifact(id: 'cache', in: "cache/{$cacheKey}.sqfs", out: static::objectUrl($device, $cachePath), depends: 'job'),
];
// Pull the cache only when a previous build saved one: a failed
// pre-job artifact aborts the job, so an unconditional download
// would break every first build.
if ($device->exists($cachePath)) {
\array_unshift($artifacts, new DownloadArtifact(id: 'cachePull', in: static::objectUrl($device, $cachePath), out: "cache/{$cacheKey}.sqfs"));
}
return [
// Docker volume / K8s PVC named by _APP_BUILDS_VOLUME, attached
// to the worker at its Appwrite path so build.sh writes output +
// cache straight onto it.
'volumes' => [
new Volume(source: System::getEnv('_APP_BUILDS_VOLUME', 'appwrite-builds'), path: APP_STORAGE_BUILDS),
],
'artifacts' => [],
'volumes' => [],
'artifacts' => $artifacts,
'environment' => [
'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => static::outputDirectory($projectId, $deploymentId),
'OPEN_RUNTIMES_BUILD_CACHE_ARTIFACT' => static::cachePath($projectId, $cacheKey),
'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => '/mnt/code/output',
'OPEN_RUNTIMES_BUILD_CACHE_ARTIFACT' => "/mnt/code/cache/{$cacheKey}.sqfs",
],
];
}
/**
* Map a builds device path to the s3://bucket/key artifact URL the sidecar
* uploads to, resolving the bucket from the same configuration getDevice()
* reads so artifact writes and device reads land on the same object. An
* S3 device with an explicit endpoint (S3-compatible stores such as MinIO)
* is path-style and already keys objects under its bucket, so the path is
* the URL as is; a virtual-host device passes paths verbatim, so the
* bucket is prepended.
*/
protected static function objectUrl(Device $device, string $path): string
{
$bucket = self::bucket($device);
return 's3://' . \ltrim(($bucket !== '' ? "/{$bucket}" : '') . $path, '/');
}
private static function bucket(Device $device): string
{
$type = $device->getType();
if (\in_array($type, [DeviceType::S3, DeviceType::AwsS3], true) && System::getEnv('_APP_STORAGE_S3_ENDPOINT', '') !== '') {
return '';
}
$connection = System::getEnv('_APP_CONNECTIONS_STORAGE', '');
if ($connection !== '') {
return \trim((new DSN($connection))->getPath() ?? '', '/');
}
$prefix = match ($type) {
DeviceType::S3, DeviceType::AwsS3 => 'S3',
DeviceType::DoSpaces => 'DO_SPACES',
DeviceType::Backblaze => 'BACKBLAZE',
DeviceType::Linode => 'LINODE',
DeviceType::Wasabi => 'WASABI',
DeviceType::Local => null,
};
return $prefix === null ? '' : System::getEnv("_APP_STORAGE_{$prefix}_BUCKET", '');
}
protected static function version(Document $resource): string
{
return $resource->getCollection() === 'sites' ? 'v5' : $resource->getAttribute('version', 'v2');
@@ -3,7 +3,6 @@
namespace Appwrite\Platform\Modules\Functions\Workers;
use Appwrite\Bus\Events\RuleUpdated;
use Appwrite\Deployment\Deployments;
use Appwrite\Deployment\Detection;
use Appwrite\Deployment\GitAction;
use Appwrite\Event\Event;
@@ -88,7 +87,6 @@ class Jobs extends Action
->inject('platform')
->inject('plan')
->inject('bus')
->inject('deployments')
->callback($this->action(...));
}
@@ -111,7 +109,6 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): void {
$event = JobsMessage::fromArray($message->getPayload());
@@ -120,7 +117,7 @@ class Jobs extends Action
return;
}
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForScreenshots, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $deploymentId, $bus, $deployments): void {
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForScreenshots, $publisherForUsage, $usage, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $deploymentId, $bus): void {
if ($event->id !== '') {
$key = 'jobs-event-' . $event->id;
if ($cache->load($key, self::DEDUPE_TTL) !== false) {
@@ -138,10 +135,10 @@ class Jobs extends Action
$deployment = match ($event->event) {
'orchestrator.job.log' => $this->onLog($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $vcsFactory, $platform),
'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments),
'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments),
'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments),
default => $this->onCallback($event->event, $dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments),
'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus),
'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus),
'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus),
default => $this->onCallback($event->event, $dbForProject, $dbForPlatform, $project, $deployment, $event->data, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus),
};
// Console realtime on every callback (log stream + status).
@@ -229,7 +226,6 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): Document {
return $deployment;
}
@@ -279,7 +275,6 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): Document {
if (($data['artifactId'] ?? '') === 'manifest') {
// A failed manifest degrades to an empty listing (detection
@@ -288,7 +283,7 @@ class Jobs extends Action
$files = \is_array($manifest) ? (array) ($manifest['files'] ?? []) : [];
$cache->save('jobs-manifest-' . $deployment->getId(), ['files' => \array_values($files)]);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus);
}
if (($data['artifactId'] ?? '') !== 'sourceSize' || ($data['status'] ?? '') !== 'success') {
@@ -329,7 +324,6 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): Document {
if ($exitCode !== 0) {
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus);
@@ -337,7 +331,7 @@ class Jobs extends Action
$cache->save('jobs-exit-' . $deployment->getId(), true);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus);
}
/**
@@ -358,11 +352,10 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): Document {
$cache->save('jobs-complete-' . $deployment->getId(), true);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus, $deployments);
return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $publisherForScreenshots, $deviceForBuilds, $vcsFactory, $cache, $platform, $plan, $bus);
}
/**
@@ -383,7 +376,6 @@ class Jobs extends Action
array $platform,
array $plan,
Bus $bus,
Deployments $deployments,
): Document {
if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'], true)) {
return $deployment; // already finalized
@@ -408,10 +400,6 @@ class Jobs extends Action
}
}
// The build job wrote its artifact where the storage strategy put it;
// bring it to buildPath before checking for it.
$deployments->store($deployment);
$path = (string) $deployment->getAttribute('buildPath', '');
if ($path === '' || ! $deviceForBuilds->exists($path)) {
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build produced no output artifact.', $usage, $publisherForUsage, $publisherForScreenshots, $vcsFactory, $platform, $bus);
@@ -1034,9 +1034,9 @@ final class FunctionsCustomServerTest extends Scope
/**
* The build job writes onto the builds volume while the executor and the
* download endpoint read the deployment through the configured storage
* device. Runs in the `s3` CI group against MinIO, where the artifact has
* to be moved off the volume and keyed under the bucket; on the default
* local device the two are the same path.
* device. Runs in the `s3` CI group against MinIO, where the sidecar
* uploads the artifact as an s3:// object keyed under the bucket; on the
* default local device build.sh writes it straight to buildPath.
*/
#[Group('s3')]
public function testDeploymentBuildOutputIsServedFromTheBuildsDevice(): void