Merge pull request #12890 from appwrite/fix/opaque-build-output-path

fix: persist opaque build output paths
This commit is contained in:
Torsten Dittmann authored and GitHub committed 2026-07-17 19:48:32 +04:00
commit a6d30130ba
10 files changed
+66 -33

No files matched your search

+1 -2
View File
@@ -552,7 +552,6 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S
'site' => '',
};
$source = $deployment->getAttribute('buildPath', '');
$extension = str_ends_with($source, '.tar') ? 'tar' : 'tar.gz';
$startCommand = $runtime['startCommand'];
if ($type === 'site') {
@@ -573,7 +572,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S
$runtimeEntrypoint = match ($version) {
'v2' => '',
default => "cp /tmp/code.$extension /mnt/code/code.$extension && nohup helpers/start.sh \"$startCommand\"",
default => "cp /tmp/code.* /mnt/code/ && nohup helpers/start.sh \"$startCommand\"",
};
$entrypoint = match ($type) {
@@ -24,7 +24,7 @@ use Utopia\System\System;
* + unarchive, run by the sidecar) — a GET has no request-body cap, so large
* sources are fine. The build output and package-manager cache, by default,
* go on a mounted volume: the builds storage volume is attached to the build
* worker at its Appwrite path, so build.sh writes code.tar.gz + the cache
* worker at its Appwrite path, so build.sh writes its artifact + the cache
* squashfs straight onto the volume Appwrite already reads. That keeps the
* multi-hundred-MB output off the (capped) HTTP upload path and out of the
* Appwrite process. Deployments that need a different strategy (e.g. S3
@@ -82,8 +82,6 @@ readonly class Orchestrator extends Backend
$deployment = $this->upload($resource, $deployment);
$this->deactivateOthers($resource, $deployment);
// Pre-declare buildPath so build.sh writes output straight onto the
// mounted builds volume, at the path this deployment expects.
$deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
'status' => 'waiting',
'buildPath' => static::buildPath($this->project->getId(), $deployment->getId()),
@@ -168,6 +166,7 @@ readonly class Orchestrator extends Backend
$command = $deployment->getAttribute('buildCommands', '');
$env = self::variables($project, $function, $deployment, $runtime, $cpus, $memory, $endpoint, $timeout) + [
'OPEN_RUNTIMES_BUILD_INPUT_DIR' => '/mnt/code/source',
'OPEN_RUNTIMES_BUILD_COMPRESSION' => static::compression(),
] + $output['environment'];
return [
@@ -213,13 +212,37 @@ readonly class Orchestrator extends Backend
}
/**
* The build output path on the builds volume. build.sh writes code.tar.gz
* into OPEN_RUNTIMES_BUILD_OUTPUT_DIR (this file's directory); pre-computed
* so it can be persisted on the deployment before the job is submitted.
* The build output directory on the builds volume. The produced artifact's
* complete path is discovered and persisted after the job finishes.
*/
public static function outputDirectory(string $projectId, string $deploymentId): string
{
return APP_STORAGE_BUILDS . "/app-{$projectId}/{$deploymentId}";
}
/**
* The build output path on the builds volume, declared at submission.
*/
public static function buildPath(string $projectId, string $deploymentId): string
{
return APP_STORAGE_BUILDS . "/app-{$projectId}/{$deploymentId}/code.tar.gz";
return static::outputDirectory($projectId, $deploymentId) . '/' . static::artifact();
}
/**
* The artifact filename build.sh produces for the configured compression.
*/
public static function artifact(): string
{
return match (static::compression()) {
'none' => 'code.tar',
'squashfs' => 'code.sqfs',
default => 'code.tar.gz',
};
}
protected static function compression(): string
{
return System::getEnv('_APP_COMPUTE_BUILD_COMPRESSION', 'gzip');
}
/**
@@ -237,9 +260,9 @@ readonly class Orchestrator extends Backend
}
/**
* Where build.sh's output (code.tar.gz) and package-manager cache
* Where build.sh's output artifact and package-manager cache
* (a squashfs) land, and what the job needs to get them there. The
* default mounts the shared builds volume at buildPath()/cachePath();
* default mounts the shared builds volume at outputDirectory()/cachePath();
* build.sh only cares that OPEN_RUNTIMES_BUILD_OUTPUT_DIR/_CACHE_ARTIFACT
* point somewhere on its local filesystem, volume-backed or not — so a
* strategy without a shared volume (e.g. S3) instead points them at a
@@ -268,7 +291,7 @@ readonly class Orchestrator extends Backend
],
'artifacts' => [],
'environment' => [
'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => \dirname(static::buildPath($projectId, $deploymentId)),
'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => static::outputDirectory($projectId, $deploymentId),
'OPEN_RUNTIMES_BUILD_CACHE_ARTIFACT' => static::cachePath($projectId, $cacheKey),
],
];
@@ -125,11 +125,16 @@ class Get extends Action
throw new Exception(Exception::DEPLOYMENT_NOT_FOUND);
}
$contentType = $type === 'source' ? 'application/gzip' : 'application/octet-stream';
$filename = $type === 'source'
? $deploymentId . '-source.tar.gz'
: $deploymentId . '-output-' . \basename($path);
$response
->setContentType('application/gzip')
->setContentType($contentType)
->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days
->addHeader('X-Peak', \memory_get_peak_usage())
->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '-' . $type . '.tar.gz"');
->addHeader('Content-Disposition', 'attachment; filename="' . $filename . '"');
$size = $device->getFileSize($path);
$rangeHeader = $request->getHeaderLine('range');
@@ -3,6 +3,7 @@
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Status;
use Appwrite\Deployment\Backend;
use Appwrite\Deployment\Backend\Orchestrator;
use Appwrite\Event\Event;
use Appwrite\Extend\Exception;
use Appwrite\SDK\AuthType;
@@ -92,7 +93,7 @@ class Update extends Action
$duration = $endTime->getTimestamp() - $startTime->getTimestamp();
try {
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration)));
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator)));
} catch (TransactionException) {
$deployment = $dbForProject->getDocument('deployments', $deployment->getId());
@@ -105,7 +106,7 @@ class Update extends Action
}
if ($deployment->getAttribute('status') !== 'canceled') {
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration)));
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator)));
}
}
@@ -123,14 +124,13 @@ class Update extends Action
}
/**
* The sparse update marking a build canceled. Jobs-backed builds (identified
* by a buildPath set at submission) have no cancel worker to write the
* closing log line the executor's Builds worker adds, so it is appended here;
* executor deployments get it from their worker.
* The sparse update marking a build canceled. Jobs-backed builds have no
* cancel worker to write the closing log line the executor's Builds worker
* adds, so it is appended here; executor deployments get it from their worker.
*
* @return array<string, mixed>
*/
private function cancel(Document $deployment, int $duration): array
private function cancel(Document $deployment, int $duration, bool $appendLog): array
{
$update = [
'buildEndedAt' => DateTime::now(),
@@ -138,7 +138,7 @@ class Update extends Action
'status' => 'canceled',
];
if ($deployment->getAttribute('buildPath', '') !== '') {
if ($appendLog) {
$logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n";
$update['buildLogs'] = \substr($logs, -APP_LOG_LENGTH_LIMIT);
}
@@ -417,8 +417,7 @@ class Create extends Base
}
$source = $deployment->getAttribute('buildPath', '');
$extension = str_ends_with($source, '.tar') ? 'tar' : 'tar.gz';
$command = $version === 'v2' ? '' : "cp /tmp/code.$extension /mnt/code/code.$extension && nohup helpers/start.sh \"$command\"";
$command = $version === 'v2' ? '' : "cp /tmp/code.* /mnt/code/ && nohup helpers/start.sh \"$command\"";
try {
$executionResponse = $executor->createExecution(
projectId: $project->getId(),
@@ -2,7 +2,6 @@
namespace Appwrite\Platform\Modules\Functions\Workers;
use Appwrite\Deployment\Backend\Orchestrator;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Func as FunctionMessage;
use Appwrite\Event\Message\Jobs as JobsMessage;
@@ -266,8 +265,12 @@ class Jobs extends Action
Device $deviceForBuilds,
VcsFactory $vcsFactory,
): Document {
$path = Orchestrator::buildPath($project->getId(), $deployment->getId());
$size = $deviceForBuilds->exists($path) ? $deviceForBuilds->getFileSize($path) : 0;
$path = (string) $deployment->getAttribute('buildPath', '');
if ($path === '' || ! $deviceForBuilds->exists($path)) {
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build produced no output artifact.', $usage, $publisherForUsage, $vcsFactory);
}
$size = $deviceForBuilds->getFileSize($path);
$limit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000');
if ($limit !== 0 && $size > $limit) {
@@ -100,14 +100,19 @@ class Get extends Action
throw new Exception(Exception::DEPLOYMENT_NOT_FOUND);
}
$contentType = $type === 'source' ? 'application/gzip' : 'application/octet-stream';
$filename = $type === 'source'
? $deploymentId . '-source.tar.gz'
: $deploymentId . '-output-' . \basename($path);
$size = $device->getFileSize($path);
$rangeHeader = $request->getHeaderLine('range');
$response
->setContentType('application/gzip')
->setContentType($contentType)
->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days
->addHeader('X-Peak', \memory_get_peak_usage())
->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '-' . $type . '.tar.gz"');
->addHeader('Content-Disposition', 'attachment; filename="' . $filename . '"');
if (!empty($rangeHeader)) {
$start = $request->getRangeStart();
+1 -2
View File
@@ -550,8 +550,7 @@ class Functions extends Action
}
$source = $deployment->getAttribute('buildPath', '');
$extension = str_ends_with($source, '.tar') ? 'tar' : 'tar.gz';
$command = $version === 'v2' ? '' : "cp /tmp/code.$extension /mnt/code/code.$extension && nohup helpers/start.sh \"$command\"";
$command = $version === 'v2' ? '' : "cp /tmp/code.* /mnt/code/ && nohup helpers/start.sh \"$command\"";
try {
$executionResponse = $executor->createExecution(
projectId: $project->getId(),
@@ -899,7 +899,7 @@ final class FunctionsConsoleClientTest extends Scope
$response = $this->getDeploymentDownload($functionId, $deploymentId, 'output');
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals('application/gzip', $response['headers']['content-type']);
$this->assertEquals('application/octet-stream', $response['headers']['content-type']);
$this->assertGreaterThan(0, $response['headers']['content-length']);
$this->assertGreaterThan(0, \strlen($response['body']));
@@ -2341,7 +2341,7 @@ final class SitesCustomServerTest extends Scope
$response = $this->getDeploymentDownload($siteId, $deploymentId, 'output');
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals('application/gzip', $response['headers']['content-type']);
$this->assertEquals('application/octet-stream', $response['headers']['content-type']);
$this->assertGreaterThan(0, $response['headers']['content-length']);
$this->assertGreaterThan(0, \strlen($response['body']));