mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Merge pull request #12890 from appwrite/fix/opaque-build-output-path
fix: persist opaque build output paths
This commit is contained in:
10 files changed
+66
-33
No files matched your search
@@ -552,7 +552,6 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S
|
||||
'site' => '',
|
||||
};
|
||||
$source = $deployment->getAttribute('buildPath', '');
|
||||
$extension = str_ends_with($source, '.tar') ? 'tar' : 'tar.gz';
|
||||
|
||||
$startCommand = $runtime['startCommand'];
|
||||
if ($type === 'site') {
|
||||
@@ -573,7 +572,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S
|
||||
|
||||
$runtimeEntrypoint = match ($version) {
|
||||
'v2' => '',
|
||||
default => "cp /tmp/code.$extension /mnt/code/code.$extension && nohup helpers/start.sh \"$startCommand\"",
|
||||
default => "cp /tmp/code.* /mnt/code/ && nohup helpers/start.sh \"$startCommand\"",
|
||||
};
|
||||
|
||||
$entrypoint = match ($type) {
|
||||
|
||||
@@ -24,7 +24,7 @@ use Utopia\System\System;
|
||||
* + unarchive, run by the sidecar) — a GET has no request-body cap, so large
|
||||
* sources are fine. The build output and package-manager cache, by default,
|
||||
* go on a mounted volume: the builds storage volume is attached to the build
|
||||
* worker at its Appwrite path, so build.sh writes code.tar.gz + the cache
|
||||
* worker at its Appwrite path, so build.sh writes its artifact + the cache
|
||||
* squashfs straight onto the volume Appwrite already reads. That keeps the
|
||||
* multi-hundred-MB output off the (capped) HTTP upload path and out of the
|
||||
* Appwrite process. Deployments that need a different strategy (e.g. S3
|
||||
@@ -82,8 +82,6 @@ readonly class Orchestrator extends Backend
|
||||
$deployment = $this->upload($resource, $deployment);
|
||||
$this->deactivateOthers($resource, $deployment);
|
||||
|
||||
// Pre-declare buildPath so build.sh writes output straight onto the
|
||||
// mounted builds volume, at the path this deployment expects.
|
||||
$deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
|
||||
'status' => 'waiting',
|
||||
'buildPath' => static::buildPath($this->project->getId(), $deployment->getId()),
|
||||
@@ -168,6 +166,7 @@ readonly class Orchestrator extends Backend
|
||||
$command = $deployment->getAttribute('buildCommands', '');
|
||||
$env = self::variables($project, $function, $deployment, $runtime, $cpus, $memory, $endpoint, $timeout) + [
|
||||
'OPEN_RUNTIMES_BUILD_INPUT_DIR' => '/mnt/code/source',
|
||||
'OPEN_RUNTIMES_BUILD_COMPRESSION' => static::compression(),
|
||||
] + $output['environment'];
|
||||
|
||||
return [
|
||||
@@ -213,13 +212,37 @@ readonly class Orchestrator extends Backend
|
||||
}
|
||||
|
||||
/**
|
||||
* The build output path on the builds volume. build.sh writes code.tar.gz
|
||||
* into OPEN_RUNTIMES_BUILD_OUTPUT_DIR (this file's directory); pre-computed
|
||||
* so it can be persisted on the deployment before the job is submitted.
|
||||
* The build output directory on the builds volume. The produced artifact's
|
||||
* complete path is discovered and persisted after the job finishes.
|
||||
*/
|
||||
public static function outputDirectory(string $projectId, string $deploymentId): string
|
||||
{
|
||||
return APP_STORAGE_BUILDS . "/app-{$projectId}/{$deploymentId}";
|
||||
}
|
||||
|
||||
/**
|
||||
* The build output path on the builds volume, declared at submission.
|
||||
*/
|
||||
public static function buildPath(string $projectId, string $deploymentId): string
|
||||
{
|
||||
return APP_STORAGE_BUILDS . "/app-{$projectId}/{$deploymentId}/code.tar.gz";
|
||||
return static::outputDirectory($projectId, $deploymentId) . '/' . static::artifact();
|
||||
}
|
||||
|
||||
/**
|
||||
* The artifact filename build.sh produces for the configured compression.
|
||||
*/
|
||||
public static function artifact(): string
|
||||
{
|
||||
return match (static::compression()) {
|
||||
'none' => 'code.tar',
|
||||
'squashfs' => 'code.sqfs',
|
||||
default => 'code.tar.gz',
|
||||
};
|
||||
}
|
||||
|
||||
protected static function compression(): string
|
||||
{
|
||||
return System::getEnv('_APP_COMPUTE_BUILD_COMPRESSION', 'gzip');
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -237,9 +260,9 @@ readonly class Orchestrator extends Backend
|
||||
}
|
||||
|
||||
/**
|
||||
* Where build.sh's output (code.tar.gz) and package-manager cache
|
||||
* Where build.sh's output artifact and package-manager cache
|
||||
* (a squashfs) land, and what the job needs to get them there. The
|
||||
* default mounts the shared builds volume at buildPath()/cachePath();
|
||||
* default mounts the shared builds volume at outputDirectory()/cachePath();
|
||||
* build.sh only cares that OPEN_RUNTIMES_BUILD_OUTPUT_DIR/_CACHE_ARTIFACT
|
||||
* point somewhere on its local filesystem, volume-backed or not — so a
|
||||
* strategy without a shared volume (e.g. S3) instead points them at a
|
||||
@@ -268,7 +291,7 @@ readonly class Orchestrator extends Backend
|
||||
],
|
||||
'artifacts' => [],
|
||||
'environment' => [
|
||||
'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => \dirname(static::buildPath($projectId, $deploymentId)),
|
||||
'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => static::outputDirectory($projectId, $deploymentId),
|
||||
'OPEN_RUNTIMES_BUILD_CACHE_ARTIFACT' => static::cachePath($projectId, $cacheKey),
|
||||
],
|
||||
];
|
||||
|
||||
@@ -125,11 +125,16 @@ class Get extends Action
|
||||
throw new Exception(Exception::DEPLOYMENT_NOT_FOUND);
|
||||
}
|
||||
|
||||
$contentType = $type === 'source' ? 'application/gzip' : 'application/octet-stream';
|
||||
$filename = $type === 'source'
|
||||
? $deploymentId . '-source.tar.gz'
|
||||
: $deploymentId . '-output-' . \basename($path);
|
||||
|
||||
$response
|
||||
->setContentType('application/gzip')
|
||||
->setContentType($contentType)
|
||||
->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days
|
||||
->addHeader('X-Peak', \memory_get_peak_usage())
|
||||
->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '-' . $type . '.tar.gz"');
|
||||
->addHeader('Content-Disposition', 'attachment; filename="' . $filename . '"');
|
||||
|
||||
$size = $device->getFileSize($path);
|
||||
$rangeHeader = $request->getHeaderLine('range');
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Status;
|
||||
|
||||
use Appwrite\Deployment\Backend;
|
||||
use Appwrite\Deployment\Backend\Orchestrator;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\SDK\AuthType;
|
||||
@@ -92,7 +93,7 @@ class Update extends Action
|
||||
$duration = $endTime->getTimestamp() - $startTime->getTimestamp();
|
||||
|
||||
try {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration)));
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator)));
|
||||
} catch (TransactionException) {
|
||||
$deployment = $dbForProject->getDocument('deployments', $deployment->getId());
|
||||
|
||||
@@ -105,7 +106,7 @@ class Update extends Action
|
||||
}
|
||||
|
||||
if ($deployment->getAttribute('status') !== 'canceled') {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration)));
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,14 +124,13 @@ class Update extends Action
|
||||
}
|
||||
|
||||
/**
|
||||
* The sparse update marking a build canceled. Jobs-backed builds (identified
|
||||
* by a buildPath set at submission) have no cancel worker to write the
|
||||
* closing log line the executor's Builds worker adds, so it is appended here;
|
||||
* executor deployments get it from their worker.
|
||||
* The sparse update marking a build canceled. Jobs-backed builds have no
|
||||
* cancel worker to write the closing log line the executor's Builds worker
|
||||
* adds, so it is appended here; executor deployments get it from their worker.
|
||||
*
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function cancel(Document $deployment, int $duration): array
|
||||
private function cancel(Document $deployment, int $duration, bool $appendLog): array
|
||||
{
|
||||
$update = [
|
||||
'buildEndedAt' => DateTime::now(),
|
||||
@@ -138,7 +138,7 @@ class Update extends Action
|
||||
'status' => 'canceled',
|
||||
];
|
||||
|
||||
if ($deployment->getAttribute('buildPath', '') !== '') {
|
||||
if ($appendLog) {
|
||||
$logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n";
|
||||
$update['buildLogs'] = \substr($logs, -APP_LOG_LENGTH_LIMIT);
|
||||
}
|
||||
|
||||
@@ -417,8 +417,7 @@ class Create extends Base
|
||||
}
|
||||
|
||||
$source = $deployment->getAttribute('buildPath', '');
|
||||
$extension = str_ends_with($source, '.tar') ? 'tar' : 'tar.gz';
|
||||
$command = $version === 'v2' ? '' : "cp /tmp/code.$extension /mnt/code/code.$extension && nohup helpers/start.sh \"$command\"";
|
||||
$command = $version === 'v2' ? '' : "cp /tmp/code.* /mnt/code/ && nohup helpers/start.sh \"$command\"";
|
||||
try {
|
||||
$executionResponse = $executor->createExecution(
|
||||
projectId: $project->getId(),
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Workers;
|
||||
|
||||
use Appwrite\Deployment\Backend\Orchestrator;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Func as FunctionMessage;
|
||||
use Appwrite\Event\Message\Jobs as JobsMessage;
|
||||
@@ -266,8 +265,12 @@ class Jobs extends Action
|
||||
Device $deviceForBuilds,
|
||||
VcsFactory $vcsFactory,
|
||||
): Document {
|
||||
$path = Orchestrator::buildPath($project->getId(), $deployment->getId());
|
||||
$size = $deviceForBuilds->exists($path) ? $deviceForBuilds->getFileSize($path) : 0;
|
||||
$path = (string) $deployment->getAttribute('buildPath', '');
|
||||
if ($path === '' || ! $deviceForBuilds->exists($path)) {
|
||||
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build produced no output artifact.', $usage, $publisherForUsage, $vcsFactory);
|
||||
}
|
||||
|
||||
$size = $deviceForBuilds->getFileSize($path);
|
||||
|
||||
$limit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000');
|
||||
if ($limit !== 0 && $size > $limit) {
|
||||
|
||||
@@ -100,14 +100,19 @@ class Get extends Action
|
||||
throw new Exception(Exception::DEPLOYMENT_NOT_FOUND);
|
||||
}
|
||||
|
||||
$contentType = $type === 'source' ? 'application/gzip' : 'application/octet-stream';
|
||||
$filename = $type === 'source'
|
||||
? $deploymentId . '-source.tar.gz'
|
||||
: $deploymentId . '-output-' . \basename($path);
|
||||
|
||||
$size = $device->getFileSize($path);
|
||||
$rangeHeader = $request->getHeaderLine('range');
|
||||
|
||||
$response
|
||||
->setContentType('application/gzip')
|
||||
->setContentType($contentType)
|
||||
->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days
|
||||
->addHeader('X-Peak', \memory_get_peak_usage())
|
||||
->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '-' . $type . '.tar.gz"');
|
||||
->addHeader('Content-Disposition', 'attachment; filename="' . $filename . '"');
|
||||
|
||||
if (!empty($rangeHeader)) {
|
||||
$start = $request->getRangeStart();
|
||||
|
||||
@@ -550,8 +550,7 @@ class Functions extends Action
|
||||
}
|
||||
|
||||
$source = $deployment->getAttribute('buildPath', '');
|
||||
$extension = str_ends_with($source, '.tar') ? 'tar' : 'tar.gz';
|
||||
$command = $version === 'v2' ? '' : "cp /tmp/code.$extension /mnt/code/code.$extension && nohup helpers/start.sh \"$command\"";
|
||||
$command = $version === 'v2' ? '' : "cp /tmp/code.* /mnt/code/ && nohup helpers/start.sh \"$command\"";
|
||||
try {
|
||||
$executionResponse = $executor->createExecution(
|
||||
projectId: $project->getId(),
|
||||
|
||||
@@ -899,7 +899,7 @@ final class FunctionsConsoleClientTest extends Scope
|
||||
|
||||
$response = $this->getDeploymentDownload($functionId, $deploymentId, 'output');
|
||||
$this->assertEquals(200, $response['headers']['status-code']);
|
||||
$this->assertEquals('application/gzip', $response['headers']['content-type']);
|
||||
$this->assertEquals('application/octet-stream', $response['headers']['content-type']);
|
||||
$this->assertGreaterThan(0, $response['headers']['content-length']);
|
||||
$this->assertGreaterThan(0, \strlen($response['body']));
|
||||
|
||||
|
||||
@@ -2341,7 +2341,7 @@ final class SitesCustomServerTest extends Scope
|
||||
|
||||
$response = $this->getDeploymentDownload($siteId, $deploymentId, 'output');
|
||||
$this->assertEquals(200, $response['headers']['status-code']);
|
||||
$this->assertEquals('application/gzip', $response['headers']['content-type']);
|
||||
$this->assertEquals('application/octet-stream', $response['headers']['content-type']);
|
||||
$this->assertGreaterThan(0, $response['headers']['content-length']);
|
||||
$this->assertGreaterThan(0, \strlen($response['body']));
|
||||
|
||||
|
||||
Reference in new issue
Block a user