diff --git a/app/controllers/general.php b/app/controllers/general.php index bb47346e67..fd622a4784 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -552,7 +552,6 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S 'site' => '', }; $source = $deployment->getAttribute('buildPath', ''); - $extension = str_ends_with($source, '.tar') ? 'tar' : 'tar.gz'; $startCommand = $runtime['startCommand']; if ($type === 'site') { @@ -573,7 +572,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S $runtimeEntrypoint = match ($version) { 'v2' => '', - default => "cp /tmp/code.$extension /mnt/code/code.$extension && nohup helpers/start.sh \"$startCommand\"", + default => "cp /tmp/code.* /mnt/code/ && nohup helpers/start.sh \"$startCommand\"", }; $entrypoint = match ($type) { diff --git a/src/Appwrite/Deployment/Backend/Orchestrator.php b/src/Appwrite/Deployment/Backend/Orchestrator.php index 0288224953..a003b036a0 100644 --- a/src/Appwrite/Deployment/Backend/Orchestrator.php +++ b/src/Appwrite/Deployment/Backend/Orchestrator.php @@ -24,7 +24,7 @@ use Utopia\System\System; * + unarchive, run by the sidecar) — a GET has no request-body cap, so large * sources are fine. The build output and package-manager cache, by default, * go on a mounted volume: the builds storage volume is attached to the build - * worker at its Appwrite path, so build.sh writes code.tar.gz + the cache + * worker at its Appwrite path, so build.sh writes its artifact + the cache * squashfs straight onto the volume Appwrite already reads. That keeps the * multi-hundred-MB output off the (capped) HTTP upload path and out of the * Appwrite process. Deployments that need a different strategy (e.g. S3 @@ -82,8 +82,6 @@ readonly class Orchestrator extends Backend $deployment = $this->upload($resource, $deployment); $this->deactivateOthers($resource, $deployment); - // Pre-declare buildPath so build.sh writes output straight onto the - // mounted builds volume, at the path this deployment expects. $deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ 'status' => 'waiting', 'buildPath' => static::buildPath($this->project->getId(), $deployment->getId()), @@ -168,6 +166,7 @@ readonly class Orchestrator extends Backend $command = $deployment->getAttribute('buildCommands', ''); $env = self::variables($project, $function, $deployment, $runtime, $cpus, $memory, $endpoint, $timeout) + [ 'OPEN_RUNTIMES_BUILD_INPUT_DIR' => '/mnt/code/source', + 'OPEN_RUNTIMES_BUILD_COMPRESSION' => static::compression(), ] + $output['environment']; return [ @@ -213,13 +212,37 @@ readonly class Orchestrator extends Backend } /** - * The build output path on the builds volume. build.sh writes code.tar.gz - * into OPEN_RUNTIMES_BUILD_OUTPUT_DIR (this file's directory); pre-computed - * so it can be persisted on the deployment before the job is submitted. + * The build output directory on the builds volume. The produced artifact's + * complete path is discovered and persisted after the job finishes. + */ + public static function outputDirectory(string $projectId, string $deploymentId): string + { + return APP_STORAGE_BUILDS . "/app-{$projectId}/{$deploymentId}"; + } + + /** + * The build output path on the builds volume, declared at submission. */ public static function buildPath(string $projectId, string $deploymentId): string { - return APP_STORAGE_BUILDS . "/app-{$projectId}/{$deploymentId}/code.tar.gz"; + return static::outputDirectory($projectId, $deploymentId) . '/' . static::artifact(); + } + + /** + * The artifact filename build.sh produces for the configured compression. + */ + public static function artifact(): string + { + return match (static::compression()) { + 'none' => 'code.tar', + 'squashfs' => 'code.sqfs', + default => 'code.tar.gz', + }; + } + + protected static function compression(): string + { + return System::getEnv('_APP_COMPUTE_BUILD_COMPRESSION', 'gzip'); } /** @@ -237,9 +260,9 @@ readonly class Orchestrator extends Backend } /** - * Where build.sh's output (code.tar.gz) and package-manager cache + * Where build.sh's output artifact and package-manager cache * (a squashfs) land, and what the job needs to get them there. The - * default mounts the shared builds volume at buildPath()/cachePath(); + * default mounts the shared builds volume at outputDirectory()/cachePath(); * build.sh only cares that OPEN_RUNTIMES_BUILD_OUTPUT_DIR/_CACHE_ARTIFACT * point somewhere on its local filesystem, volume-backed or not — so a * strategy without a shared volume (e.g. S3) instead points them at a @@ -268,7 +291,7 @@ readonly class Orchestrator extends Backend ], 'artifacts' => [], 'environment' => [ - 'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => \dirname(static::buildPath($projectId, $deploymentId)), + 'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => static::outputDirectory($projectId, $deploymentId), 'OPEN_RUNTIMES_BUILD_CACHE_ARTIFACT' => static::cachePath($projectId, $cacheKey), ], ]; diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php index f089ac9a57..2b950d0afe 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php @@ -125,11 +125,16 @@ class Get extends Action throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } + $contentType = $type === 'source' ? 'application/gzip' : 'application/octet-stream'; + $filename = $type === 'source' + ? $deploymentId . '-source.tar.gz' + : $deploymentId . '-output-' . \basename($path); + $response - ->setContentType('application/gzip') + ->setContentType($contentType) ->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days ->addHeader('X-Peak', \memory_get_peak_usage()) - ->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '-' . $type . '.tar.gz"'); + ->addHeader('Content-Disposition', 'attachment; filename="' . $filename . '"'); $size = $device->getFileSize($path); $rangeHeader = $request->getHeaderLine('range'); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php index 961e3182b6..1380428bf7 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Status; use Appwrite\Deployment\Backend; +use Appwrite\Deployment\Backend\Orchestrator; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; @@ -92,7 +93,7 @@ class Update extends Action $duration = $endTime->getTimestamp() - $startTime->getTimestamp(); try { - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration))); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); } catch (TransactionException) { $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); @@ -105,7 +106,7 @@ class Update extends Action } if ($deployment->getAttribute('status') !== 'canceled') { - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration))); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration, $deployments instanceof Orchestrator))); } } @@ -123,14 +124,13 @@ class Update extends Action } /** - * The sparse update marking a build canceled. Jobs-backed builds (identified - * by a buildPath set at submission) have no cancel worker to write the - * closing log line the executor's Builds worker adds, so it is appended here; - * executor deployments get it from their worker. + * The sparse update marking a build canceled. Jobs-backed builds have no + * cancel worker to write the closing log line the executor's Builds worker + * adds, so it is appended here; executor deployments get it from their worker. * * @return array */ - private function cancel(Document $deployment, int $duration): array + private function cancel(Document $deployment, int $duration, bool $appendLog): array { $update = [ 'buildEndedAt' => DateTime::now(), @@ -138,7 +138,7 @@ class Update extends Action 'status' => 'canceled', ]; - if ($deployment->getAttribute('buildPath', '') !== '') { + if ($appendLog) { $logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n"; $update['buildLogs'] = \substr($logs, -APP_LOG_LENGTH_LIMIT); } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php index 44f2b7b362..6fb5783194 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php @@ -417,8 +417,7 @@ class Create extends Base } $source = $deployment->getAttribute('buildPath', ''); - $extension = str_ends_with($source, '.tar') ? 'tar' : 'tar.gz'; - $command = $version === 'v2' ? '' : "cp /tmp/code.$extension /mnt/code/code.$extension && nohup helpers/start.sh \"$command\""; + $command = $version === 'v2' ? '' : "cp /tmp/code.* /mnt/code/ && nohup helpers/start.sh \"$command\""; try { $executionResponse = $executor->createExecution( projectId: $project->getId(), diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index 07a390c7f6..29d1a4842a 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -2,7 +2,6 @@ namespace Appwrite\Platform\Modules\Functions\Workers; -use Appwrite\Deployment\Backend\Orchestrator; use Appwrite\Event\Event; use Appwrite\Event\Message\Func as FunctionMessage; use Appwrite\Event\Message\Jobs as JobsMessage; @@ -266,8 +265,12 @@ class Jobs extends Action Device $deviceForBuilds, VcsFactory $vcsFactory, ): Document { - $path = Orchestrator::buildPath($project->getId(), $deployment->getId()); - $size = $deviceForBuilds->exists($path) ? $deviceForBuilds->getFileSize($path) : 0; + $path = (string) $deployment->getAttribute('buildPath', ''); + if ($path === '' || ! $deviceForBuilds->exists($path)) { + return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build produced no output artifact.', $usage, $publisherForUsage, $vcsFactory); + } + + $size = $deviceForBuilds->getFileSize($path); $limit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000'); if ($limit !== 0 && $size > $limit) { diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php index 79ef3d2b1d..40bdbe8e8e 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php @@ -100,14 +100,19 @@ class Get extends Action throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } + $contentType = $type === 'source' ? 'application/gzip' : 'application/octet-stream'; + $filename = $type === 'source' + ? $deploymentId . '-source.tar.gz' + : $deploymentId . '-output-' . \basename($path); + $size = $device->getFileSize($path); $rangeHeader = $request->getHeaderLine('range'); $response - ->setContentType('application/gzip') + ->setContentType($contentType) ->addHeader('Cache-Control', 'private, max-age=3888000') // 45 days ->addHeader('X-Peak', \memory_get_peak_usage()) - ->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '-' . $type . '.tar.gz"'); + ->addHeader('Content-Disposition', 'attachment; filename="' . $filename . '"'); if (!empty($rangeHeader)) { $start = $request->getRangeStart(); diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index c09b95b6f0..bdc93700af 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -550,8 +550,7 @@ class Functions extends Action } $source = $deployment->getAttribute('buildPath', ''); - $extension = str_ends_with($source, '.tar') ? 'tar' : 'tar.gz'; - $command = $version === 'v2' ? '' : "cp /tmp/code.$extension /mnt/code/code.$extension && nohup helpers/start.sh \"$command\""; + $command = $version === 'v2' ? '' : "cp /tmp/code.* /mnt/code/ && nohup helpers/start.sh \"$command\""; try { $executionResponse = $executor->createExecution( projectId: $project->getId(), diff --git a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php index 8f8e9e7f35..e350b500de 100644 --- a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php @@ -899,7 +899,7 @@ final class FunctionsConsoleClientTest extends Scope $response = $this->getDeploymentDownload($functionId, $deploymentId, 'output'); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals('application/gzip', $response['headers']['content-type']); + $this->assertEquals('application/octet-stream', $response['headers']['content-type']); $this->assertGreaterThan(0, $response['headers']['content-length']); $this->assertGreaterThan(0, \strlen($response['body'])); diff --git a/tests/e2e/Services/Sites/SitesCustomServerTest.php b/tests/e2e/Services/Sites/SitesCustomServerTest.php index 59efaa7a6a..10fe086f37 100644 --- a/tests/e2e/Services/Sites/SitesCustomServerTest.php +++ b/tests/e2e/Services/Sites/SitesCustomServerTest.php @@ -2341,7 +2341,7 @@ final class SitesCustomServerTest extends Scope $response = $this->getDeploymentDownload($siteId, $deploymentId, 'output'); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals('application/gzip', $response['headers']['content-type']); + $this->assertEquals('application/octet-stream', $response['headers']['content-type']); $this->assertGreaterThan(0, $response['headers']['content-length']); $this->assertGreaterThan(0, \strlen($response['body']));