feat(compute): build manual-upload functions via open-runtimes jobs-service

Adds an alternative, stateless build backend for manual-upload function
deployments using the open-runtimes jobs-service (orchestrator) instead of
the executor.

The build runs as a stateless job: source, output, and the package-manager
build cache all cross the boundary via the jobs-service artifacts system
(presigned download/upload), and progress arrives as CloudEvents callbacks
handled by a new worker-jobs. createDeployment submits the job in the request
flow; the executor path is untouched for every other build trigger.

- Orchestrator\Jobs: thin jobs-service HTTP client (POST /v1/jobs)
- Appwrite\Compute\Job: builds the job payload (image, command, env, artifacts)
- Appwrite\Deployment\Token: signed presign tokens (source/output/cache)
- Functions/Http/Deployments/Archive: token-authed output + cache upload (PUT)
- Functions/Http/Jobs/Event: HMAC-verified CloudEvents callback ingress
- Functions/Workers/Jobs: applies log/exit callbacks under a per-deployment
  lock; on completion fires the deployment.update fan-out (webhooks +
  event-functions + realtime) and refreshes latestDeployment*, mirroring the
  executor Builds worker
- Deployments/Download/Get: presigned source + build-cache download
- Build cache via OPEN_RUNTIMES_BUILD_* dir overrides: pure artifacts, no shell
  bridge (per-function npm/yarn/pnpm cache reused across builds)
- DI/queue wiring, docker services (orchestrator-jobs, worker-jobs), test-build.sh

Verified end-to-end: deploy -> ready -> activated -> executes (200); cache
miss -> save -> hit; latestDeploymentStatus + webhook/function fan-out fire.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
loks0nandClaude Opus 4.8 committed 2026-07-10 16:46:02 +01:00
1 parent d8b569b564
commit d1b1ab9fb0
27 files changed
+1401 -104

No files matched your search

+5
View File
@@ -117,6 +117,11 @@ _APP_SITES_TIMEOUT=30
_APP_EXECUTOR_HOST=http://exc1/v1
_APP_EXECUTOR_SECRET=your-secret-key
_APP_EXECUTOR_IMAGES=openruntimes/node:v5-22,openruntimes/static:v5-1
_APP_BUILDS_BACKEND=orchestrator
_APP_BUILDS_VOLUME=appwrite-builds
_APP_JOBS_HOST=http://orchestrator-jobs:8080
_APP_JOBS_SECRET=your-secret-key
_APP_JOBS_ENDPOINT=http://appwrite
_APP_BROWSER_HOST=http://appwrite-browser:3000/v1
_APP_DOCKER_HUB_USERNAME=
_APP_DOCKER_HUB_PASSWORD=
+1
View File
@@ -90,6 +90,7 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/queue-count-processing && \
chmod +x /usr/local/bin/queue-count-success && \
chmod +x /usr/local/bin/worker-builds && \
chmod +x /usr/local/bin/worker-jobs && \
chmod +x /usr/local/bin/worker-screenshots && \
chmod +x /usr/local/bin/worker-certificates && \
chmod +x /usr/local/bin/worker-databases && \
+47
View File
@@ -979,6 +979,53 @@ return [
'question' => '',
'filter' => ''
],
[
'name' => '_APP_BUILDS_BACKEND',
'description' => 'Backend that builds manual-upload function deployments: "executor" (default; the open-runtimes executor, via the Builds worker) or "orchestrator" (the open-runtimes jobs-service, submitted in the request flow). Other build flows always use the executor.',
'introduction' => '1.9.0',
'default' => 'executor',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_BUILDS_VOLUME',
'description' => 'The Docker volume (or Kubernetes PersistentVolumeClaim) holding build storage, attached to jobs-service build workers so they write output directly onto it. Must match the storage the "builds" device is backed by. Only used when _APP_BUILDS_BACKEND is "orchestrator".',
'introduction' => '1.9.0',
'default' => 'appwrite-builds',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_JOBS_HOST',
'description' => 'The host used by Appwrite to communicate with the open-runtimes jobs-service that builds manual-upload function deployments.',
'introduction' => '1.9.0',
'default' => 'http://orchestrator-jobs:8080',
'required' => false,
'overwrite' => true,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_JOBS_SECRET',
'description' => 'The secret used to authenticate with the jobs-service and to sign/verify job callback (HMAC) requests. Make sure to change this.',
'introduction' => '1.9.0',
'default' => 'your-secret-key',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_JOBS_ENDPOINT',
'description' => 'Internal Appwrite endpoint the jobs-service (and the containers it spawns) use to reach the API over the Docker network for presigned artifact + callback URLs.',
'introduction' => '1.9.0',
'default' => 'http://appwrite',
'required' => false,
'overwrite' => true,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_BROWSER_HOST',
'description' => 'The host used by Appwrite to communicate with the browser service for screenshots.',
+18
View File
@@ -9,6 +9,7 @@ use Appwrite\Event\Publisher\Database as DatabasePublisher;
use Appwrite\Event\Publisher\Delete as DeletePublisher;
use Appwrite\Event\Publisher\Execution as ExecutionPublisher;
use Appwrite\Event\Publisher\Func as FunctionPublisher;
use Appwrite\Event\Publisher\Jobs as JobsPublisher;
use Appwrite\Event\Publisher\Mail as MailPublisher;
use Appwrite\Event\Publisher\Messaging as MessagingPublisher;
use Appwrite\Event\Publisher\Migration as MigrationPublisher;
@@ -18,10 +19,13 @@ use Appwrite\Event\Publisher\StatsResources as StatsResourcesPublisher;
use Appwrite\Event\Publisher\Usage as UsagePublisher;
use Appwrite\Platform\Modules\Storage\Config\StorageCacheControl;
use Executor\Executor;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis;
use Utopia\Cache\Adapter\Pool as CachePool;
use Utopia\Cache\Adapter\Sharding;
use Utopia\Cache\Cache;
use Utopia\Client;
use Utopia\Client\Adapter\Curl\Client as CurlAdapter;
use Utopia\Config\Config;
use Utopia\Console;
use Utopia\Database\Document;
@@ -66,6 +70,13 @@ $container->set('localeCodes', fn () => array_map(fn ($locale) => $locale['code'
$container->set('executor', fn () => new Executor(), []);
$container->set('jobs', fn () => new Jobs(
(new Client(new CurlAdapter()))
->withBaseUri(System::getEnv('_APP_JOBS_HOST', ''))
->withBearerAuth(System::getEnv('_APP_JOBS_SECRET', ''))
->withTimeout(30)
), []);
$container->set('telemetry', fn () => new NoTelemetry(), []);
$container->set('authorization', fn () => new Authorization(), []);
@@ -131,6 +142,11 @@ $container->set('publisherForBuilds', fn (Publisher $publisher) => new BuildPubl
new Queue(System::getEnv('_APP_BUILDS_QUEUE_NAME', Event::BUILDS_QUEUE_NAME))
), ['publisher']);
$container->set('publisherForJobs', fn (Publisher $publisher) => new JobsPublisher(
$publisher,
new Queue(System::getEnv('_APP_JOBS_QUEUE_NAME', Event::JOBS_QUEUE_NAME))
), ['publisher']);
$container->set('publisherForDatabase', fn (Publisher $publisherDatabases) => new DatabasePublisher(
$publisherDatabases,
new Queue(System::getEnv('_APP_DATABASE_QUEUE_NAME', Event::DATABASE_QUEUE_NAME))
@@ -333,6 +349,8 @@ function getDevice(string $root, string $connection = ''): Device
}
}
$container->set('geodb', fn ($register) => $register->get('geodb'), ['register']);
$container->set('passwordsDictionary', fn ($register) => $register->get('passwordsDictionary'), ['register']);
$container->set('servers', function () {
+6
View File
@@ -580,6 +580,12 @@ return function (Container $context): void {
if (! \is_string($projectId)) {
$projectId = $request->getHeaderLine('x-appwrite-project', '');
}
// For non-GET requests getParam() reads the body, so a project passed
// as a query parameter (e.g. presigned artifact URLs) is only visible
// via getQuery().
if (empty($projectId)) {
$projectId = (string) $request->getQuery('project', '');
}
// Backwards compatibility for new services, originally project resources
// These endpoints moved from /v1/projects/:projectId/<resource> to /v1/<resource>
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
exec php /usr/src/code/app/worker.php jobs "$@"
+5 -3
View File
@@ -44,7 +44,7 @@
}
],
"require": {
"php": ">=8.3.0",
"php": ">=8.4.0",
"ext-curl": "*",
"ext-imagick": "*",
"ext-mbstring": "*",
@@ -94,7 +94,7 @@
"utopia-php/storage": "2.*",
"utopia-php/system": "0.10.*",
"utopia-php/telemetry": "^0.4",
"utopia-php/vcs": "4.*",
"utopia-php/vcs": "^4.3.0",
"utopia-php/websocket": "1.0.*",
"matomo/device-detector": "6.4.*",
"dragonmantank/cron-expression": "3.4.*",
@@ -104,7 +104,9 @@
"spomky-labs/otphp": "11.*",
"webonyx/graphql-php": "15.32.*",
"league/csv": "9.14.*",
"enshrined/svg-sanitize": "0.22.*"
"enshrined/svg-sanitize": "0.22.*",
"utopia-php/client": "^0.2.2",
"open-runtimes/sdk-for-php": "^0.2.0"
},
"require-dev": {
"ext-fileinfo": "*",
Generated
+154 -8
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "d67184492e8ec8670d0e31f02d10b421",
"content-hash": "a453a68b2d3ee2f962af0f040f000a8a",
"packages": [
{
"name": "adhocore/jwt",
@@ -1224,6 +1224,49 @@
],
"time": "2023-11-08T09:30:43+00:00"
},
{
"name": "open-runtimes/sdk-for-php",
"version": "0.2.0",
"source": {
"type": "git",
"url": "https://github.com/open-runtimes/sdk-for-php.git",
"reference": "e5cf7852630b69b419f8705482d8ff663ab7ab42"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/open-runtimes/sdk-for-php/zipball/e5cf7852630b69b419f8705482d8ff663ab7ab42",
"reference": "e5cf7852630b69b419f8705482d8ff663ab7ab42",
"shasum": ""
},
"require": {
"ext-curl": "*",
"ext-json": "*",
"php": ">=8.4",
"utopia-php/client": "^0.2.2"
},
"require-dev": {
"laravel/pint": "^1.22",
"phpstan/phpstan": "^2.1",
"phpunit/phpunit": "^13.1",
"rector/rector": "^2.4"
},
"type": "library",
"autoload": {
"psr-4": {
"OpenRuntimes\\Orchestrator\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"description": "PHP SDK for the Open Runtimes orchestrator Jobs API.",
"support": {
"issues": "https://github.com/open-runtimes/sdk-for-php/issues",
"source": "https://github.com/open-runtimes/sdk-for-php/tree/0.2.0"
},
"time": "2026-07-08T11:28:50+00:00"
},
{
"name": "open-telemetry/api",
"version": "1.9.0",
@@ -3539,6 +3582,63 @@
},
"time": "2026-06-10T16:25:51+00:00"
},
{
"name": "utopia-php/client",
"version": "0.2.2",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/client.git",
"reference": "a5e1ff4e76003ba91be8b76208695469481cf633"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/client/zipball/a5e1ff4e76003ba91be8b76208695469481cf633",
"reference": "a5e1ff4e76003ba91be8b76208695469481cf633",
"shasum": ""
},
"require": {
"php": ">=8.4",
"psr/http-client": "^1.0",
"psr/http-factory": "^1.0",
"psr/http-message": "^1.1 || ^2.0",
"utopia-php/pools": "^1.0",
"utopia-php/psr7": "^0.1",
"utopia-php/span": "^1.1 || ^3.0 || ^4.0"
},
"require-dev": {
"swoole/ide-helper": "^6.0"
},
"suggest": {
"ext-curl": "Required to use the cURL HTTP client adapter.",
"ext-simplexml": "Required to decode XML responses with Response::xml().",
"ext-swoole": "Required to use the Swoole coroutine HTTP client adapter."
},
"type": "library",
"autoload": {
"psr-4": {
"Utopia\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"description": "A lightweight PSR-18 HTTP client with cURL and Swoole coroutine backends",
"keywords": [
"client",
"curl",
"http",
"php",
"psr-18",
"swoole",
"utopia"
],
"support": {
"issues": "https://github.com/utopia-php/client/issues",
"source": "https://github.com/utopia-php/client/tree/0.2.2"
},
"time": "2026-07-06T11:47:31+00:00"
},
{
"name": "utopia-php/compression",
"version": "0.1.5",
@@ -4676,6 +4776,52 @@
},
"time": "2020-10-24T07:04:59+00:00"
},
{
"name": "utopia-php/psr7",
"version": "0.1.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/psr7.git",
"reference": "079dd7f9a51a769ab560d8b6057e276ecd5dae41"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/psr7/zipball/079dd7f9a51a769ab560d8b6057e276ecd5dae41",
"reference": "079dd7f9a51a769ab560d8b6057e276ecd5dae41",
"shasum": ""
},
"require": {
"php": ">=8.4",
"psr/http-factory": "^1.0",
"psr/http-message": "^1.1 || ^2.0"
},
"suggest": {
"ext-simplexml": "Required to decode XML responses with Response::xml()."
},
"type": "library",
"autoload": {
"psr-4": {
"Utopia\\Psr7\\": "src/Psr7/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"description": "PSR-7 HTTP message implementations and PSR-17 factories for Utopia",
"keywords": [
"http",
"php",
"psr-17",
"psr-7",
"utopia"
],
"support": {
"issues": "https://github.com/utopia-php/psr7/issues",
"source": "https://github.com/utopia-php/psr7/tree/0.1.0"
},
"time": "2026-07-06T11:47:31+00:00"
},
{
"name": "utopia-php/query",
"version": "0.1.1",
@@ -5133,16 +5279,16 @@
},
{
"name": "utopia-php/vcs",
"version": "4.2.4",
"version": "4.4.1",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/vcs.git",
"reference": "f19c61daa1e9594734d85ae25b3e90ba2b350d33"
"reference": "75f36b9db5c957829a3625a91293b164f74faa0d"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/vcs/zipball/f19c61daa1e9594734d85ae25b3e90ba2b350d33",
"reference": "f19c61daa1e9594734d85ae25b3e90ba2b350d33",
"url": "https://api.github.com/repos/utopia-php/vcs/zipball/75f36b9db5c957829a3625a91293b164f74faa0d",
"reference": "75f36b9db5c957829a3625a91293b164f74faa0d",
"shasum": ""
},
"require": {
@@ -5176,9 +5322,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/vcs/issues",
"source": "https://github.com/utopia-php/vcs/tree/4.2.4"
"source": "https://github.com/utopia-php/vcs/tree/4.4.1"
},
"time": "2026-06-25T13:27:19+00:00"
"time": "2026-07-08T18:15:35+00:00"
},
{
"name": "utopia-php/websocket",
@@ -8372,7 +8518,7 @@
"prefer-stable": true,
"prefer-lowest": false,
"platform": {
"php": ">=8.3.0",
"php": ">=8.4.0",
"ext-curl": "*",
"ext-imagick": "*",
"ext-mbstring": "*",
+12
View File
@@ -113,6 +113,18 @@ services:
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
appwrite-worker-jobs:
build:
context: .
target: development
args:
DEBUG: false
TESTING: true
VERSION: dev
image: appwrite-dev
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
appwrite-worker-screenshots:
build:
context: .
+72 -1
View File
@@ -170,6 +170,11 @@ services:
- _APP_DOMAIN_SITES
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
- _APP_BUILDS_BACKEND
- _APP_BUILDS_VOLUME
- _APP_JOBS_HOST
- _APP_JOBS_SECRET
- _APP_JOBS_ENDPOINT
- _APP_LOGGING_CONFIG
- _APP_LOCKING_ENABLED
- _APP_MAINTENANCE_INTERVAL
@@ -512,6 +517,47 @@ services:
- _APP_DOMAIN_SITES
extra_hosts:
- host.docker.internal:host-gateway
appwrite-worker-jobs:
entrypoint: worker-jobs
logging:
driver: json-file
options:
max-file: "5"
max-size: 10m
container_name: appwrite-worker-jobs
image: ${_APP_IMAGE:-appwrite/appwrite}:${_APP_VERSION:-latest}
networks:
- appwrite
volumes:
- appwrite-builds:/storage/builds:rw
depends_on:
- redis
- ${_APP_DB_HOST:-mongodb}
environment:
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_WORKER_MAX_COROUTINES=8
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
- _APP_JOBS_HOST
- _APP_JOBS_SECRET
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_DB_ADAPTER
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_LOGGING_CONFIG
- _APP_DATABASE_SHARED_TABLES
extra_hosts:
- host.docker.internal:host-gateway
appwrite-worker-screenshots:
entrypoint: worker-screenshots
logging:
@@ -1188,6 +1234,28 @@ services:
timeout: 3s
retries: 20
start_period: 5s
orchestrator-jobs:
container_name: orchestrator-jobs
hostname: orchestrator-jobs
logging:
driver: json-file
options:
max-file: "5"
max-size: 10m
image: ghcr.io/open-runtimes/orchestrator/jobs-service:0.4.2
restart: unless-stopped
user: root
networks:
- appwrite
volumes:
- /var/run/docker.sock:/var/run/docker.sock
environment:
- ORCHESTRATOR_BACKEND=docker
- ORCHESTRATOR_NETWORK=appwrite
- ARTIFACT_ENDPOINT=http://orchestrator-jobs:8080
- JOB_SIDECAR_IMAGE=ghcr.io/open-runtimes/orchestrator/job-sidecar:0.4.2
mariadb:
image: mariadb:10.11
container_name: appwrite-mariadb
@@ -1354,7 +1422,10 @@ volumes:
appwrite-certificates: null
appwrite-functions: null
appwrite-sites: null
appwrite-builds: null
# Pinned name (unprefixed) so the jobs-service can attach it to build workers
# by a stable name across environments (see _APP_BUILDS_VOLUME).
appwrite-builds:
name: appwrite-builds
appwrite-config: null
appwrite-models: null
openruntimes-node-modules-cache: null
+219
View File
@@ -0,0 +1,219 @@
<?php
namespace Appwrite\Compute;
use Ahc\Jwt\JWT;
use Appwrite\Deployment\Token;
use OpenRuntimes\Orchestrator\Enum\CallbackEvent;
use OpenRuntimes\Orchestrator\Model\Artifact\DownloadArtifact;
use OpenRuntimes\Orchestrator\Model\Artifact\UnarchiveArtifact;
use OpenRuntimes\Orchestrator\Model\Callback;
use OpenRuntimes\Orchestrator\Model\Volume;
use Utopia\Config\Config;
use Utopia\Database\Document;
use Utopia\System\System;
/**
* Builds an open-runtimes jobs-service job payload for a function deployment.
*
* Source crosses the boundary via the artifacts system (presigned GET download
* + unarchive, run by the sidecar) — a GET has no request-body cap, so large
* sources are fine. The build output and package-manager cache instead go on a
* mounted volume: the builds storage volume is attached to the build worker at
* its Appwrite path, so build.sh writes code.tar.gz + the cache squashfs
* straight onto the volume Appwrite already reads. That keeps the multi-hundred-MB
* output off the (capped) HTTP upload path and out of the Appwrite process.
*
* Covers function deployments whose source is a tarball: manual upload,
* duplicate/rebuild, and templates (public GitHub tarball via `$template`).
*/
final class Job
{
/**
* @return array<string, mixed> Named arguments for OpenRuntimes\Orchestrator\Jobs::create().
*/
public static function build(
Document $project,
Document $function,
Document $deployment,
array $platform,
?Document $template = null,
): array {
$projectId = $project->getId();
$deploymentId = $deployment->getId();
$timeout = (int) System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900);
$version = $function->getAttribute('version', 'v2');
$runtime = self::runtime($function, $version);
$spec = Config::getParam('specifications')[$function->getAttribute('buildSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT)];
$cpus = (float) ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT);
$memory = \max((int) ($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT), 1024);
// The jobs-service (and the containers it spawns) reach Appwrite over
// the internal Docker network, so the presigned + callback URLs use an
// internal endpoint when configured, falling back to the public host.
$protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https';
$endpoint = System::getEnv('_APP_JOBS_ENDPOINT', "$protocol://{$platform['apiHostname']}");
// Source artifacts: manual/duplicate deployments download a presigned
// tarball from Appwrite; template deployments pull the public GitHub
// tarball directly (the unarchive auto-strips the "{repo}-{ref}/" wrapper
// and, via subdir, extracts just the template's rootDirectory). Either
// way the build input lands in /mnt/code/source.
if ($template !== null) {
$owner = $template->getAttribute('ownerName', '');
$repository = $template->getAttribute('repositoryName', '');
$reference = $template->getAttribute('referenceValue', '');
$rootDirectory = \trim($template->getAttribute('rootDirectory', ''), '/');
$sourceArtifacts = [
new DownloadArtifact(id: 'source', in: "https://codeload.github.com/{$owner}/{$repository}/tar.gz/{$reference}", out: 'source.tar.gz'),
new UnarchiveArtifact(id: 'extract', in: 'source.tar.gz', out: 'source', subdir: $rootDirectory !== '' ? $rootDirectory : null),
];
} else {
// Presigned source-download URL (GET, no request-body cap), fetched by
// the sidecar. Bound to this deployment + direction; valid for the whole
// build window plus transfer slack.
$ttl = $timeout + 300;
$base = "{$endpoint}/v1/functions/{$function->getId()}/deployments/{$deploymentId}";
$sourceUrl = "{$base}/download?" . \http_build_query([
'type' => Token::TYPE_SOURCE,
'project' => $projectId,
'token' => Token::sign($deploymentId, Token::TYPE_SOURCE, $ttl),
]);
$sourceArtifacts = [
new DownloadArtifact(id: 'source', in: $sourceUrl, out: 'source.tar.gz'),
new UnarchiveArtifact(id: 'extract', in: 'source.tar.gz', out: 'source'),
];
}
// Output + cache live on the mounted builds volume (see class doc): the
// build writes code.tar.gz to the deployment's build dir and the cache
// squashfs to its per-function keyed path — both directly on the volume
// Appwrite reads, so nothing large crosses the HTTP boundary.
$cacheKey = self::cacheKey($projectId, $function->getId(), $runtime['image'] ?? '');
$command = $deployment->getAttribute('buildCommands', '');
$env = self::variables($project, $function, $deployment, $runtime, $cpus, $memory, $endpoint, $timeout) + [
'OPEN_RUNTIMES_BUILD_INPUT_DIR' => '/mnt/code/source',
'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => \dirname(self::buildPath($projectId, $deploymentId)),
'OPEN_RUNTIMES_BUILD_CACHE_ARTIFACT' => self::cachePath($projectId, $cacheKey),
];
return [
'id' => self::id($projectId, $deploymentId),
'image' => $runtime['image'],
'command' => '/usr/local/server/helpers/build.sh ' . \escapeshellarg($command),
'cpu' => $cpus,
'memory' => $memory,
'timeoutSeconds' => $timeout,
'workspace' => '/mnt/code',
'meta' => [
'projectId' => $projectId,
'deploymentId' => $deploymentId,
'resourceId' => $function->getId(),
'resourceType' => 'functions',
],
// The orchestrator expects environment as a string->string map.
'environment' => \array_map('strval', $env),
'artifacts' => $sourceArtifacts,
// Attach the builds storage volume (Docker volume / K8s PVC named by
// _APP_BUILDS_VOLUME) to the worker at its Appwrite path so build.sh
// writes output + cache straight onto it.
'volumes' => [
new Volume(source: System::getEnv('_APP_BUILDS_VOLUME', 'appwrite-builds'), path: APP_STORAGE_BUILDS),
],
'callback' => new Callback(
url: "{$endpoint}/v1/jobs/event?" . \http_build_query(['project' => $projectId]),
events: [CallbackEvent::Log, CallbackEvent::Exit],
key: System::getEnv('_APP_JOBS_SECRET', ''),
),
];
}
/**
* The jobs-service job id for a deployment build (used to submit and cancel).
*/
public static function id(string $projectId, string $deploymentId): string
{
return "{$projectId}-{$deploymentId}-build";
}
/**
* The build output path on the builds volume. build.sh writes code.tar.gz
* into OPEN_RUNTIMES_BUILD_OUTPUT_DIR (this file's directory); pre-computed
* so it can be persisted on the deployment before the job is submitted.
*/
public static function buildPath(string $projectId, string $deploymentId): string
{
return APP_STORAGE_BUILDS . "/app-{$projectId}/{$deploymentId}/code.tar.gz";
}
/**
* Deterministic build-cache key, shared across a function's deployments so
* package-manager caches (npm/yarn/pnpm) survive between builds.
*/
public static function cacheKey(string $projectId, string $functionId, string $image): string
{
return \substr(\hash('sha256', "{$projectId}:{$functionId}:{$image}"), 0, 48);
}
public static function cachePath(string $projectId, string $cacheKey): string
{
return APP_STORAGE_BUILDS . "/app-{$projectId}/cache/{$cacheKey}.sqfs";
}
private static function runtime(Document $function, string $version): array
{
$runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []);
$runtime = $runtimes[$function->getAttribute('runtime')] ?? null;
if ($runtime === null) {
throw new \Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
}
return $runtime;
}
private static function variables(
Document $project,
Document $function,
Document $deployment,
array $runtime,
float $cpus,
int $memory,
string $endpoint,
int $timeout,
): array {
$vars = [];
foreach ($function->getAttribute('varsProject', []) as $var) {
$vars[$var->getAttribute('key')] = $var->getAttribute('value', '');
}
foreach ($function->getAttribute('vars', []) as $var) {
$vars[$var->getAttribute('key')] = $var->getAttribute('value', '');
}
$apiKey = (new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $timeout, 0))->encode([
'projectId' => $project->getId(),
'scopes' => $function->getAttribute('scopes', []),
]);
return \array_merge($vars, [
// Consumed by the open-runtimes build helper (build.sh).
'OPEN_RUNTIMES_ENTRYPOINT' => $deployment->getAttribute('entrypoint', ''),
'OPEN_RUNTIMES_OUTPUT_DIRECTORY' => $deployment->getAttribute('buildOutput', '') ?: $function->getAttribute('outputDirectory', ''),
'APPWRITE_VERSION' => APP_VERSION_STABLE,
'APPWRITE_REGION' => $project->getAttribute('region'),
'APPWRITE_DEPLOYMENT_TYPE' => $deployment->getAttribute('type', ''),
'APPWRITE_FUNCTION_API_ENDPOINT' => "{$endpoint}/v1",
'APPWRITE_FUNCTION_API_KEY' => API_KEY_EPHEMERAL . '_' . $apiKey,
'APPWRITE_FUNCTION_ID' => $function->getId(),
'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'),
'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(),
'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(),
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '',
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '',
'APPWRITE_FUNCTION_CPUS' => $cpus,
'APPWRITE_FUNCTION_MEMORY' => $memory,
'OPEN_RUNTIMES_NFT' => System::getEnv('_APP_OPEN_RUNTIMES_NFT', 'enabled'),
]);
}
}
+53
View File
@@ -0,0 +1,53 @@
<?php
namespace Appwrite\Deployment;
use Ahc\Jwt\JWT;
use Utopia\System\System;
/**
* Short-lived, self-contained presigned token for a deployment artifact.
*
* Signed with `_APP_OPENSSL_KEY_V1` (HS256), it lets the jobs-service sidecar
* hit the deployment source-download endpoint without a session or API key. The
* token binds the deployment id and the artifact type ("source"), so a leaked
* token cannot be repurposed for another deployment or artifact.
*/
final class Token
{
public const TYPE_SOURCE = 'source';
public static function sign(string $deploymentId, string $type, int $ttl): string
{
return (new JWT(self::key(), 'HS256', $ttl))->encode([
'deploymentId' => $deploymentId,
'type' => $type,
]);
}
/**
* Verify a token against the expected deployment id and artifact type.
* Returns true only when the signature, expiry, deployment id and type
* all match.
*/
public static function verify(string $token, string $deploymentId, string $type): bool
{
if ($token === '') {
return false;
}
try {
$payload = (new JWT(self::key(), 'HS256'))->decode($token);
} catch (\Throwable) {
return false;
}
return ($payload['deploymentId'] ?? '') === $deploymentId
&& ($payload['type'] ?? '') === $type;
}
private static function key(): string
{
return System::getEnv('_APP_OPENSSL_KEY_V1', '');
}
}
+3
View File
@@ -43,6 +43,9 @@ class Event
public const BUILDS_QUEUE_NAME = 'v1-builds';
public const BUILDS_CLASS_NAME = 'BuildsV1';
public const JOBS_QUEUE_NAME = 'v1-jobs';
public const JOBS_CLASS_NAME = 'JobsV1';
public const SCREENSHOTS_QUEUE_NAME = 'v1-screenshots';
public const SCREENSHOTS_CLASS_NAME = 'ScreenshotsV1';
+42
View File
@@ -0,0 +1,42 @@
<?php
namespace Appwrite\Event\Message;
use Utopia\Database\Document;
/**
* A single jobs-service callback (CloudEvents envelope) handed off to the jobs
* worker. `project` provides the DI context so the worker can resolve
* `dbForProject`; `event` is the CloudEvent `type`; `data` is the event `data`
* payload; `id` is the CloudEvent id used for de-duplication.
*/
final class Jobs extends Base
{
public function __construct(
public readonly Document $project,
public readonly string $id,
public readonly string $event,
public readonly array $data = [],
) {
}
public function toArray(): array
{
return [
'project' => $this->project->getArrayCopy(),
'id' => $this->id,
'event' => $this->event,
'data' => $this->data,
];
}
public static function fromArray(array $data): static
{
return new self(
project: new Document($data['project'] ?? []),
id: $data['id'] ?? '',
event: $data['event'] ?? '',
data: $data['data'] ?? [],
);
}
}
+22
View File
@@ -0,0 +1,22 @@
<?php
namespace Appwrite\Event\Publisher;
use Appwrite\Event\Message\Jobs as JobsMessage;
use Utopia\Queue\Publisher;
use Utopia\Queue\Queue;
readonly class Jobs extends Base
{
public function __construct(
Publisher $publisher,
protected Queue $queue
) {
parent::__construct($publisher);
}
public function enqueue(JobsMessage $message, ?Queue $queue = null): string|bool
{
return $this->publish($queue ?? $this->queue, $message);
}
}
@@ -2,6 +2,7 @@
namespace Appwrite\Platform\Modules\Functions\Http\Deployments;
use Appwrite\Compute\Job;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Build as BuildMessage;
use Appwrite\Event\Publisher\Build as BuildPublisher;
@@ -12,6 +13,7 @@ use Appwrite\SDK\Method;
use Appwrite\SDK\MethodType;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Response;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
@@ -91,6 +93,7 @@ class Create extends Action
->inject('deviceForFunctions')
->inject('deviceForLocal')
->inject('publisherForBuilds')
->inject('jobs')
->inject('plan')
->inject('authorization')
->inject('platform')
@@ -112,6 +115,7 @@ class Create extends Action
Device $deviceForFunctions,
Device $deviceForLocal,
BuildPublisher $publisherForBuilds,
Jobs $jobs,
array $plan,
Authorization $authorization,
array $platform,
@@ -287,7 +291,7 @@ class Create extends Action
}
try {
$locks($lockKey, 600, function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $functionId, $path, &$metadata, $mergeUploadMetadata, $platform, $project, $publisherForBuilds, $queueForEvents, $response, $type): void {
$locks($lockKey, 600, function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $functionId, $path, &$metadata, $mergeUploadMetadata, $platform, $project, $publisherForBuilds, $jobs, $queueForEvents, $response, $type): void {
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
$uploaded = 0;
@@ -328,6 +332,21 @@ class Create extends Action
$fileSize = $deviceForFunctions->getFileSize($path);
// Build backend for manual-upload function deployments:
// 'orchestrator' (open-runtimes jobs-service, submitted below
// in the request flow) or 'executor' (default; enqueued to the
// Builds worker). Selected by _APP_BUILDS_BACKEND.
$useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
// Fields marking the deployment as queued. The build worker
// promotes 'waiting' → 'building' on the first callback. The
// jobs path also pre-declares buildPath so build.sh writes the
// output straight onto the mounted builds volume.
$buildFields = ['status' => 'waiting'];
if ($useJobs) {
$buildFields['buildPath'] = Job::buildPath($project->getId(), $deploymentId);
}
if ($deployment->isEmpty()) {
$deployment = $dbForProject->createDocument('deployments', new Document([
'$id' => $deploymentId,
@@ -349,7 +368,8 @@ class Create extends Action
'sourceChunksUploaded' => $chunksUploaded,
'activate' => $activate,
'sourceMetadata' => $metadata,
'type' => $type
'type' => $type,
...$buildFields,
]));
} else {
@@ -357,17 +377,22 @@ class Create extends Action
'sourceSize' => $fileSize,
'sourceChunksUploaded' => $chunksUploaded,
'sourceMetadata' => $metadata,
...$buildFields,
]));
}
// Start the build
$publisherForBuilds->enqueue(new BuildMessage(
project: $project,
resource: $function,
deployment: $deployment,
type: BUILD_TYPE_DEPLOYMENT,
platform: $platform,
));
if ($useJobs) {
$jobs->create(...Job::build($project, $function, $deployment, $platform));
} else {
// Default: build on the executor via the Builds worker.
$publisherForBuilds->enqueue(new BuildMessage(
project: $project,
resource: $function,
deployment: $deployment,
type: BUILD_TYPE_DEPLOYMENT,
platform: $platform,
));
}
} else {
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
'sourceChunksUploaded' => $chunksUploaded,
@@ -392,4 +417,5 @@ class Create extends Action
throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Deployment upload is busy. Try again.');
}
}
}
@@ -2,6 +2,7 @@
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Download;
use Appwrite\Deployment\Token;
use Appwrite\Extend\Exception;
use Appwrite\Platform\Action;
use Appwrite\SDK\AuthType;
@@ -9,13 +10,16 @@ use Appwrite\SDK\ContentType;
use Appwrite\SDK\Method;
use Appwrite\SDK\MethodType;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Database\Documents\User;
use Appwrite\Utopia\Response;
use Utopia\Database\Database;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\UID;
use Utopia\Http\Adapter\Swoole\Request;
use Utopia\Platform\Enum;
use Utopia\Platform\Scope\HTTP;
use Utopia\Storage\Device;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
class Get extends Action
@@ -35,7 +39,10 @@ class Get extends Action
->httpAlias('/v1/functions/:functionId/deployments/:deploymentId/build/download')
->groups(['api', 'functions'])
->desc('Get deployment download')
->label('scope', 'functions.read')
// Reachable by privileged callers (console/CLI via functions.read)
// and by guests holding a valid presigned token (public); the
// action enforces which one actually applies.
->label('scope', ['public', 'functions.read'])
->label('usage.resource', 'function/{request.functionId}')
->label('resourceType', RESOURCE_TYPE_FUNCTIONS)
->label('sdk', new Method(
@@ -59,11 +66,14 @@ class Get extends Action
->param('functionId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Function ID.', false, ['dbForProject'])
->param('deploymentId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Deployment ID.', false, ['dbForProject'])
->param('type', 'source', new WhiteList(['source', 'output']), 'Deployment file to download. Can be: "source", "output".', true, enum: new Enum(name: 'DeploymentDownloadType'))
->param('token', '', new Text(2048), 'Presigned source-download token for accessing this deployment without a session (jobs-service).', true)
->inject('response')
->inject('request')
->inject('dbForProject')
->inject('deviceForFunctions')
->inject('deviceForBuilds')
->inject('user')
->inject('authorization')
->callback($this->action(...));
}
@@ -71,18 +81,32 @@ class Get extends Action
string $functionId,
string $deploymentId,
string $type,
string $token,
Response $response,
Request $request,
Database $dbForProject,
Device $deviceForFunctions,
Device $deviceForBuilds
Device $deviceForBuilds,
User $user,
Authorization $authorization,
) {
$function = $dbForProject->getDocument('functions', $functionId);
// Access is granted either to a privileged caller (console/CLI via
// session or API key) or to a valid presigned token bound to this
// deployment + artifact type (used by the jobs-service).
$isPrivileged = $user->isPrivileged($authorization->getRoles()) || $user->isKey($authorization->getRoles());
if (! $isPrivileged && ! Token::verify($token, $deploymentId, $type)) {
throw new Exception(Exception::USER_UNAUTHORIZED);
}
// Access has been authorized at the endpoint level (privileged caller or
// presigned token), so read the artifact documents without re-applying
// document-level permissions (a token holder is not an authed user).
$function = $authorization->skip(fn () => $dbForProject->getDocument('functions', $functionId));
if ($function->isEmpty()) {
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
$deployment = $authorization->skip(fn () => $dbForProject->getDocument('deployments', $deploymentId));
if ($deployment->isEmpty()) {
throw new Exception(Exception::DEPLOYMENT_NOT_FOUND);
}
@@ -2,6 +2,7 @@
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Duplicate;
use Appwrite\Compute\Job;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Build as BuildMessage;
use Appwrite\Event\Publisher\Build as BuildPublisher;
@@ -10,6 +11,7 @@ use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Response;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
@@ -17,6 +19,7 @@ use Utopia\Database\Validator\UID;
use Utopia\Platform\Action;
use Utopia\Platform\Scope\HTTP;
use Utopia\Storage\Device;
use Utopia\System\System;
class Create extends Action
{
@@ -64,6 +67,7 @@ class Create extends Action
->inject('dbForProject')
->inject('queueForEvents')
->inject('publisherForBuilds')
->inject('jobs')
->inject('deviceForFunctions')
->inject('project')
->inject('platform')
@@ -78,6 +82,7 @@ class Create extends Action
Database $dbForProject,
Event $queueForEvents,
BuildPublisher $publisherForBuilds,
Jobs $jobs,
Device $deviceForFunctions,
Document $project,
array $platform
@@ -100,6 +105,11 @@ class Create extends Action
$deploymentId = ID::unique();
// Build backend for the rebuild: 'orchestrator' (jobs-service, submitted
// here) or 'executor' (default; Builds worker). The jobs path pre-declares
// buildPath so build.sh writes output straight onto the mounted volume.
$useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
$destination = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$deviceForFunctions->transfer($path, $destination, $deviceForFunctions);
@@ -117,17 +127,21 @@ class Create extends Action
'buildDuration' => null,
'buildSize' => null,
'status' => 'waiting',
'buildPath' => '',
'buildPath' => $useJobs ? Job::buildPath($project->getId(), $deploymentId) : '',
'buildLogs' => '',
]));
$publisherForBuilds->enqueue(new BuildMessage(
project: $project,
resource: $function,
deployment: $deployment,
type: BUILD_TYPE_DEPLOYMENT,
platform: $platform,
));
if ($useJobs) {
$jobs->create(...Job::build($project, $function, $deployment, $platform));
} else {
$publisherForBuilds->enqueue(new BuildMessage(
project: $project,
resource: $function,
deployment: $deployment,
type: BUILD_TYPE_DEPLOYMENT,
platform: $platform,
));
}
$queueForEvents
->setParam('functionId', $function->getId())
@@ -2,6 +2,7 @@
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Status;
use Appwrite\Compute\Job;
use Appwrite\Event\Event;
use Appwrite\Extend\Exception;
use Appwrite\SDK\AuthType;
@@ -9,6 +10,7 @@ use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Response;
use Executor\Executor;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
@@ -61,6 +63,7 @@ class Update extends Action
->inject('project')
->inject('queueForEvents')
->inject('executor')
->inject('jobs')
->callback($this->action(...));
}
@@ -71,7 +74,8 @@ class Update extends Action
Database $dbForProject,
Document $project,
Event $queueForEvents,
Executor $executor
Executor $executor,
Jobs $jobs,
) {
$function = $dbForProject->getDocument('functions', $functionId);
@@ -94,11 +98,7 @@ class Update extends Action
$duration = $endTime->getTimestamp() - $startTime->getTimestamp();
try {
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
'buildEndedAt' => DateTime::now(),
'buildDuration' => $duration,
'status' => 'canceled'
]));
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration)));
} catch (TransactionException) {
$deployment = $dbForProject->getDocument('deployments', $deployment->getId());
@@ -111,18 +111,20 @@ class Update extends Action
}
if ($deployment->getAttribute('status') !== 'canceled') {
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
'buildEndedAt' => DateTime::now(),
'buildDuration' => $duration,
'status' => 'canceled'
]));
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration)));
}
}
// Best-effort cleanup on both backends — the deployment is already
// marked 'canceled', and only one backend actually holds the build.
try {
$executor->deleteRuntime($project->getId(), $deploymentId . "-build");
} catch (\Throwable) {
// Best-effort cleanup — deployment status is already 'canceled'
}
try {
$jobs->delete(Job::id($project->getId(), $deploymentId));
} catch (\Throwable) {
}
$queueForEvents
@@ -131,4 +133,28 @@ class Update extends Action
$response->dynamic($deployment, Response::MODEL_DEPLOYMENT);
}
/**
* The sparse update marking a build canceled. Jobs-backed builds (identified
* by a buildPath set at submission) have no cancel worker to write the
* closing log line the executor's Builds worker adds, so it is appended here;
* executor deployments get it from their worker.
*
* @return array<string, mixed>
*/
private function cancel(Document $deployment, int $duration): array
{
$update = [
'buildEndedAt' => DateTime::now(),
'buildDuration' => $duration,
'status' => 'canceled',
];
if ($deployment->getAttribute('buildPath', '') !== '') {
$logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n";
$update['buildLogs'] = \substr($logs, -APP_LOG_LENGTH_LIMIT);
}
return $update;
}
}
@@ -2,6 +2,7 @@
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Template;
use Appwrite\Compute\Job;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Build as BuildMessage;
use Appwrite\Event\Publisher\Build as BuildPublisher;
@@ -12,6 +13,7 @@ use Appwrite\SDK\AuthType;
use Appwrite\SDK\Method;
use Appwrite\SDK\Response as SDKResponse;
use Appwrite\Utopia\Response;
use OpenRuntimes\Orchestrator\Jobs;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
@@ -22,6 +24,7 @@ use Utopia\Database\Validator\UID;
use Utopia\Http\Adapter\Swoole\Request;
use Utopia\Platform\Enum;
use Utopia\Platform\Scope\HTTP;
use Utopia\System\System;
use Utopia\Validator\Boolean;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
@@ -80,6 +83,7 @@ class Create extends Base
->inject('queueForEvents')
->inject('project')
->inject('publisherForBuilds')
->inject('jobs')
->inject('gitHub')
->inject('authorization')
->inject('platform')
@@ -101,6 +105,7 @@ class Create extends Base
Event $queueForEvents,
Document $project,
BuildPublisher $publisherForBuilds,
Jobs $jobs,
GitHub $github,
Authorization $authorization,
array $platform
@@ -124,6 +129,23 @@ class Create extends Base
]);
if (!empty($function->getAttribute('providerRepositoryId'))) {
// VCS-connected function: the template is merged into the user's repo
// and pushed as a commit, then that commit is built. This stays on the
// executor for now because it is a git *write*, which the jobs-service
// artifact system (download/unarchive) does not cover.
//
// Plan to move it to the jobs backend (3a — Git Data API push, no git
// binary), when utopia-php/vcs grows the bulk-push methods:
// 1. download the template tarball (codeload) + read the tree of the
// target repo/branch (github->getRepositoryTree).
// 2. for each template file: create a blob (POST .../git/blobs).
// 3. create a tree from the blobs on top of the branch's base tree
// (POST .../git/trees), then a commit (POST .../git/commits), then
// fast-forward the branch ref (PATCH .../git/refs/heads/{branch}).
// 4. take the resulting commit hash and submit the build via
// Job::build (same authenticated-tarball path as createVcsDeployment).
// This replaces clone + rsync + `git commit && git push` with ~4 API
// calls and no working tree. Until then, executor handles this branch.
$installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId'));
$deployment = $this->redeployVcsFunction(
@@ -152,6 +174,11 @@ class Create extends Base
return;
}
// Backend for the template build: 'orchestrator' (jobs-service, which
// pulls the public GitHub tarball via artifacts) or 'executor' (default;
// the Builds worker clones the repo). The jobs path pre-declares buildPath.
$useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
$deploymentId = ID::unique();
$deployment = $dbForProject->createDocument('deployments', new Document([
'$id' => $deploymentId,
@@ -173,18 +200,24 @@ class Create extends Base
'providerBranch' => $type == GitHub::CLONE_TYPE_BRANCH ? $reference : '',
'type' => 'vcs',
'activate' => $activate,
'status' => 'waiting',
'buildPath' => $useJobs ? Job::buildPath($project->getId(), $deploymentId) : '',
]));
$this->updateEmptyManualRule($project, $function, $deployment, $dbForPlatform, $authorization);
$publisherForBuilds->enqueue(new BuildMessage(
project: $project,
resource: $function,
deployment: $deployment,
type: BUILD_TYPE_DEPLOYMENT,
template: $template,
platform: $platform,
));
if ($useJobs) {
$jobs->create(...Job::build($project, $function, $deployment, $platform, $template));
} else {
$publisherForBuilds->enqueue(new BuildMessage(
project: $project,
resource: $function,
deployment: $deployment,
type: BUILD_TYPE_DEPLOYMENT,
template: $template,
platform: $platform,
));
}
$queueForEvents
->setParam('functionId', $function->getId())
@@ -0,0 +1,74 @@
<?php
namespace Appwrite\Platform\Modules\Functions\Http\Jobs\Event;
use Appwrite\Event\Message\Jobs as JobsMessage;
use Appwrite\Event\Publisher\Jobs as JobsPublisher;
use Appwrite\Extend\Exception;
use Appwrite\Platform\Action;
use Appwrite\Utopia\Response;
use OpenRuntimes\Orchestrator\Callback\Signature;
use Utopia\Database\Document;
use Utopia\Http\Adapter\Swoole\Request;
use Utopia\Platform\Scope\HTTP;
use Utopia\System\System;
/**
* Ingests an open-runtimes jobs-service CloudEvents callback. Verifies the
* HMAC signature, then hands the event off to the jobs worker which applies
* it to the deployment. Kept thin on purpose — the request returns
* immediately so the jobs-service delivery loop isn't blocked.
*/
class Create extends Action
{
use HTTP;
public static function getName()
{
return 'createJobEvent';
}
public function __construct()
{
$this
->setHttpMethod(Action::HTTP_REQUEST_METHOD_POST)
->setHttpPath('/v1/jobs/event')
->groups(['api'])
->desc('Create job event')
->label('scope', 'public')
->inject('request')
->inject('response')
->inject('publisherForJobs')
->callback($this->action(...));
}
public function action(
Request $request,
Response $response,
JobsPublisher $publisherForJobs,
): void {
$body = $request->getRawPayload();
$secret = System::getEnv('_APP_JOBS_SECRET', '');
$signature = $request->getHeaderLine('x-signature-256');
if ($secret === '' || ! Signature::verify($body, $signature, $secret)) {
throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid jobs signature.');
}
$event = \json_decode($body, true) ?? [];
// The project isn't a request-scoped resource here (the jobs-service is
// the caller), so resolve it from the job meta Appwrite authored. The
// worker reloads the full project document from this id.
$projectId = $event['data']['meta']['projectId'] ?? '';
$publisherForJobs->enqueue(new JobsMessage(
project: new Document(['$id' => $projectId]),
id: $event['id'] ?? '',
event: $event['type'] ?? '',
data: $event['data'] ?? [],
));
$response->noContent();
}
}
@@ -21,6 +21,7 @@ use Appwrite\Platform\Modules\Functions\Http\Functions\Deployment\Update as Upda
use Appwrite\Platform\Modules\Functions\Http\Functions\Get as GetFunction;
use Appwrite\Platform\Modules\Functions\Http\Functions\Update as UpdateFunction;
use Appwrite\Platform\Modules\Functions\Http\Functions\XList as ListFunctions;
use Appwrite\Platform\Modules\Functions\Http\Jobs\Event\Create as CreateJobEvent;
use Appwrite\Platform\Modules\Functions\Http\Runtimes\XList as ListRuntimes;
use Appwrite\Platform\Modules\Functions\Http\Specifications\XList as ListSpecifications;
use Appwrite\Platform\Modules\Functions\Http\Templates\Get as GetTemplate;
@@ -63,6 +64,9 @@ class Http extends Service
$this->addAction(CreateDuplicateDeployment::getName(), new CreateDuplicateDeployment());
$this->addAction(UpdateDeploymentStatus::getName(), new UpdateDeploymentStatus());
// Jobs (open-runtimes jobs-service build backend)
$this->addAction(CreateJobEvent::getName(), new CreateJobEvent());
// Executions
$this->addAction(CreateExecution::getName(), new CreateExecution());
$this->addAction(GetExecution::getName(), new GetExecution());
@@ -3,6 +3,7 @@
namespace Appwrite\Platform\Modules\Functions\Services;
use Appwrite\Platform\Modules\Functions\Workers\Builds;
use Appwrite\Platform\Modules\Functions\Workers\Jobs;
use Appwrite\Platform\Modules\Functions\Workers\Screenshots;
use Utopia\Platform\Service;
@@ -12,6 +13,7 @@ class Workers extends Service
{
$this->type = Service::TYPE_WORKER;
$this->addAction(Builds::getName(), new Builds());
$this->addAction(Jobs::getName(), new Jobs());
$this->addAction(Screenshots::getName(), new Screenshots());
}
}
@@ -5,7 +5,6 @@ namespace Appwrite\Platform\Modules\Functions\Workers;
use Ahc\Jwt\JWT;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Func as FunctionMessage;
use Appwrite\Event\Message\Usage as UsageMessage;
use Appwrite\Event\Publisher\Func as FunctionPublisher;
use Appwrite\Event\Publisher\Screenshot;
use Appwrite\Event\Publisher\Usage as UsagePublisher;
@@ -13,6 +12,7 @@ use Appwrite\Event\Realtime;
use Appwrite\Event\Webhook;
use Appwrite\Extend\Exception as AppwriteException;
use Appwrite\Filter\BranchDomain as BranchDomainFilter;
use Appwrite\Usage\Build as BuildUsage;
use Appwrite\Usage\Context;
use Appwrite\Utopia\Response\Model\Deployment;
use Appwrite\Vcs\Comment;
@@ -1259,55 +1259,7 @@ class Builds extends Action
protected function sendUsage(Document $resource, Document $deployment, Document $project, Context $usage, UsagePublisher $publisherForUsage): void
{
$spec = Config::getParam('specifications')[$resource->getAttribute('buildSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT)];
$cpus = (int) ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT);
$memory = (int) ($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT);
switch ($deployment->getAttribute('status')) {
case 'ready':
$usage
->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project
->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_SUCCESS), 1) // per function
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_SUCCESS), 1) // per function
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000);
break;
case 'failed':
$usage
->addMetric(METRIC_BUILDS_FAILED, 1) // per project
->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_FAILED), 1) // per function
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_FAILED), 1) // per function
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000);
break;
}
$usage
->addMetric(METRIC_BUILDS, 1) // per project
->addMetric(METRIC_BUILDS_STORAGE, $deployment->getAttribute('buildSize', 0))
->addMetric(METRIC_BUILDS_COMPUTE, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
->addMetric(METRIC_BUILDS_MB_SECONDS, (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus))
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS), 1) // per function
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0))
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus))
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS), 1) // per function
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0))
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus));
// Publish usage metrics
if (! $usage->isEmpty()) {
$message = new UsageMessage(
project: $project,
metrics: $usage->getMetrics(),
reduce: $usage->getReduce()
);
$publisherForUsage->enqueue($message);
$usage->reset();
}
BuildUsage::publish($usage, $resource, $deployment, $project, $publisherForUsage);
}
/**
@@ -0,0 +1,345 @@
<?php
namespace Appwrite\Platform\Modules\Functions\Workers;
use Appwrite\Compute\Job;
use Appwrite\Event\Event;
use Appwrite\Event\Message\Func as FunctionMessage;
use Appwrite\Event\Message\Jobs as JobsMessage;
use Appwrite\Event\Publisher\Func as FunctionPublisher;
use Appwrite\Event\Publisher\Usage as UsagePublisher;
use Appwrite\Event\Realtime;
use Appwrite\Event\Webhook;
use Appwrite\Usage\Build as BuildUsage;
use Appwrite\Usage\Context as UsageContext;
use Appwrite\Utopia\Response\Model\Deployment;
use Utopia\Cache\Cache;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\Storage\Device;
use Utopia\System\System;
/**
* Applies open-runtimes jobs-service callbacks to a deployment. The API
* callback endpoint verifies + enqueues the CloudEvent; this worker owns the
* deployment state transitions (log streaming and the ready/failed outcome)
* that the executor path performs inline in the Builds worker.
*
* Callbacks arrive concurrently and out of order (one log line each), so
* processing for a deployment is serialized under a per-deployment lock — that
* keeps its buildLogs a clean, monotonic append while letting different
* deployments build in parallel. Callbacks are at-least-once, so events are
* de-duplicated on the CloudEvent id (inside the lock, so dedup + apply is
* atomic and a lock timeout retries cleanly).
*/
class Jobs extends Action
{
private const DEDUPE_TTL = 3600;
private const LOCK_TTL = 30;
private const LOCK_TIMEOUT = 10.0;
public static function getName(): string
{
return 'jobs';
}
public function __construct()
{
$this
->desc('Jobs worker')
->groups(['jobs'])
->inject('message')
->inject('project')
->inject('dbForProject')
->inject('dbForPlatform')
->inject('queueForRealtime')
->inject('queueForEvents')
->inject('queueForWebhooks')
->inject('publisherForFunctions')
->inject('publisherForUsage')
->inject('usage')
->inject('deviceForBuilds')
->inject('cache')
->inject('locks')
->callback($this->action(...));
}
public function action(
Message $message,
Document $project,
Database $dbForProject,
Database $dbForPlatform,
Realtime $queueForRealtime,
Event $queueForEvents,
Webhook $queueForWebhooks,
FunctionPublisher $publisherForFunctions,
UsagePublisher $publisherForUsage,
UsageContext $usage,
Device $deviceForBuilds,
Cache $cache,
callable $locks,
): void {
$event = JobsMessage::fromArray($message->getPayload());
$deploymentId = $event->data['meta']['deploymentId'] ?? '';
if ($deploymentId === '') {
return;
}
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $cache, $deploymentId): void {
if ($event->id !== '') {
$key = 'jobs-event-' . $event->id;
if ($cache->load($key, self::DEDUPE_TTL) !== false) {
return; // already processed
}
$cache->save($key, true);
}
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
if ($deployment->isEmpty() || $deployment->getAttribute('status') === 'canceled') {
return;
}
// The build writes its output onto the mounted volume before build.sh
// exits, so the exit callback is a truthful completion signal.
$finalizing = $event->event === 'orchestrator.job.exit';
$deployment = match ($event->event) {
'orchestrator.job.log' => $this->onLog($dbForProject, $deployment, $event->data),
'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds),
default => $deployment,
};
// Console realtime on every callback (log stream + status).
$queueForRealtime
->setSubscribers(['console'])
->setProject($project)
->setEvent('functions.[functionId].deployments.[deploymentId].update')
->setParam('functionId', $deployment->getAttribute('resourceId'))
->setParam('deploymentId', $deploymentId)
->setPayload($deployment->getArrayCopy())
->trigger();
// On a real terminal outcome (not a concurrently-canceled build),
// notify webhooks + event-triggered functions of the deployment
// update (mirrors the executor Builds worker).
if ($finalizing && \in_array($deployment->getAttribute('status'), ['ready', 'failed'], true)) {
$this->dispatchUpdate($queueForEvents, $queueForWebhooks, $publisherForFunctions, $project, $deployment);
}
}, self::LOCK_TIMEOUT);
}
private function onLog(Database $dbForProject, Document $deployment, array $data): Document
{
$lines = $data['lines'] ?? [];
$chunk = \is_array($lines) ? \implode("\n", $lines) : (string) $lines;
if ($chunk === '') {
return $deployment;
}
$logs = $this->truncate($deployment->getAttribute('buildLogs', '') . $chunk . "\n");
$update = ['buildLogs' => $logs];
// First build output means the build is running: promote the queued
// deployment to 'building' and stamp its start (mirrors the executor).
if ($deployment->getAttribute('status') === 'waiting') {
$update['status'] = 'building';
$update['buildStartedAt'] = DateTime::now();
}
return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($update));
}
/**
* Resolve the build-command exit into a terminal outcome. The output is
* already on the mounted builds volume, so a zero exit enforces the size
* limit against the on-disk artifact before readying; a non-zero exit fails.
*/
private function onExit(
Database $dbForProject,
Database $dbForPlatform,
Document $project,
Document $deployment,
int $exitCode,
UsageContext $usage,
UsagePublisher $publisherForUsage,
Device $deviceForBuilds,
): Document {
if ($exitCode !== 0) {
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage);
}
$path = Job::buildPath($project->getId(), $deployment->getId());
$size = $deviceForBuilds->exists($path) ? $deviceForBuilds->getFileSize($path) : 0;
$limit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000');
if ($limit !== 0 && $size > $limit) {
$deviceForBuilds->delete($path);
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build size should be less than ' . \number_format($limit / (1000 * 1000), 2) . ' MBs.', $usage, $publisherForUsage);
}
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, true, '', $usage, $publisherForUsage, $size);
}
/**
* Apply a terminal outcome (ready or failed) to the deployment. Owns
* activation, usage and the latestDeployment pointer, mirroring the executor
* Builds worker.
*/
private function finalize(
Database $dbForProject,
Database $dbForPlatform,
Document $project,
Document $deployment,
bool $success,
string $message,
UsageContext $usage,
UsagePublisher $publisherForUsage,
int $buildSize = 0,
): Document {
$function = $dbForProject->getDocument('functions', $deployment->getAttribute('resourceId'));
$logs = $deployment->getAttribute('buildLogs', '');
$startedAt = \strtotime($deployment->getAttribute('buildStartedAt', '') ?: 'now') ?: \time();
$trailer = $success
? "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[32m Deployment finished. \033[0m\n"
: "\n" . ($message !== '' ? $message : 'Build failed.') . "\n";
$update = [
'status' => $success ? 'ready' : 'failed',
'buildEndedAt' => DateTime::now(),
'buildDuration' => \max(0, \time() - $startedAt),
'buildLogs' => $this->truncate($logs . $trailer),
];
if ($success) {
$update['buildSize'] = $buildSize;
$update['totalSize'] = $buildSize + (int) $deployment->getAttribute('sourceSize', 0);
}
// Guard against a concurrent cancel (updateDeploymentStatus): only
// transition if the build wasn't canceled between the read and this
// write, so a late outcome can't resurrect a canceled build.
$applied = $dbForProject->updateDocuments('deployments', new Document($update), [
Query::equal('$id', [$deployment->getId()]),
Query::notEqual('status', 'canceled'),
]);
$deployment = $dbForProject->getDocument('deployments', $deployment->getId());
if ($applied > 0 && $success && $deployment->getAttribute('activate') === true && ! $function->isEmpty()) {
$this->activate($dbForProject, $dbForPlatform, $project, $function, $deployment);
}
// Count the build for usage/billing once it reached a terminal outcome
// (mirrors the executor Builds worker); never for a concurrently-canceled
// build (the guard above left it 'canceled').
if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'], true) && ! $function->isEmpty()) {
BuildUsage::publish($usage, $function, $deployment, $project, $publisherForUsage);
}
// Keep the function's "latest deployment" pointer + status current.
if (! $function->isEmpty()) {
$this->updateLatestDeployment($dbForProject, $function);
}
return $deployment;
}
/**
* Point the function at this deployment (auto-activate). Mirrors the
* essential function activation from the Builds worker.
*/
private function activate(Database $dbForProject, Database $dbForPlatform, Document $project, Document $function, Document $deployment): void
{
$function = $dbForProject->updateDocument('functions', $function->getId(), new Document([
'live' => true,
'deploymentId' => $deployment->getId(),
'deploymentInternalId' => $deployment->getSequence(),
'deploymentCreatedAt' => $deployment->getCreatedAt(),
]));
$dbForPlatform->forEach('rules', function (Document $rule) use ($dbForPlatform, $deployment) {
$dbForPlatform->updateDocument('rules', $rule->getId(), new Document([
'deploymentId' => $deployment->getId(),
'deploymentInternalId' => $deployment->getSequence(),
]));
}, [
Query::equal('projectInternalId', [$project->getSequence()]),
Query::equal('type', ['deployment']),
Query::equal('deploymentResourceInternalId', [$function->getSequence()]),
Query::equal('deploymentResourceType', ['function']),
Query::equal('trigger', ['manual']),
Query::equal('deploymentVcsProviderBranch', ['']),
]);
}
/**
* Refresh the function's latestDeployment* fields from its newest
* deployment. Mirrors the Builds worker so the console reflects the current
* build status.
*/
private function updateLatestDeployment(Database $dbForProject, Document $function): void
{
$latest = $dbForProject->findOne('deployments', [
Query::equal('resourceType', ['functions']),
Query::equal('resourceInternalId', [$function->getSequence()]),
Query::orderDesc('$createdAt'),
]);
if ($latest->isEmpty()) {
return;
}
$dbForProject->updateDocument('functions', $function->getId(), new Document([
'latestDeploymentId' => $latest->getId(),
'latestDeploymentInternalId' => $latest->getSequence(),
'latestDeploymentCreatedAt' => $latest->getCreatedAt(),
'latestDeploymentStatus' => $latest->getAttribute('status', ''),
]));
}
/**
* Notify project webhooks and event-triggered functions of a deployment
* update, mirroring the executor Builds worker's fan-out. queueForEvents
* only builds the event; realtime is triggered separately in action().
*/
private function dispatchUpdate(
Event $queueForEvents,
Webhook $queueForWebhooks,
FunctionPublisher $publisherForFunctions,
Document $project,
Document $deployment,
): void {
$model = new Deployment();
$update = $queueForEvents
->setProject($project)
->setEvent('functions.[functionId].deployments.[deploymentId].update')
->setParam('functionId', $deployment->getAttribute('resourceId'))
->setParam('deploymentId', $deployment->getId())
->setPayload($deployment->getArrayCopy(\array_keys($model->getRules())));
$queueForWebhooks->from($update)->trigger();
$publisherForFunctions->enqueue(FunctionMessage::fromEvent(
event: $update->getEvent(),
params: $update->getParams(),
project: $update->getProject(),
user: $update->getUser(),
userId: $update->getUserId(),
payload: $update->getPayload(),
platform: $update->getPlatform(),
));
}
private function truncate(string $logs): string
{
$limit = APP_LOG_LENGTH_LIMIT;
if (\strlen($logs) <= $limit) {
return $logs;
}
return \substr($logs, -$limit);
}
}
+76
View File
@@ -0,0 +1,76 @@
<?php
namespace Appwrite\Usage;
use Appwrite\Event\Message\Usage as UsageMessage;
use Appwrite\Event\Publisher\Usage as UsagePublisher;
use Utopia\Config\Config;
use Utopia\Database\Document;
/**
* Emits build usage/billing metrics for a completed deployment. Shared by both
* build backends — the executor (Builds worker) and the jobs-service (Jobs
* worker) — so a build is counted identically however it was produced.
*/
final class Build
{
/**
* Record the metrics for a terminal deployment (status 'ready' or 'failed')
* onto the usage context and publish them.
*/
public static function publish(
Context $usage,
Document $resource,
Document $deployment,
Document $project,
UsagePublisher $publisherForUsage,
): void {
$spec = Config::getParam('specifications')[$resource->getAttribute('buildSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT)];
$cpus = (int) ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT);
$memory = (int) ($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT);
switch ($deployment->getAttribute('status')) {
case 'ready':
$usage
->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project
->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_SUCCESS), 1) // per function
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_SUCCESS), 1) // per function
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000);
break;
case 'failed':
$usage
->addMetric(METRIC_BUILDS_FAILED, 1) // per project
->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_FAILED), 1) // per function
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_FAILED), 1) // per function
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000);
break;
}
$usage
->addMetric(METRIC_BUILDS, 1) // per project
->addMetric(METRIC_BUILDS_STORAGE, $deployment->getAttribute('buildSize', 0))
->addMetric(METRIC_BUILDS_COMPUTE, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
->addMetric(METRIC_BUILDS_MB_SECONDS, (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus))
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS), 1) // per function
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0))
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus))
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS), 1) // per function
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0))
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus));
if (! $usage->isEmpty()) {
$publisherForUsage->enqueue(new UsageMessage(
project: $project,
metrics: $usage->getMetrics(),
reduce: $usage->getReduce()
));
$usage->reset();
}
}
}
Executable
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env bash
#
# End-to-end test of the jobs-service build slice, driven entirely through the
# Appwrite CLI's `push function` — which streams the deployment build logs live,
# exactly as an end user sees them.
#
# Self-contained: bootstraps a throwaway console account + project, logs the CLI
# in, then pushes a tiny Node function twice — so the second build reuses the
# package-manager build cache the first one saved ("Build cache hit").
#
# Optional env:
# ENDPOINT Appwrite endpoint (default: http://localhost/v1)
# FUNC_ID function id (default: testbuild)
# COMMANDS build command (default: npm install)
#
set -euo pipefail
ENDPOINT="${ENDPOINT:-http://localhost/v1}"
FUNC_ID="${FUNC_ID:-testbuild}"
COMMANDS="${COMMANDS:-npm install}"
EMAIL="tb-$(date +%s)@example.com"
PASSWORD="password123"
id() { python3 -c 'import sys,json;print(json.load(sys.stdin).get("$id",""))'; }
console() { curl -s -c "$JAR" -b "$JAR" -H "X-Appwrite-Project: console" -H "Content-Type: application/json" "$@"; }
# --- Bootstrap a project owned by a fresh console account (REST) -------------
JAR="$(mktemp)"
console -X POST "$ENDPOINT/account" -d "{\"userId\":\"unique()\",\"email\":\"$EMAIL\",\"password\":\"$PASSWORD\",\"name\":\"Test\"}" >/dev/null
console -X POST "$ENDPOINT/account/sessions/email" -d "{\"email\":\"$EMAIL\",\"password\":\"$PASSWORD\"}" >/dev/null
TEAM="$(console -X POST "$ENDPOINT/teams" -d '{"teamId":"unique()","name":"t"}' | id)"
PROJECT_ID="$(console -X POST "$ENDPOINT/projects" -d "{\"projectId\":\"unique()\",\"name\":\"p\",\"teamId\":\"$TEAM\",\"region\":\"default\"}" | id)"
# --- Log the CLI in (push uses a session, not an API key) --------------------
appwrite login --email "$EMAIL" --password "$PASSWORD" --endpoint "$ENDPOINT" >/dev/null
# --- Generate a minimal Appwrite project to push -----------------------------
WORK="$(mktemp -d)"
mkdir -p "$WORK/functions/$FUNC_ID"
printf 'module.exports = async ({ res }) => res.json({ ok: true });\n' > "$WORK/functions/$FUNC_ID/main.js"
# A real dependency so `npm install` populates a cache worth reusing.
printf '{ "name": "fn", "version": "1.0.0", "main": "main.js", "dependencies": { "ms": "2.1.3" } }\n' > "$WORK/functions/$FUNC_ID/package.json"
cat > "$WORK/appwrite.json" <<EOF
{
"projectId": "$PROJECT_ID",
"functions": [
{
"\$id": "$FUNC_ID",
"name": "$FUNC_ID",
"runtime": "node-22",
"execute": ["any"],
"entrypoint": "main.js",
"commands": "$COMMANDS",
"path": "functions/$FUNC_ID"
}
]
}
EOF
# --- Push + stream the build (real user experience) --------------------------
# Push twice against the same function: the first build seeds the build cache,
# the second reuses it (watch for "Build cache hit" in the second run's logs).
cd "$WORK"
echo "Project $PROJECT_ID — first build (cold cache), logs stream below:"
echo
appwrite push function --function-id "$FUNC_ID" --activate --force
echo
echo "Second build (should hit the build cache seeded above):"
echo
appwrite push function --function-id "$FUNC_ID" --activate --force