mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
feat(compute): build manual-upload functions via open-runtimes jobs-service
Adds an alternative, stateless build backend for manual-upload function deployments using the open-runtimes jobs-service (orchestrator) instead of the executor. The build runs as a stateless job: source, output, and the package-manager build cache all cross the boundary via the jobs-service artifacts system (presigned download/upload), and progress arrives as CloudEvents callbacks handled by a new worker-jobs. createDeployment submits the job in the request flow; the executor path is untouched for every other build trigger. - Orchestrator\Jobs: thin jobs-service HTTP client (POST /v1/jobs) - Appwrite\Compute\Job: builds the job payload (image, command, env, artifacts) - Appwrite\Deployment\Token: signed presign tokens (source/output/cache) - Functions/Http/Deployments/Archive: token-authed output + cache upload (PUT) - Functions/Http/Jobs/Event: HMAC-verified CloudEvents callback ingress - Functions/Workers/Jobs: applies log/exit callbacks under a per-deployment lock; on completion fires the deployment.update fan-out (webhooks + event-functions + realtime) and refreshes latestDeployment*, mirroring the executor Builds worker - Deployments/Download/Get: presigned source + build-cache download - Build cache via OPEN_RUNTIMES_BUILD_* dir overrides: pure artifacts, no shell bridge (per-function npm/yarn/pnpm cache reused across builds) - DI/queue wiring, docker services (orchestrator-jobs, worker-jobs), test-build.sh Verified end-to-end: deploy -> ready -> activated -> executes (200); cache miss -> save -> hit; latestDeploymentStatus + webhook/function fan-out fire. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d8b569b564
commit
d1b1ab9fb0
27 files changed
+1401
-104
No files matched your search
@@ -117,6 +117,11 @@ _APP_SITES_TIMEOUT=30
|
||||
_APP_EXECUTOR_HOST=http://exc1/v1
|
||||
_APP_EXECUTOR_SECRET=your-secret-key
|
||||
_APP_EXECUTOR_IMAGES=openruntimes/node:v5-22,openruntimes/static:v5-1
|
||||
_APP_BUILDS_BACKEND=orchestrator
|
||||
_APP_BUILDS_VOLUME=appwrite-builds
|
||||
_APP_JOBS_HOST=http://orchestrator-jobs:8080
|
||||
_APP_JOBS_SECRET=your-secret-key
|
||||
_APP_JOBS_ENDPOINT=http://appwrite
|
||||
_APP_BROWSER_HOST=http://appwrite-browser:3000/v1
|
||||
_APP_DOCKER_HUB_USERNAME=
|
||||
_APP_DOCKER_HUB_PASSWORD=
|
||||
|
||||
@@ -90,6 +90,7 @@ RUN chmod +x /usr/local/bin/doctor && \
|
||||
chmod +x /usr/local/bin/queue-count-processing && \
|
||||
chmod +x /usr/local/bin/queue-count-success && \
|
||||
chmod +x /usr/local/bin/worker-builds && \
|
||||
chmod +x /usr/local/bin/worker-jobs && \
|
||||
chmod +x /usr/local/bin/worker-screenshots && \
|
||||
chmod +x /usr/local/bin/worker-certificates && \
|
||||
chmod +x /usr/local/bin/worker-databases && \
|
||||
|
||||
@@ -979,6 +979,53 @@ return [
|
||||
'question' => '',
|
||||
'filter' => ''
|
||||
],
|
||||
[
|
||||
'name' => '_APP_BUILDS_BACKEND',
|
||||
'description' => 'Backend that builds manual-upload function deployments: "executor" (default; the open-runtimes executor, via the Builds worker) or "orchestrator" (the open-runtimes jobs-service, submitted in the request flow). Other build flows always use the executor.',
|
||||
'introduction' => '1.9.0',
|
||||
'default' => 'executor',
|
||||
'required' => false,
|
||||
'question' => '',
|
||||
'filter' => ''
|
||||
],
|
||||
[
|
||||
'name' => '_APP_BUILDS_VOLUME',
|
||||
'description' => 'The Docker volume (or Kubernetes PersistentVolumeClaim) holding build storage, attached to jobs-service build workers so they write output directly onto it. Must match the storage the "builds" device is backed by. Only used when _APP_BUILDS_BACKEND is "orchestrator".',
|
||||
'introduction' => '1.9.0',
|
||||
'default' => 'appwrite-builds',
|
||||
'required' => false,
|
||||
'question' => '',
|
||||
'filter' => ''
|
||||
],
|
||||
[
|
||||
'name' => '_APP_JOBS_HOST',
|
||||
'description' => 'The host used by Appwrite to communicate with the open-runtimes jobs-service that builds manual-upload function deployments.',
|
||||
'introduction' => '1.9.0',
|
||||
'default' => 'http://orchestrator-jobs:8080',
|
||||
'required' => false,
|
||||
'overwrite' => true,
|
||||
'question' => '',
|
||||
'filter' => ''
|
||||
],
|
||||
[
|
||||
'name' => '_APP_JOBS_SECRET',
|
||||
'description' => 'The secret used to authenticate with the jobs-service and to sign/verify job callback (HMAC) requests. Make sure to change this.',
|
||||
'introduction' => '1.9.0',
|
||||
'default' => 'your-secret-key',
|
||||
'required' => false,
|
||||
'question' => '',
|
||||
'filter' => ''
|
||||
],
|
||||
[
|
||||
'name' => '_APP_JOBS_ENDPOINT',
|
||||
'description' => 'Internal Appwrite endpoint the jobs-service (and the containers it spawns) use to reach the API over the Docker network for presigned artifact + callback URLs.',
|
||||
'introduction' => '1.9.0',
|
||||
'default' => 'http://appwrite',
|
||||
'required' => false,
|
||||
'overwrite' => true,
|
||||
'question' => '',
|
||||
'filter' => ''
|
||||
],
|
||||
[
|
||||
'name' => '_APP_BROWSER_HOST',
|
||||
'description' => 'The host used by Appwrite to communicate with the browser service for screenshots.',
|
||||
|
||||
@@ -9,6 +9,7 @@ use Appwrite\Event\Publisher\Database as DatabasePublisher;
|
||||
use Appwrite\Event\Publisher\Delete as DeletePublisher;
|
||||
use Appwrite\Event\Publisher\Execution as ExecutionPublisher;
|
||||
use Appwrite\Event\Publisher\Func as FunctionPublisher;
|
||||
use Appwrite\Event\Publisher\Jobs as JobsPublisher;
|
||||
use Appwrite\Event\Publisher\Mail as MailPublisher;
|
||||
use Appwrite\Event\Publisher\Messaging as MessagingPublisher;
|
||||
use Appwrite\Event\Publisher\Migration as MigrationPublisher;
|
||||
@@ -18,10 +19,13 @@ use Appwrite\Event\Publisher\StatsResources as StatsResourcesPublisher;
|
||||
use Appwrite\Event\Publisher\Usage as UsagePublisher;
|
||||
use Appwrite\Platform\Modules\Storage\Config\StorageCacheControl;
|
||||
use Executor\Executor;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis;
|
||||
use Utopia\Cache\Adapter\Pool as CachePool;
|
||||
use Utopia\Cache\Adapter\Sharding;
|
||||
use Utopia\Cache\Cache;
|
||||
use Utopia\Client;
|
||||
use Utopia\Client\Adapter\Curl\Client as CurlAdapter;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Console;
|
||||
use Utopia\Database\Document;
|
||||
@@ -66,6 +70,13 @@ $container->set('localeCodes', fn () => array_map(fn ($locale) => $locale['code'
|
||||
|
||||
$container->set('executor', fn () => new Executor(), []);
|
||||
|
||||
$container->set('jobs', fn () => new Jobs(
|
||||
(new Client(new CurlAdapter()))
|
||||
->withBaseUri(System::getEnv('_APP_JOBS_HOST', ''))
|
||||
->withBearerAuth(System::getEnv('_APP_JOBS_SECRET', ''))
|
||||
->withTimeout(30)
|
||||
), []);
|
||||
|
||||
$container->set('telemetry', fn () => new NoTelemetry(), []);
|
||||
|
||||
$container->set('authorization', fn () => new Authorization(), []);
|
||||
@@ -131,6 +142,11 @@ $container->set('publisherForBuilds', fn (Publisher $publisher) => new BuildPubl
|
||||
new Queue(System::getEnv('_APP_BUILDS_QUEUE_NAME', Event::BUILDS_QUEUE_NAME))
|
||||
), ['publisher']);
|
||||
|
||||
$container->set('publisherForJobs', fn (Publisher $publisher) => new JobsPublisher(
|
||||
$publisher,
|
||||
new Queue(System::getEnv('_APP_JOBS_QUEUE_NAME', Event::JOBS_QUEUE_NAME))
|
||||
), ['publisher']);
|
||||
|
||||
$container->set('publisherForDatabase', fn (Publisher $publisherDatabases) => new DatabasePublisher(
|
||||
$publisherDatabases,
|
||||
new Queue(System::getEnv('_APP_DATABASE_QUEUE_NAME', Event::DATABASE_QUEUE_NAME))
|
||||
@@ -333,6 +349,8 @@ function getDevice(string $root, string $connection = ''): Device
|
||||
}
|
||||
}
|
||||
|
||||
$container->set('geodb', fn ($register) => $register->get('geodb'), ['register']);
|
||||
|
||||
$container->set('passwordsDictionary', fn ($register) => $register->get('passwordsDictionary'), ['register']);
|
||||
|
||||
$container->set('servers', function () {
|
||||
|
||||
@@ -580,6 +580,12 @@ return function (Container $context): void {
|
||||
if (! \is_string($projectId)) {
|
||||
$projectId = $request->getHeaderLine('x-appwrite-project', '');
|
||||
}
|
||||
// For non-GET requests getParam() reads the body, so a project passed
|
||||
// as a query parameter (e.g. presigned artifact URLs) is only visible
|
||||
// via getQuery().
|
||||
if (empty($projectId)) {
|
||||
$projectId = (string) $request->getQuery('project', '');
|
||||
}
|
||||
|
||||
// Backwards compatibility for new services, originally project resources
|
||||
// These endpoints moved from /v1/projects/:projectId/<resource> to /v1/<resource>
|
||||
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
|
||||
exec php /usr/src/code/app/worker.php jobs "$@"
|
||||
+5
-3
@@ -44,7 +44,7 @@
|
||||
}
|
||||
],
|
||||
"require": {
|
||||
"php": ">=8.3.0",
|
||||
"php": ">=8.4.0",
|
||||
"ext-curl": "*",
|
||||
"ext-imagick": "*",
|
||||
"ext-mbstring": "*",
|
||||
@@ -94,7 +94,7 @@
|
||||
"utopia-php/storage": "2.*",
|
||||
"utopia-php/system": "0.10.*",
|
||||
"utopia-php/telemetry": "^0.4",
|
||||
"utopia-php/vcs": "4.*",
|
||||
"utopia-php/vcs": "^4.3.0",
|
||||
"utopia-php/websocket": "1.0.*",
|
||||
"matomo/device-detector": "6.4.*",
|
||||
"dragonmantank/cron-expression": "3.4.*",
|
||||
@@ -104,7 +104,9 @@
|
||||
"spomky-labs/otphp": "11.*",
|
||||
"webonyx/graphql-php": "15.32.*",
|
||||
"league/csv": "9.14.*",
|
||||
"enshrined/svg-sanitize": "0.22.*"
|
||||
"enshrined/svg-sanitize": "0.22.*",
|
||||
"utopia-php/client": "^0.2.2",
|
||||
"open-runtimes/sdk-for-php": "^0.2.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"ext-fileinfo": "*",
|
||||
|
||||
Generated
+154
-8
@@ -4,7 +4,7 @@
|
||||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "d67184492e8ec8670d0e31f02d10b421",
|
||||
"content-hash": "a453a68b2d3ee2f962af0f040f000a8a",
|
||||
"packages": [
|
||||
{
|
||||
"name": "adhocore/jwt",
|
||||
@@ -1224,6 +1224,49 @@
|
||||
],
|
||||
"time": "2023-11-08T09:30:43+00:00"
|
||||
},
|
||||
{
|
||||
"name": "open-runtimes/sdk-for-php",
|
||||
"version": "0.2.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/open-runtimes/sdk-for-php.git",
|
||||
"reference": "e5cf7852630b69b419f8705482d8ff663ab7ab42"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/open-runtimes/sdk-for-php/zipball/e5cf7852630b69b419f8705482d8ff663ab7ab42",
|
||||
"reference": "e5cf7852630b69b419f8705482d8ff663ab7ab42",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"ext-curl": "*",
|
||||
"ext-json": "*",
|
||||
"php": ">=8.4",
|
||||
"utopia-php/client": "^0.2.2"
|
||||
},
|
||||
"require-dev": {
|
||||
"laravel/pint": "^1.22",
|
||||
"phpstan/phpstan": "^2.1",
|
||||
"phpunit/phpunit": "^13.1",
|
||||
"rector/rector": "^2.4"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"OpenRuntimes\\Orchestrator\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"description": "PHP SDK for the Open Runtimes orchestrator Jobs API.",
|
||||
"support": {
|
||||
"issues": "https://github.com/open-runtimes/sdk-for-php/issues",
|
||||
"source": "https://github.com/open-runtimes/sdk-for-php/tree/0.2.0"
|
||||
},
|
||||
"time": "2026-07-08T11:28:50+00:00"
|
||||
},
|
||||
{
|
||||
"name": "open-telemetry/api",
|
||||
"version": "1.9.0",
|
||||
@@ -3539,6 +3582,63 @@
|
||||
},
|
||||
"time": "2026-06-10T16:25:51+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/client",
|
||||
"version": "0.2.2",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/client.git",
|
||||
"reference": "a5e1ff4e76003ba91be8b76208695469481cf633"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/client/zipball/a5e1ff4e76003ba91be8b76208695469481cf633",
|
||||
"reference": "a5e1ff4e76003ba91be8b76208695469481cf633",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=8.4",
|
||||
"psr/http-client": "^1.0",
|
||||
"psr/http-factory": "^1.0",
|
||||
"psr/http-message": "^1.1 || ^2.0",
|
||||
"utopia-php/pools": "^1.0",
|
||||
"utopia-php/psr7": "^0.1",
|
||||
"utopia-php/span": "^1.1 || ^3.0 || ^4.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"swoole/ide-helper": "^6.0"
|
||||
},
|
||||
"suggest": {
|
||||
"ext-curl": "Required to use the cURL HTTP client adapter.",
|
||||
"ext-simplexml": "Required to decode XML responses with Response::xml().",
|
||||
"ext-swoole": "Required to use the Swoole coroutine HTTP client adapter."
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Utopia\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"description": "A lightweight PSR-18 HTTP client with cURL and Swoole coroutine backends",
|
||||
"keywords": [
|
||||
"client",
|
||||
"curl",
|
||||
"http",
|
||||
"php",
|
||||
"psr-18",
|
||||
"swoole",
|
||||
"utopia"
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/utopia-php/client/issues",
|
||||
"source": "https://github.com/utopia-php/client/tree/0.2.2"
|
||||
},
|
||||
"time": "2026-07-06T11:47:31+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/compression",
|
||||
"version": "0.1.5",
|
||||
@@ -4676,6 +4776,52 @@
|
||||
},
|
||||
"time": "2020-10-24T07:04:59+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/psr7",
|
||||
"version": "0.1.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/psr7.git",
|
||||
"reference": "079dd7f9a51a769ab560d8b6057e276ecd5dae41"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/psr7/zipball/079dd7f9a51a769ab560d8b6057e276ecd5dae41",
|
||||
"reference": "079dd7f9a51a769ab560d8b6057e276ecd5dae41",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=8.4",
|
||||
"psr/http-factory": "^1.0",
|
||||
"psr/http-message": "^1.1 || ^2.0"
|
||||
},
|
||||
"suggest": {
|
||||
"ext-simplexml": "Required to decode XML responses with Response::xml()."
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Utopia\\Psr7\\": "src/Psr7/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"description": "PSR-7 HTTP message implementations and PSR-17 factories for Utopia",
|
||||
"keywords": [
|
||||
"http",
|
||||
"php",
|
||||
"psr-17",
|
||||
"psr-7",
|
||||
"utopia"
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/utopia-php/psr7/issues",
|
||||
"source": "https://github.com/utopia-php/psr7/tree/0.1.0"
|
||||
},
|
||||
"time": "2026-07-06T11:47:31+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/query",
|
||||
"version": "0.1.1",
|
||||
@@ -5133,16 +5279,16 @@
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/vcs",
|
||||
"version": "4.2.4",
|
||||
"version": "4.4.1",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/vcs.git",
|
||||
"reference": "f19c61daa1e9594734d85ae25b3e90ba2b350d33"
|
||||
"reference": "75f36b9db5c957829a3625a91293b164f74faa0d"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/vcs/zipball/f19c61daa1e9594734d85ae25b3e90ba2b350d33",
|
||||
"reference": "f19c61daa1e9594734d85ae25b3e90ba2b350d33",
|
||||
"url": "https://api.github.com/repos/utopia-php/vcs/zipball/75f36b9db5c957829a3625a91293b164f74faa0d",
|
||||
"reference": "75f36b9db5c957829a3625a91293b164f74faa0d",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
@@ -5176,9 +5322,9 @@
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/utopia-php/vcs/issues",
|
||||
"source": "https://github.com/utopia-php/vcs/tree/4.2.4"
|
||||
"source": "https://github.com/utopia-php/vcs/tree/4.4.1"
|
||||
},
|
||||
"time": "2026-06-25T13:27:19+00:00"
|
||||
"time": "2026-07-08T18:15:35+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/websocket",
|
||||
@@ -8372,7 +8518,7 @@
|
||||
"prefer-stable": true,
|
||||
"prefer-lowest": false,
|
||||
"platform": {
|
||||
"php": ">=8.3.0",
|
||||
"php": ">=8.4.0",
|
||||
"ext-curl": "*",
|
||||
"ext-imagick": "*",
|
||||
"ext-mbstring": "*",
|
||||
|
||||
@@ -113,6 +113,18 @@ services:
|
||||
volumes:
|
||||
- ./app:/usr/src/code/app
|
||||
- ./src:/usr/src/code/src
|
||||
appwrite-worker-jobs:
|
||||
build:
|
||||
context: .
|
||||
target: development
|
||||
args:
|
||||
DEBUG: false
|
||||
TESTING: true
|
||||
VERSION: dev
|
||||
image: appwrite-dev
|
||||
volumes:
|
||||
- ./app:/usr/src/code/app
|
||||
- ./src:/usr/src/code/src
|
||||
appwrite-worker-screenshots:
|
||||
build:
|
||||
context: .
|
||||
|
||||
+72
-1
@@ -170,6 +170,11 @@ services:
|
||||
- _APP_DOMAIN_SITES
|
||||
- _APP_EXECUTOR_SECRET
|
||||
- _APP_EXECUTOR_HOST
|
||||
- _APP_BUILDS_BACKEND
|
||||
- _APP_BUILDS_VOLUME
|
||||
- _APP_JOBS_HOST
|
||||
- _APP_JOBS_SECRET
|
||||
- _APP_JOBS_ENDPOINT
|
||||
- _APP_LOGGING_CONFIG
|
||||
- _APP_LOCKING_ENABLED
|
||||
- _APP_MAINTENANCE_INTERVAL
|
||||
@@ -512,6 +517,47 @@ services:
|
||||
- _APP_DOMAIN_SITES
|
||||
extra_hosts:
|
||||
- host.docker.internal:host-gateway
|
||||
|
||||
appwrite-worker-jobs:
|
||||
entrypoint: worker-jobs
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-file: "5"
|
||||
max-size: 10m
|
||||
container_name: appwrite-worker-jobs
|
||||
image: ${_APP_IMAGE:-appwrite/appwrite}:${_APP_VERSION:-latest}
|
||||
networks:
|
||||
- appwrite
|
||||
volumes:
|
||||
- appwrite-builds:/storage/builds:rw
|
||||
depends_on:
|
||||
- redis
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=8
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
- _APP_JOBS_HOST
|
||||
- _APP_JOBS_SECRET
|
||||
- _APP_REDIS_HOST
|
||||
- _APP_REDIS_PORT
|
||||
- _APP_REDIS_USER
|
||||
- _APP_REDIS_PASS
|
||||
- _APP_DB_ADAPTER
|
||||
- _APP_DB_HOST
|
||||
- _APP_DB_PORT
|
||||
- _APP_DB_SCHEMA
|
||||
- _APP_DB_USER
|
||||
- _APP_DB_PASS
|
||||
- _APP_LOGGING_CONFIG
|
||||
- _APP_DATABASE_SHARED_TABLES
|
||||
extra_hosts:
|
||||
- host.docker.internal:host-gateway
|
||||
|
||||
appwrite-worker-screenshots:
|
||||
entrypoint: worker-screenshots
|
||||
logging:
|
||||
@@ -1188,6 +1234,28 @@ services:
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
|
||||
orchestrator-jobs:
|
||||
container_name: orchestrator-jobs
|
||||
hostname: orchestrator-jobs
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-file: "5"
|
||||
max-size: 10m
|
||||
image: ghcr.io/open-runtimes/orchestrator/jobs-service:0.4.2
|
||||
restart: unless-stopped
|
||||
user: root
|
||||
networks:
|
||||
- appwrite
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
environment:
|
||||
- ORCHESTRATOR_BACKEND=docker
|
||||
- ORCHESTRATOR_NETWORK=appwrite
|
||||
- ARTIFACT_ENDPOINT=http://orchestrator-jobs:8080
|
||||
- JOB_SIDECAR_IMAGE=ghcr.io/open-runtimes/orchestrator/job-sidecar:0.4.2
|
||||
|
||||
mariadb:
|
||||
image: mariadb:10.11
|
||||
container_name: appwrite-mariadb
|
||||
@@ -1354,7 +1422,10 @@ volumes:
|
||||
appwrite-certificates: null
|
||||
appwrite-functions: null
|
||||
appwrite-sites: null
|
||||
appwrite-builds: null
|
||||
# Pinned name (unprefixed) so the jobs-service can attach it to build workers
|
||||
# by a stable name across environments (see _APP_BUILDS_VOLUME).
|
||||
appwrite-builds:
|
||||
name: appwrite-builds
|
||||
appwrite-config: null
|
||||
appwrite-models: null
|
||||
openruntimes-node-modules-cache: null
|
||||
@@ -0,0 +1,219 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Compute;
|
||||
|
||||
use Ahc\Jwt\JWT;
|
||||
use Appwrite\Deployment\Token;
|
||||
use OpenRuntimes\Orchestrator\Enum\CallbackEvent;
|
||||
use OpenRuntimes\Orchestrator\Model\Artifact\DownloadArtifact;
|
||||
use OpenRuntimes\Orchestrator\Model\Artifact\UnarchiveArtifact;
|
||||
use OpenRuntimes\Orchestrator\Model\Callback;
|
||||
use OpenRuntimes\Orchestrator\Model\Volume;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\System\System;
|
||||
|
||||
/**
|
||||
* Builds an open-runtimes jobs-service job payload for a function deployment.
|
||||
*
|
||||
* Source crosses the boundary via the artifacts system (presigned GET download
|
||||
* + unarchive, run by the sidecar) — a GET has no request-body cap, so large
|
||||
* sources are fine. The build output and package-manager cache instead go on a
|
||||
* mounted volume: the builds storage volume is attached to the build worker at
|
||||
* its Appwrite path, so build.sh writes code.tar.gz + the cache squashfs
|
||||
* straight onto the volume Appwrite already reads. That keeps the multi-hundred-MB
|
||||
* output off the (capped) HTTP upload path and out of the Appwrite process.
|
||||
*
|
||||
* Covers function deployments whose source is a tarball: manual upload,
|
||||
* duplicate/rebuild, and templates (public GitHub tarball via `$template`).
|
||||
*/
|
||||
final class Job
|
||||
{
|
||||
/**
|
||||
* @return array<string, mixed> Named arguments for OpenRuntimes\Orchestrator\Jobs::create().
|
||||
*/
|
||||
public static function build(
|
||||
Document $project,
|
||||
Document $function,
|
||||
Document $deployment,
|
||||
array $platform,
|
||||
?Document $template = null,
|
||||
): array {
|
||||
$projectId = $project->getId();
|
||||
$deploymentId = $deployment->getId();
|
||||
$timeout = (int) System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900);
|
||||
|
||||
$version = $function->getAttribute('version', 'v2');
|
||||
$runtime = self::runtime($function, $version);
|
||||
$spec = Config::getParam('specifications')[$function->getAttribute('buildSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT)];
|
||||
$cpus = (float) ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT);
|
||||
$memory = \max((int) ($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT), 1024);
|
||||
|
||||
// The jobs-service (and the containers it spawns) reach Appwrite over
|
||||
// the internal Docker network, so the presigned + callback URLs use an
|
||||
// internal endpoint when configured, falling back to the public host.
|
||||
$protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https';
|
||||
$endpoint = System::getEnv('_APP_JOBS_ENDPOINT', "$protocol://{$platform['apiHostname']}");
|
||||
|
||||
// Source artifacts: manual/duplicate deployments download a presigned
|
||||
// tarball from Appwrite; template deployments pull the public GitHub
|
||||
// tarball directly (the unarchive auto-strips the "{repo}-{ref}/" wrapper
|
||||
// and, via subdir, extracts just the template's rootDirectory). Either
|
||||
// way the build input lands in /mnt/code/source.
|
||||
if ($template !== null) {
|
||||
$owner = $template->getAttribute('ownerName', '');
|
||||
$repository = $template->getAttribute('repositoryName', '');
|
||||
$reference = $template->getAttribute('referenceValue', '');
|
||||
$rootDirectory = \trim($template->getAttribute('rootDirectory', ''), '/');
|
||||
$sourceArtifacts = [
|
||||
new DownloadArtifact(id: 'source', in: "https://codeload.github.com/{$owner}/{$repository}/tar.gz/{$reference}", out: 'source.tar.gz'),
|
||||
new UnarchiveArtifact(id: 'extract', in: 'source.tar.gz', out: 'source', subdir: $rootDirectory !== '' ? $rootDirectory : null),
|
||||
];
|
||||
} else {
|
||||
// Presigned source-download URL (GET, no request-body cap), fetched by
|
||||
// the sidecar. Bound to this deployment + direction; valid for the whole
|
||||
// build window plus transfer slack.
|
||||
$ttl = $timeout + 300;
|
||||
$base = "{$endpoint}/v1/functions/{$function->getId()}/deployments/{$deploymentId}";
|
||||
$sourceUrl = "{$base}/download?" . \http_build_query([
|
||||
'type' => Token::TYPE_SOURCE,
|
||||
'project' => $projectId,
|
||||
'token' => Token::sign($deploymentId, Token::TYPE_SOURCE, $ttl),
|
||||
]);
|
||||
$sourceArtifacts = [
|
||||
new DownloadArtifact(id: 'source', in: $sourceUrl, out: 'source.tar.gz'),
|
||||
new UnarchiveArtifact(id: 'extract', in: 'source.tar.gz', out: 'source'),
|
||||
];
|
||||
}
|
||||
|
||||
// Output + cache live on the mounted builds volume (see class doc): the
|
||||
// build writes code.tar.gz to the deployment's build dir and the cache
|
||||
// squashfs to its per-function keyed path — both directly on the volume
|
||||
// Appwrite reads, so nothing large crosses the HTTP boundary.
|
||||
$cacheKey = self::cacheKey($projectId, $function->getId(), $runtime['image'] ?? '');
|
||||
$command = $deployment->getAttribute('buildCommands', '');
|
||||
$env = self::variables($project, $function, $deployment, $runtime, $cpus, $memory, $endpoint, $timeout) + [
|
||||
'OPEN_RUNTIMES_BUILD_INPUT_DIR' => '/mnt/code/source',
|
||||
'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => \dirname(self::buildPath($projectId, $deploymentId)),
|
||||
'OPEN_RUNTIMES_BUILD_CACHE_ARTIFACT' => self::cachePath($projectId, $cacheKey),
|
||||
];
|
||||
|
||||
return [
|
||||
'id' => self::id($projectId, $deploymentId),
|
||||
'image' => $runtime['image'],
|
||||
'command' => '/usr/local/server/helpers/build.sh ' . \escapeshellarg($command),
|
||||
'cpu' => $cpus,
|
||||
'memory' => $memory,
|
||||
'timeoutSeconds' => $timeout,
|
||||
'workspace' => '/mnt/code',
|
||||
'meta' => [
|
||||
'projectId' => $projectId,
|
||||
'deploymentId' => $deploymentId,
|
||||
'resourceId' => $function->getId(),
|
||||
'resourceType' => 'functions',
|
||||
],
|
||||
// The orchestrator expects environment as a string->string map.
|
||||
'environment' => \array_map('strval', $env),
|
||||
'artifacts' => $sourceArtifacts,
|
||||
// Attach the builds storage volume (Docker volume / K8s PVC named by
|
||||
// _APP_BUILDS_VOLUME) to the worker at its Appwrite path so build.sh
|
||||
// writes output + cache straight onto it.
|
||||
'volumes' => [
|
||||
new Volume(source: System::getEnv('_APP_BUILDS_VOLUME', 'appwrite-builds'), path: APP_STORAGE_BUILDS),
|
||||
],
|
||||
'callback' => new Callback(
|
||||
url: "{$endpoint}/v1/jobs/event?" . \http_build_query(['project' => $projectId]),
|
||||
events: [CallbackEvent::Log, CallbackEvent::Exit],
|
||||
key: System::getEnv('_APP_JOBS_SECRET', ''),
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* The jobs-service job id for a deployment build (used to submit and cancel).
|
||||
*/
|
||||
public static function id(string $projectId, string $deploymentId): string
|
||||
{
|
||||
return "{$projectId}-{$deploymentId}-build";
|
||||
}
|
||||
|
||||
/**
|
||||
* The build output path on the builds volume. build.sh writes code.tar.gz
|
||||
* into OPEN_RUNTIMES_BUILD_OUTPUT_DIR (this file's directory); pre-computed
|
||||
* so it can be persisted on the deployment before the job is submitted.
|
||||
*/
|
||||
public static function buildPath(string $projectId, string $deploymentId): string
|
||||
{
|
||||
return APP_STORAGE_BUILDS . "/app-{$projectId}/{$deploymentId}/code.tar.gz";
|
||||
}
|
||||
|
||||
/**
|
||||
* Deterministic build-cache key, shared across a function's deployments so
|
||||
* package-manager caches (npm/yarn/pnpm) survive between builds.
|
||||
*/
|
||||
public static function cacheKey(string $projectId, string $functionId, string $image): string
|
||||
{
|
||||
return \substr(\hash('sha256', "{$projectId}:{$functionId}:{$image}"), 0, 48);
|
||||
}
|
||||
|
||||
public static function cachePath(string $projectId, string $cacheKey): string
|
||||
{
|
||||
return APP_STORAGE_BUILDS . "/app-{$projectId}/cache/{$cacheKey}.sqfs";
|
||||
}
|
||||
|
||||
private static function runtime(Document $function, string $version): array
|
||||
{
|
||||
$runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []);
|
||||
$runtime = $runtimes[$function->getAttribute('runtime')] ?? null;
|
||||
if ($runtime === null) {
|
||||
throw new \Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
|
||||
}
|
||||
|
||||
return $runtime;
|
||||
}
|
||||
|
||||
private static function variables(
|
||||
Document $project,
|
||||
Document $function,
|
||||
Document $deployment,
|
||||
array $runtime,
|
||||
float $cpus,
|
||||
int $memory,
|
||||
string $endpoint,
|
||||
int $timeout,
|
||||
): array {
|
||||
$vars = [];
|
||||
|
||||
foreach ($function->getAttribute('varsProject', []) as $var) {
|
||||
$vars[$var->getAttribute('key')] = $var->getAttribute('value', '');
|
||||
}
|
||||
foreach ($function->getAttribute('vars', []) as $var) {
|
||||
$vars[$var->getAttribute('key')] = $var->getAttribute('value', '');
|
||||
}
|
||||
|
||||
$apiKey = (new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $timeout, 0))->encode([
|
||||
'projectId' => $project->getId(),
|
||||
'scopes' => $function->getAttribute('scopes', []),
|
||||
]);
|
||||
|
||||
return \array_merge($vars, [
|
||||
// Consumed by the open-runtimes build helper (build.sh).
|
||||
'OPEN_RUNTIMES_ENTRYPOINT' => $deployment->getAttribute('entrypoint', ''),
|
||||
'OPEN_RUNTIMES_OUTPUT_DIRECTORY' => $deployment->getAttribute('buildOutput', '') ?: $function->getAttribute('outputDirectory', ''),
|
||||
'APPWRITE_VERSION' => APP_VERSION_STABLE,
|
||||
'APPWRITE_REGION' => $project->getAttribute('region'),
|
||||
'APPWRITE_DEPLOYMENT_TYPE' => $deployment->getAttribute('type', ''),
|
||||
'APPWRITE_FUNCTION_API_ENDPOINT' => "{$endpoint}/v1",
|
||||
'APPWRITE_FUNCTION_API_KEY' => API_KEY_EPHEMERAL . '_' . $apiKey,
|
||||
'APPWRITE_FUNCTION_ID' => $function->getId(),
|
||||
'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'),
|
||||
'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(),
|
||||
'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(),
|
||||
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '',
|
||||
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '',
|
||||
'APPWRITE_FUNCTION_CPUS' => $cpus,
|
||||
'APPWRITE_FUNCTION_MEMORY' => $memory,
|
||||
'OPEN_RUNTIMES_NFT' => System::getEnv('_APP_OPEN_RUNTIMES_NFT', 'enabled'),
|
||||
]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Deployment;
|
||||
|
||||
use Ahc\Jwt\JWT;
|
||||
use Utopia\System\System;
|
||||
|
||||
/**
|
||||
* Short-lived, self-contained presigned token for a deployment artifact.
|
||||
*
|
||||
* Signed with `_APP_OPENSSL_KEY_V1` (HS256), it lets the jobs-service sidecar
|
||||
* hit the deployment source-download endpoint without a session or API key. The
|
||||
* token binds the deployment id and the artifact type ("source"), so a leaked
|
||||
* token cannot be repurposed for another deployment or artifact.
|
||||
*/
|
||||
final class Token
|
||||
{
|
||||
public const TYPE_SOURCE = 'source';
|
||||
|
||||
public static function sign(string $deploymentId, string $type, int $ttl): string
|
||||
{
|
||||
return (new JWT(self::key(), 'HS256', $ttl))->encode([
|
||||
'deploymentId' => $deploymentId,
|
||||
'type' => $type,
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify a token against the expected deployment id and artifact type.
|
||||
* Returns true only when the signature, expiry, deployment id and type
|
||||
* all match.
|
||||
*/
|
||||
public static function verify(string $token, string $deploymentId, string $type): bool
|
||||
{
|
||||
if ($token === '') {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
$payload = (new JWT(self::key(), 'HS256'))->decode($token);
|
||||
} catch (\Throwable) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return ($payload['deploymentId'] ?? '') === $deploymentId
|
||||
&& ($payload['type'] ?? '') === $type;
|
||||
}
|
||||
|
||||
private static function key(): string
|
||||
{
|
||||
return System::getEnv('_APP_OPENSSL_KEY_V1', '');
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,9 @@ class Event
|
||||
public const BUILDS_QUEUE_NAME = 'v1-builds';
|
||||
public const BUILDS_CLASS_NAME = 'BuildsV1';
|
||||
|
||||
public const JOBS_QUEUE_NAME = 'v1-jobs';
|
||||
public const JOBS_CLASS_NAME = 'JobsV1';
|
||||
|
||||
public const SCREENSHOTS_QUEUE_NAME = 'v1-screenshots';
|
||||
public const SCREENSHOTS_CLASS_NAME = 'ScreenshotsV1';
|
||||
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Event\Message;
|
||||
|
||||
use Utopia\Database\Document;
|
||||
|
||||
/**
|
||||
* A single jobs-service callback (CloudEvents envelope) handed off to the jobs
|
||||
* worker. `project` provides the DI context so the worker can resolve
|
||||
* `dbForProject`; `event` is the CloudEvent `type`; `data` is the event `data`
|
||||
* payload; `id` is the CloudEvent id used for de-duplication.
|
||||
*/
|
||||
final class Jobs extends Base
|
||||
{
|
||||
public function __construct(
|
||||
public readonly Document $project,
|
||||
public readonly string $id,
|
||||
public readonly string $event,
|
||||
public readonly array $data = [],
|
||||
) {
|
||||
}
|
||||
|
||||
public function toArray(): array
|
||||
{
|
||||
return [
|
||||
'project' => $this->project->getArrayCopy(),
|
||||
'id' => $this->id,
|
||||
'event' => $this->event,
|
||||
'data' => $this->data,
|
||||
];
|
||||
}
|
||||
|
||||
public static function fromArray(array $data): static
|
||||
{
|
||||
return new self(
|
||||
project: new Document($data['project'] ?? []),
|
||||
id: $data['id'] ?? '',
|
||||
event: $data['event'] ?? '',
|
||||
data: $data['data'] ?? [],
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Event\Publisher;
|
||||
|
||||
use Appwrite\Event\Message\Jobs as JobsMessage;
|
||||
use Utopia\Queue\Publisher;
|
||||
use Utopia\Queue\Queue;
|
||||
|
||||
readonly class Jobs extends Base
|
||||
{
|
||||
public function __construct(
|
||||
Publisher $publisher,
|
||||
protected Queue $queue
|
||||
) {
|
||||
parent::__construct($publisher);
|
||||
}
|
||||
|
||||
public function enqueue(JobsMessage $message, ?Queue $queue = null): string|bool
|
||||
{
|
||||
return $this->publish($queue ?? $this->queue, $message);
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments;
|
||||
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Build as BuildMessage;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
@@ -12,6 +13,7 @@ use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\MethodType;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Response;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
@@ -91,6 +93,7 @@ class Create extends Action
|
||||
->inject('deviceForFunctions')
|
||||
->inject('deviceForLocal')
|
||||
->inject('publisherForBuilds')
|
||||
->inject('jobs')
|
||||
->inject('plan')
|
||||
->inject('authorization')
|
||||
->inject('platform')
|
||||
@@ -112,6 +115,7 @@ class Create extends Action
|
||||
Device $deviceForFunctions,
|
||||
Device $deviceForLocal,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
Jobs $jobs,
|
||||
array $plan,
|
||||
Authorization $authorization,
|
||||
array $platform,
|
||||
@@ -287,7 +291,7 @@ class Create extends Action
|
||||
}
|
||||
|
||||
try {
|
||||
$locks($lockKey, 600, function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $functionId, $path, &$metadata, $mergeUploadMetadata, $platform, $project, $publisherForBuilds, $queueForEvents, $response, $type): void {
|
||||
$locks($lockKey, 600, function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $functionId, $path, &$metadata, $mergeUploadMetadata, $platform, $project, $publisherForBuilds, $jobs, $queueForEvents, $response, $type): void {
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
$uploaded = 0;
|
||||
|
||||
@@ -328,6 +332,21 @@ class Create extends Action
|
||||
|
||||
$fileSize = $deviceForFunctions->getFileSize($path);
|
||||
|
||||
// Build backend for manual-upload function deployments:
|
||||
// 'orchestrator' (open-runtimes jobs-service, submitted below
|
||||
// in the request flow) or 'executor' (default; enqueued to the
|
||||
// Builds worker). Selected by _APP_BUILDS_BACKEND.
|
||||
$useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
|
||||
|
||||
// Fields marking the deployment as queued. The build worker
|
||||
// promotes 'waiting' → 'building' on the first callback. The
|
||||
// jobs path also pre-declares buildPath so build.sh writes the
|
||||
// output straight onto the mounted builds volume.
|
||||
$buildFields = ['status' => 'waiting'];
|
||||
if ($useJobs) {
|
||||
$buildFields['buildPath'] = Job::buildPath($project->getId(), $deploymentId);
|
||||
}
|
||||
|
||||
if ($deployment->isEmpty()) {
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
@@ -349,7 +368,8 @@ class Create extends Action
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'activate' => $activate,
|
||||
'sourceMetadata' => $metadata,
|
||||
'type' => $type
|
||||
'type' => $type,
|
||||
...$buildFields,
|
||||
]));
|
||||
|
||||
} else {
|
||||
@@ -357,17 +377,22 @@ class Create extends Action
|
||||
'sourceSize' => $fileSize,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'sourceMetadata' => $metadata,
|
||||
...$buildFields,
|
||||
]));
|
||||
}
|
||||
|
||||
// Start the build
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
platform: $platform,
|
||||
));
|
||||
if ($useJobs) {
|
||||
$jobs->create(...Job::build($project, $function, $deployment, $platform));
|
||||
} else {
|
||||
// Default: build on the executor via the Builds worker.
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
platform: $platform,
|
||||
));
|
||||
}
|
||||
} else {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
@@ -392,4 +417,5 @@ class Create extends Action
|
||||
throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Deployment upload is busy. Try again.');
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Download;
|
||||
|
||||
use Appwrite\Deployment\Token;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\Platform\Action;
|
||||
use Appwrite\SDK\AuthType;
|
||||
@@ -9,13 +10,16 @@ use Appwrite\SDK\ContentType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\MethodType;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Database\Documents\User;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Database\Validator\UID;
|
||||
use Utopia\Http\Adapter\Swoole\Request;
|
||||
use Utopia\Platform\Enum;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\Storage\Device;
|
||||
use Utopia\Validator\Text;
|
||||
use Utopia\Validator\WhiteList;
|
||||
|
||||
class Get extends Action
|
||||
@@ -35,7 +39,10 @@ class Get extends Action
|
||||
->httpAlias('/v1/functions/:functionId/deployments/:deploymentId/build/download')
|
||||
->groups(['api', 'functions'])
|
||||
->desc('Get deployment download')
|
||||
->label('scope', 'functions.read')
|
||||
// Reachable by privileged callers (console/CLI via functions.read)
|
||||
// and by guests holding a valid presigned token (public); the
|
||||
// action enforces which one actually applies.
|
||||
->label('scope', ['public', 'functions.read'])
|
||||
->label('usage.resource', 'function/{request.functionId}')
|
||||
->label('resourceType', RESOURCE_TYPE_FUNCTIONS)
|
||||
->label('sdk', new Method(
|
||||
@@ -59,11 +66,14 @@ class Get extends Action
|
||||
->param('functionId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Function ID.', false, ['dbForProject'])
|
||||
->param('deploymentId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Deployment ID.', false, ['dbForProject'])
|
||||
->param('type', 'source', new WhiteList(['source', 'output']), 'Deployment file to download. Can be: "source", "output".', true, enum: new Enum(name: 'DeploymentDownloadType'))
|
||||
->param('token', '', new Text(2048), 'Presigned source-download token for accessing this deployment without a session (jobs-service).', true)
|
||||
->inject('response')
|
||||
->inject('request')
|
||||
->inject('dbForProject')
|
||||
->inject('deviceForFunctions')
|
||||
->inject('deviceForBuilds')
|
||||
->inject('user')
|
||||
->inject('authorization')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
@@ -71,18 +81,32 @@ class Get extends Action
|
||||
string $functionId,
|
||||
string $deploymentId,
|
||||
string $type,
|
||||
string $token,
|
||||
Response $response,
|
||||
Request $request,
|
||||
Database $dbForProject,
|
||||
Device $deviceForFunctions,
|
||||
Device $deviceForBuilds
|
||||
Device $deviceForBuilds,
|
||||
User $user,
|
||||
Authorization $authorization,
|
||||
) {
|
||||
$function = $dbForProject->getDocument('functions', $functionId);
|
||||
// Access is granted either to a privileged caller (console/CLI via
|
||||
// session or API key) or to a valid presigned token bound to this
|
||||
// deployment + artifact type (used by the jobs-service).
|
||||
$isPrivileged = $user->isPrivileged($authorization->getRoles()) || $user->isKey($authorization->getRoles());
|
||||
if (! $isPrivileged && ! Token::verify($token, $deploymentId, $type)) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED);
|
||||
}
|
||||
|
||||
// Access has been authorized at the endpoint level (privileged caller or
|
||||
// presigned token), so read the artifact documents without re-applying
|
||||
// document-level permissions (a token holder is not an authed user).
|
||||
$function = $authorization->skip(fn () => $dbForProject->getDocument('functions', $functionId));
|
||||
if ($function->isEmpty()) {
|
||||
throw new Exception(Exception::FUNCTION_NOT_FOUND);
|
||||
}
|
||||
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
$deployment = $authorization->skip(fn () => $dbForProject->getDocument('deployments', $deploymentId));
|
||||
if ($deployment->isEmpty()) {
|
||||
throw new Exception(Exception::DEPLOYMENT_NOT_FOUND);
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Duplicate;
|
||||
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Build as BuildMessage;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
@@ -10,6 +11,7 @@ use Appwrite\SDK\AuthType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Response;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
@@ -17,6 +19,7 @@ use Utopia\Database\Validator\UID;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\Storage\Device;
|
||||
use Utopia\System\System;
|
||||
|
||||
class Create extends Action
|
||||
{
|
||||
@@ -64,6 +67,7 @@ class Create extends Action
|
||||
->inject('dbForProject')
|
||||
->inject('queueForEvents')
|
||||
->inject('publisherForBuilds')
|
||||
->inject('jobs')
|
||||
->inject('deviceForFunctions')
|
||||
->inject('project')
|
||||
->inject('platform')
|
||||
@@ -78,6 +82,7 @@ class Create extends Action
|
||||
Database $dbForProject,
|
||||
Event $queueForEvents,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
Jobs $jobs,
|
||||
Device $deviceForFunctions,
|
||||
Document $project,
|
||||
array $platform
|
||||
@@ -100,6 +105,11 @@ class Create extends Action
|
||||
|
||||
$deploymentId = ID::unique();
|
||||
|
||||
// Build backend for the rebuild: 'orchestrator' (jobs-service, submitted
|
||||
// here) or 'executor' (default; Builds worker). The jobs path pre-declares
|
||||
// buildPath so build.sh writes output straight onto the mounted volume.
|
||||
$useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
|
||||
|
||||
$destination = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
|
||||
$deviceForFunctions->transfer($path, $destination, $deviceForFunctions);
|
||||
|
||||
@@ -117,17 +127,21 @@ class Create extends Action
|
||||
'buildDuration' => null,
|
||||
'buildSize' => null,
|
||||
'status' => 'waiting',
|
||||
'buildPath' => '',
|
||||
'buildPath' => $useJobs ? Job::buildPath($project->getId(), $deploymentId) : '',
|
||||
'buildLogs' => '',
|
||||
]));
|
||||
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
platform: $platform,
|
||||
));
|
||||
if ($useJobs) {
|
||||
$jobs->create(...Job::build($project, $function, $deployment, $platform));
|
||||
} else {
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
platform: $platform,
|
||||
));
|
||||
}
|
||||
|
||||
$queueForEvents
|
||||
->setParam('functionId', $function->getId())
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Status;
|
||||
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\SDK\AuthType;
|
||||
@@ -9,6 +10,7 @@ use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Executor\Executor;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
@@ -61,6 +63,7 @@ class Update extends Action
|
||||
->inject('project')
|
||||
->inject('queueForEvents')
|
||||
->inject('executor')
|
||||
->inject('jobs')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
@@ -71,7 +74,8 @@ class Update extends Action
|
||||
Database $dbForProject,
|
||||
Document $project,
|
||||
Event $queueForEvents,
|
||||
Executor $executor
|
||||
Executor $executor,
|
||||
Jobs $jobs,
|
||||
) {
|
||||
$function = $dbForProject->getDocument('functions', $functionId);
|
||||
|
||||
@@ -94,11 +98,7 @@ class Update extends Action
|
||||
$duration = $endTime->getTimestamp() - $startTime->getTimestamp();
|
||||
|
||||
try {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
|
||||
'buildEndedAt' => DateTime::now(),
|
||||
'buildDuration' => $duration,
|
||||
'status' => 'canceled'
|
||||
]));
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration)));
|
||||
} catch (TransactionException) {
|
||||
$deployment = $dbForProject->getDocument('deployments', $deployment->getId());
|
||||
|
||||
@@ -111,18 +111,20 @@ class Update extends Action
|
||||
}
|
||||
|
||||
if ($deployment->getAttribute('status') !== 'canceled') {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
|
||||
'buildEndedAt' => DateTime::now(),
|
||||
'buildDuration' => $duration,
|
||||
'status' => 'canceled'
|
||||
]));
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration)));
|
||||
}
|
||||
}
|
||||
|
||||
// Best-effort cleanup on both backends — the deployment is already
|
||||
// marked 'canceled', and only one backend actually holds the build.
|
||||
try {
|
||||
$executor->deleteRuntime($project->getId(), $deploymentId . "-build");
|
||||
} catch (\Throwable) {
|
||||
// Best-effort cleanup — deployment status is already 'canceled'
|
||||
}
|
||||
|
||||
try {
|
||||
$jobs->delete(Job::id($project->getId(), $deploymentId));
|
||||
} catch (\Throwable) {
|
||||
}
|
||||
|
||||
$queueForEvents
|
||||
@@ -131,4 +133,28 @@ class Update extends Action
|
||||
|
||||
$response->dynamic($deployment, Response::MODEL_DEPLOYMENT);
|
||||
}
|
||||
|
||||
/**
|
||||
* The sparse update marking a build canceled. Jobs-backed builds (identified
|
||||
* by a buildPath set at submission) have no cancel worker to write the
|
||||
* closing log line the executor's Builds worker adds, so it is appended here;
|
||||
* executor deployments get it from their worker.
|
||||
*
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function cancel(Document $deployment, int $duration): array
|
||||
{
|
||||
$update = [
|
||||
'buildEndedAt' => DateTime::now(),
|
||||
'buildDuration' => $duration,
|
||||
'status' => 'canceled',
|
||||
];
|
||||
|
||||
if ($deployment->getAttribute('buildPath', '') !== '') {
|
||||
$logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n";
|
||||
$update['buildLogs'] = \substr($logs, -APP_LOG_LENGTH_LIMIT);
|
||||
}
|
||||
|
||||
return $update;
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Template;
|
||||
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Build as BuildMessage;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
@@ -12,6 +13,7 @@ use Appwrite\SDK\AuthType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Response;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
@@ -22,6 +24,7 @@ use Utopia\Database\Validator\UID;
|
||||
use Utopia\Http\Adapter\Swoole\Request;
|
||||
use Utopia\Platform\Enum;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\System\System;
|
||||
use Utopia\Validator\Boolean;
|
||||
use Utopia\Validator\Text;
|
||||
use Utopia\Validator\WhiteList;
|
||||
@@ -80,6 +83,7 @@ class Create extends Base
|
||||
->inject('queueForEvents')
|
||||
->inject('project')
|
||||
->inject('publisherForBuilds')
|
||||
->inject('jobs')
|
||||
->inject('gitHub')
|
||||
->inject('authorization')
|
||||
->inject('platform')
|
||||
@@ -101,6 +105,7 @@ class Create extends Base
|
||||
Event $queueForEvents,
|
||||
Document $project,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
Jobs $jobs,
|
||||
GitHub $github,
|
||||
Authorization $authorization,
|
||||
array $platform
|
||||
@@ -124,6 +129,23 @@ class Create extends Base
|
||||
]);
|
||||
|
||||
if (!empty($function->getAttribute('providerRepositoryId'))) {
|
||||
// VCS-connected function: the template is merged into the user's repo
|
||||
// and pushed as a commit, then that commit is built. This stays on the
|
||||
// executor for now because it is a git *write*, which the jobs-service
|
||||
// artifact system (download/unarchive) does not cover.
|
||||
//
|
||||
// Plan to move it to the jobs backend (3a — Git Data API push, no git
|
||||
// binary), when utopia-php/vcs grows the bulk-push methods:
|
||||
// 1. download the template tarball (codeload) + read the tree of the
|
||||
// target repo/branch (github->getRepositoryTree).
|
||||
// 2. for each template file: create a blob (POST .../git/blobs).
|
||||
// 3. create a tree from the blobs on top of the branch's base tree
|
||||
// (POST .../git/trees), then a commit (POST .../git/commits), then
|
||||
// fast-forward the branch ref (PATCH .../git/refs/heads/{branch}).
|
||||
// 4. take the resulting commit hash and submit the build via
|
||||
// Job::build (same authenticated-tarball path as createVcsDeployment).
|
||||
// This replaces clone + rsync + `git commit && git push` with ~4 API
|
||||
// calls and no working tree. Until then, executor handles this branch.
|
||||
$installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId'));
|
||||
|
||||
$deployment = $this->redeployVcsFunction(
|
||||
@@ -152,6 +174,11 @@ class Create extends Base
|
||||
return;
|
||||
}
|
||||
|
||||
// Backend for the template build: 'orchestrator' (jobs-service, which
|
||||
// pulls the public GitHub tarball via artifacts) or 'executor' (default;
|
||||
// the Builds worker clones the repo). The jobs path pre-declares buildPath.
|
||||
$useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
|
||||
|
||||
$deploymentId = ID::unique();
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
@@ -173,18 +200,24 @@ class Create extends Base
|
||||
'providerBranch' => $type == GitHub::CLONE_TYPE_BRANCH ? $reference : '',
|
||||
'type' => 'vcs',
|
||||
'activate' => $activate,
|
||||
'status' => 'waiting',
|
||||
'buildPath' => $useJobs ? Job::buildPath($project->getId(), $deploymentId) : '',
|
||||
]));
|
||||
|
||||
$this->updateEmptyManualRule($project, $function, $deployment, $dbForPlatform, $authorization);
|
||||
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
template: $template,
|
||||
platform: $platform,
|
||||
));
|
||||
if ($useJobs) {
|
||||
$jobs->create(...Job::build($project, $function, $deployment, $platform, $template));
|
||||
} else {
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
template: $template,
|
||||
platform: $platform,
|
||||
));
|
||||
}
|
||||
|
||||
$queueForEvents
|
||||
->setParam('functionId', $function->getId())
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Jobs\Event;
|
||||
|
||||
use Appwrite\Event\Message\Jobs as JobsMessage;
|
||||
use Appwrite\Event\Publisher\Jobs as JobsPublisher;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\Platform\Action;
|
||||
use Appwrite\Utopia\Response;
|
||||
use OpenRuntimes\Orchestrator\Callback\Signature;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Http\Adapter\Swoole\Request;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\System\System;
|
||||
|
||||
/**
|
||||
* Ingests an open-runtimes jobs-service CloudEvents callback. Verifies the
|
||||
* HMAC signature, then hands the event off to the jobs worker which applies
|
||||
* it to the deployment. Kept thin on purpose — the request returns
|
||||
* immediately so the jobs-service delivery loop isn't blocked.
|
||||
*/
|
||||
class Create extends Action
|
||||
{
|
||||
use HTTP;
|
||||
|
||||
public static function getName()
|
||||
{
|
||||
return 'createJobEvent';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->setHttpMethod(Action::HTTP_REQUEST_METHOD_POST)
|
||||
->setHttpPath('/v1/jobs/event')
|
||||
->groups(['api'])
|
||||
->desc('Create job event')
|
||||
->label('scope', 'public')
|
||||
->inject('request')
|
||||
->inject('response')
|
||||
->inject('publisherForJobs')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
public function action(
|
||||
Request $request,
|
||||
Response $response,
|
||||
JobsPublisher $publisherForJobs,
|
||||
): void {
|
||||
$body = $request->getRawPayload();
|
||||
$secret = System::getEnv('_APP_JOBS_SECRET', '');
|
||||
$signature = $request->getHeaderLine('x-signature-256');
|
||||
|
||||
if ($secret === '' || ! Signature::verify($body, $signature, $secret)) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid jobs signature.');
|
||||
}
|
||||
|
||||
$event = \json_decode($body, true) ?? [];
|
||||
|
||||
// The project isn't a request-scoped resource here (the jobs-service is
|
||||
// the caller), so resolve it from the job meta Appwrite authored. The
|
||||
// worker reloads the full project document from this id.
|
||||
$projectId = $event['data']['meta']['projectId'] ?? '';
|
||||
|
||||
$publisherForJobs->enqueue(new JobsMessage(
|
||||
project: new Document(['$id' => $projectId]),
|
||||
id: $event['id'] ?? '',
|
||||
event: $event['type'] ?? '',
|
||||
data: $event['data'] ?? [],
|
||||
));
|
||||
|
||||
$response->noContent();
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ use Appwrite\Platform\Modules\Functions\Http\Functions\Deployment\Update as Upda
|
||||
use Appwrite\Platform\Modules\Functions\Http\Functions\Get as GetFunction;
|
||||
use Appwrite\Platform\Modules\Functions\Http\Functions\Update as UpdateFunction;
|
||||
use Appwrite\Platform\Modules\Functions\Http\Functions\XList as ListFunctions;
|
||||
use Appwrite\Platform\Modules\Functions\Http\Jobs\Event\Create as CreateJobEvent;
|
||||
use Appwrite\Platform\Modules\Functions\Http\Runtimes\XList as ListRuntimes;
|
||||
use Appwrite\Platform\Modules\Functions\Http\Specifications\XList as ListSpecifications;
|
||||
use Appwrite\Platform\Modules\Functions\Http\Templates\Get as GetTemplate;
|
||||
@@ -63,6 +64,9 @@ class Http extends Service
|
||||
$this->addAction(CreateDuplicateDeployment::getName(), new CreateDuplicateDeployment());
|
||||
$this->addAction(UpdateDeploymentStatus::getName(), new UpdateDeploymentStatus());
|
||||
|
||||
// Jobs (open-runtimes jobs-service build backend)
|
||||
$this->addAction(CreateJobEvent::getName(), new CreateJobEvent());
|
||||
|
||||
// Executions
|
||||
$this->addAction(CreateExecution::getName(), new CreateExecution());
|
||||
$this->addAction(GetExecution::getName(), new GetExecution());
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
namespace Appwrite\Platform\Modules\Functions\Services;
|
||||
|
||||
use Appwrite\Platform\Modules\Functions\Workers\Builds;
|
||||
use Appwrite\Platform\Modules\Functions\Workers\Jobs;
|
||||
use Appwrite\Platform\Modules\Functions\Workers\Screenshots;
|
||||
use Utopia\Platform\Service;
|
||||
|
||||
@@ -12,6 +13,7 @@ class Workers extends Service
|
||||
{
|
||||
$this->type = Service::TYPE_WORKER;
|
||||
$this->addAction(Builds::getName(), new Builds());
|
||||
$this->addAction(Jobs::getName(), new Jobs());
|
||||
$this->addAction(Screenshots::getName(), new Screenshots());
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,6 @@ namespace Appwrite\Platform\Modules\Functions\Workers;
|
||||
use Ahc\Jwt\JWT;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Func as FunctionMessage;
|
||||
use Appwrite\Event\Message\Usage as UsageMessage;
|
||||
use Appwrite\Event\Publisher\Func as FunctionPublisher;
|
||||
use Appwrite\Event\Publisher\Screenshot;
|
||||
use Appwrite\Event\Publisher\Usage as UsagePublisher;
|
||||
@@ -13,6 +12,7 @@ use Appwrite\Event\Realtime;
|
||||
use Appwrite\Event\Webhook;
|
||||
use Appwrite\Extend\Exception as AppwriteException;
|
||||
use Appwrite\Filter\BranchDomain as BranchDomainFilter;
|
||||
use Appwrite\Usage\Build as BuildUsage;
|
||||
use Appwrite\Usage\Context;
|
||||
use Appwrite\Utopia\Response\Model\Deployment;
|
||||
use Appwrite\Vcs\Comment;
|
||||
@@ -1259,55 +1259,7 @@ class Builds extends Action
|
||||
|
||||
protected function sendUsage(Document $resource, Document $deployment, Document $project, Context $usage, UsagePublisher $publisherForUsage): void
|
||||
{
|
||||
$spec = Config::getParam('specifications')[$resource->getAttribute('buildSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT)];
|
||||
$cpus = (int) ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT);
|
||||
$memory = (int) ($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT);
|
||||
|
||||
switch ($deployment->getAttribute('status')) {
|
||||
case 'ready':
|
||||
$usage
|
||||
->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project
|
||||
->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_SUCCESS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_SUCCESS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000);
|
||||
break;
|
||||
case 'failed':
|
||||
$usage
|
||||
->addMetric(METRIC_BUILDS_FAILED, 1) // per project
|
||||
->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_FAILED), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_FAILED), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000);
|
||||
break;
|
||||
}
|
||||
|
||||
$usage
|
||||
->addMetric(METRIC_BUILDS, 1) // per project
|
||||
->addMetric(METRIC_BUILDS_STORAGE, $deployment->getAttribute('buildSize', 0))
|
||||
->addMetric(METRIC_BUILDS_COMPUTE, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(METRIC_BUILDS_MB_SECONDS, (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus))
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0))
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus))
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0))
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus));
|
||||
|
||||
// Publish usage metrics
|
||||
if (! $usage->isEmpty()) {
|
||||
$message = new UsageMessage(
|
||||
project: $project,
|
||||
metrics: $usage->getMetrics(),
|
||||
reduce: $usage->getReduce()
|
||||
);
|
||||
$publisherForUsage->enqueue($message);
|
||||
$usage->reset();
|
||||
}
|
||||
BuildUsage::publish($usage, $resource, $deployment, $project, $publisherForUsage);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Workers;
|
||||
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Func as FunctionMessage;
|
||||
use Appwrite\Event\Message\Jobs as JobsMessage;
|
||||
use Appwrite\Event\Publisher\Func as FunctionPublisher;
|
||||
use Appwrite\Event\Publisher\Usage as UsagePublisher;
|
||||
use Appwrite\Event\Realtime;
|
||||
use Appwrite\Event\Webhook;
|
||||
use Appwrite\Usage\Build as BuildUsage;
|
||||
use Appwrite\Usage\Context as UsageContext;
|
||||
use Appwrite\Utopia\Response\Model\Deployment;
|
||||
use Utopia\Cache\Cache;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Queue\Message;
|
||||
use Utopia\Storage\Device;
|
||||
use Utopia\System\System;
|
||||
|
||||
/**
|
||||
* Applies open-runtimes jobs-service callbacks to a deployment. The API
|
||||
* callback endpoint verifies + enqueues the CloudEvent; this worker owns the
|
||||
* deployment state transitions (log streaming and the ready/failed outcome)
|
||||
* that the executor path performs inline in the Builds worker.
|
||||
*
|
||||
* Callbacks arrive concurrently and out of order (one log line each), so
|
||||
* processing for a deployment is serialized under a per-deployment lock — that
|
||||
* keeps its buildLogs a clean, monotonic append while letting different
|
||||
* deployments build in parallel. Callbacks are at-least-once, so events are
|
||||
* de-duplicated on the CloudEvent id (inside the lock, so dedup + apply is
|
||||
* atomic and a lock timeout retries cleanly).
|
||||
*/
|
||||
class Jobs extends Action
|
||||
{
|
||||
private const DEDUPE_TTL = 3600;
|
||||
private const LOCK_TTL = 30;
|
||||
private const LOCK_TIMEOUT = 10.0;
|
||||
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'jobs';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->desc('Jobs worker')
|
||||
->groups(['jobs'])
|
||||
->inject('message')
|
||||
->inject('project')
|
||||
->inject('dbForProject')
|
||||
->inject('dbForPlatform')
|
||||
->inject('queueForRealtime')
|
||||
->inject('queueForEvents')
|
||||
->inject('queueForWebhooks')
|
||||
->inject('publisherForFunctions')
|
||||
->inject('publisherForUsage')
|
||||
->inject('usage')
|
||||
->inject('deviceForBuilds')
|
||||
->inject('cache')
|
||||
->inject('locks')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
public function action(
|
||||
Message $message,
|
||||
Document $project,
|
||||
Database $dbForProject,
|
||||
Database $dbForPlatform,
|
||||
Realtime $queueForRealtime,
|
||||
Event $queueForEvents,
|
||||
Webhook $queueForWebhooks,
|
||||
FunctionPublisher $publisherForFunctions,
|
||||
UsagePublisher $publisherForUsage,
|
||||
UsageContext $usage,
|
||||
Device $deviceForBuilds,
|
||||
Cache $cache,
|
||||
callable $locks,
|
||||
): void {
|
||||
$event = JobsMessage::fromArray($message->getPayload());
|
||||
|
||||
$deploymentId = $event->data['meta']['deploymentId'] ?? '';
|
||||
if ($deploymentId === '') {
|
||||
return;
|
||||
}
|
||||
|
||||
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $cache, $deploymentId): void {
|
||||
if ($event->id !== '') {
|
||||
$key = 'jobs-event-' . $event->id;
|
||||
if ($cache->load($key, self::DEDUPE_TTL) !== false) {
|
||||
return; // already processed
|
||||
}
|
||||
$cache->save($key, true);
|
||||
}
|
||||
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
if ($deployment->isEmpty() || $deployment->getAttribute('status') === 'canceled') {
|
||||
return;
|
||||
}
|
||||
|
||||
// The build writes its output onto the mounted volume before build.sh
|
||||
// exits, so the exit callback is a truthful completion signal.
|
||||
$finalizing = $event->event === 'orchestrator.job.exit';
|
||||
|
||||
$deployment = match ($event->event) {
|
||||
'orchestrator.job.log' => $this->onLog($dbForProject, $deployment, $event->data),
|
||||
'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds),
|
||||
default => $deployment,
|
||||
};
|
||||
|
||||
// Console realtime on every callback (log stream + status).
|
||||
$queueForRealtime
|
||||
->setSubscribers(['console'])
|
||||
->setProject($project)
|
||||
->setEvent('functions.[functionId].deployments.[deploymentId].update')
|
||||
->setParam('functionId', $deployment->getAttribute('resourceId'))
|
||||
->setParam('deploymentId', $deploymentId)
|
||||
->setPayload($deployment->getArrayCopy())
|
||||
->trigger();
|
||||
|
||||
// On a real terminal outcome (not a concurrently-canceled build),
|
||||
// notify webhooks + event-triggered functions of the deployment
|
||||
// update (mirrors the executor Builds worker).
|
||||
if ($finalizing && \in_array($deployment->getAttribute('status'), ['ready', 'failed'], true)) {
|
||||
$this->dispatchUpdate($queueForEvents, $queueForWebhooks, $publisherForFunctions, $project, $deployment);
|
||||
}
|
||||
}, self::LOCK_TIMEOUT);
|
||||
}
|
||||
|
||||
private function onLog(Database $dbForProject, Document $deployment, array $data): Document
|
||||
{
|
||||
$lines = $data['lines'] ?? [];
|
||||
$chunk = \is_array($lines) ? \implode("\n", $lines) : (string) $lines;
|
||||
if ($chunk === '') {
|
||||
return $deployment;
|
||||
}
|
||||
|
||||
$logs = $this->truncate($deployment->getAttribute('buildLogs', '') . $chunk . "\n");
|
||||
$update = ['buildLogs' => $logs];
|
||||
|
||||
// First build output means the build is running: promote the queued
|
||||
// deployment to 'building' and stamp its start (mirrors the executor).
|
||||
if ($deployment->getAttribute('status') === 'waiting') {
|
||||
$update['status'] = 'building';
|
||||
$update['buildStartedAt'] = DateTime::now();
|
||||
}
|
||||
|
||||
return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($update));
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the build-command exit into a terminal outcome. The output is
|
||||
* already on the mounted builds volume, so a zero exit enforces the size
|
||||
* limit against the on-disk artifact before readying; a non-zero exit fails.
|
||||
*/
|
||||
private function onExit(
|
||||
Database $dbForProject,
|
||||
Database $dbForPlatform,
|
||||
Document $project,
|
||||
Document $deployment,
|
||||
int $exitCode,
|
||||
UsageContext $usage,
|
||||
UsagePublisher $publisherForUsage,
|
||||
Device $deviceForBuilds,
|
||||
): Document {
|
||||
if ($exitCode !== 0) {
|
||||
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage);
|
||||
}
|
||||
|
||||
$path = Job::buildPath($project->getId(), $deployment->getId());
|
||||
$size = $deviceForBuilds->exists($path) ? $deviceForBuilds->getFileSize($path) : 0;
|
||||
|
||||
$limit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000');
|
||||
if ($limit !== 0 && $size > $limit) {
|
||||
$deviceForBuilds->delete($path);
|
||||
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build size should be less than ' . \number_format($limit / (1000 * 1000), 2) . ' MBs.', $usage, $publisherForUsage);
|
||||
}
|
||||
|
||||
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, true, '', $usage, $publisherForUsage, $size);
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a terminal outcome (ready or failed) to the deployment. Owns
|
||||
* activation, usage and the latestDeployment pointer, mirroring the executor
|
||||
* Builds worker.
|
||||
*/
|
||||
private function finalize(
|
||||
Database $dbForProject,
|
||||
Database $dbForPlatform,
|
||||
Document $project,
|
||||
Document $deployment,
|
||||
bool $success,
|
||||
string $message,
|
||||
UsageContext $usage,
|
||||
UsagePublisher $publisherForUsage,
|
||||
int $buildSize = 0,
|
||||
): Document {
|
||||
$function = $dbForProject->getDocument('functions', $deployment->getAttribute('resourceId'));
|
||||
|
||||
$logs = $deployment->getAttribute('buildLogs', '');
|
||||
$startedAt = \strtotime($deployment->getAttribute('buildStartedAt', '') ?: 'now') ?: \time();
|
||||
$trailer = $success
|
||||
? "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[32m Deployment finished. \033[0m\n"
|
||||
: "\n" . ($message !== '' ? $message : 'Build failed.') . "\n";
|
||||
$update = [
|
||||
'status' => $success ? 'ready' : 'failed',
|
||||
'buildEndedAt' => DateTime::now(),
|
||||
'buildDuration' => \max(0, \time() - $startedAt),
|
||||
'buildLogs' => $this->truncate($logs . $trailer),
|
||||
];
|
||||
if ($success) {
|
||||
$update['buildSize'] = $buildSize;
|
||||
$update['totalSize'] = $buildSize + (int) $deployment->getAttribute('sourceSize', 0);
|
||||
}
|
||||
|
||||
// Guard against a concurrent cancel (updateDeploymentStatus): only
|
||||
// transition if the build wasn't canceled between the read and this
|
||||
// write, so a late outcome can't resurrect a canceled build.
|
||||
$applied = $dbForProject->updateDocuments('deployments', new Document($update), [
|
||||
Query::equal('$id', [$deployment->getId()]),
|
||||
Query::notEqual('status', 'canceled'),
|
||||
]);
|
||||
$deployment = $dbForProject->getDocument('deployments', $deployment->getId());
|
||||
|
||||
if ($applied > 0 && $success && $deployment->getAttribute('activate') === true && ! $function->isEmpty()) {
|
||||
$this->activate($dbForProject, $dbForPlatform, $project, $function, $deployment);
|
||||
}
|
||||
|
||||
// Count the build for usage/billing once it reached a terminal outcome
|
||||
// (mirrors the executor Builds worker); never for a concurrently-canceled
|
||||
// build (the guard above left it 'canceled').
|
||||
if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'], true) && ! $function->isEmpty()) {
|
||||
BuildUsage::publish($usage, $function, $deployment, $project, $publisherForUsage);
|
||||
}
|
||||
|
||||
// Keep the function's "latest deployment" pointer + status current.
|
||||
if (! $function->isEmpty()) {
|
||||
$this->updateLatestDeployment($dbForProject, $function);
|
||||
}
|
||||
|
||||
return $deployment;
|
||||
}
|
||||
|
||||
/**
|
||||
* Point the function at this deployment (auto-activate). Mirrors the
|
||||
* essential function activation from the Builds worker.
|
||||
*/
|
||||
private function activate(Database $dbForProject, Database $dbForPlatform, Document $project, Document $function, Document $deployment): void
|
||||
{
|
||||
$function = $dbForProject->updateDocument('functions', $function->getId(), new Document([
|
||||
'live' => true,
|
||||
'deploymentId' => $deployment->getId(),
|
||||
'deploymentInternalId' => $deployment->getSequence(),
|
||||
'deploymentCreatedAt' => $deployment->getCreatedAt(),
|
||||
]));
|
||||
|
||||
$dbForPlatform->forEach('rules', function (Document $rule) use ($dbForPlatform, $deployment) {
|
||||
$dbForPlatform->updateDocument('rules', $rule->getId(), new Document([
|
||||
'deploymentId' => $deployment->getId(),
|
||||
'deploymentInternalId' => $deployment->getSequence(),
|
||||
]));
|
||||
}, [
|
||||
Query::equal('projectInternalId', [$project->getSequence()]),
|
||||
Query::equal('type', ['deployment']),
|
||||
Query::equal('deploymentResourceInternalId', [$function->getSequence()]),
|
||||
Query::equal('deploymentResourceType', ['function']),
|
||||
Query::equal('trigger', ['manual']),
|
||||
Query::equal('deploymentVcsProviderBranch', ['']),
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refresh the function's latestDeployment* fields from its newest
|
||||
* deployment. Mirrors the Builds worker so the console reflects the current
|
||||
* build status.
|
||||
*/
|
||||
private function updateLatestDeployment(Database $dbForProject, Document $function): void
|
||||
{
|
||||
$latest = $dbForProject->findOne('deployments', [
|
||||
Query::equal('resourceType', ['functions']),
|
||||
Query::equal('resourceInternalId', [$function->getSequence()]),
|
||||
Query::orderDesc('$createdAt'),
|
||||
]);
|
||||
|
||||
if ($latest->isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
$dbForProject->updateDocument('functions', $function->getId(), new Document([
|
||||
'latestDeploymentId' => $latest->getId(),
|
||||
'latestDeploymentInternalId' => $latest->getSequence(),
|
||||
'latestDeploymentCreatedAt' => $latest->getCreatedAt(),
|
||||
'latestDeploymentStatus' => $latest->getAttribute('status', ''),
|
||||
]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify project webhooks and event-triggered functions of a deployment
|
||||
* update, mirroring the executor Builds worker's fan-out. queueForEvents
|
||||
* only builds the event; realtime is triggered separately in action().
|
||||
*/
|
||||
private function dispatchUpdate(
|
||||
Event $queueForEvents,
|
||||
Webhook $queueForWebhooks,
|
||||
FunctionPublisher $publisherForFunctions,
|
||||
Document $project,
|
||||
Document $deployment,
|
||||
): void {
|
||||
$model = new Deployment();
|
||||
$update = $queueForEvents
|
||||
->setProject($project)
|
||||
->setEvent('functions.[functionId].deployments.[deploymentId].update')
|
||||
->setParam('functionId', $deployment->getAttribute('resourceId'))
|
||||
->setParam('deploymentId', $deployment->getId())
|
||||
->setPayload($deployment->getArrayCopy(\array_keys($model->getRules())));
|
||||
|
||||
$queueForWebhooks->from($update)->trigger();
|
||||
|
||||
$publisherForFunctions->enqueue(FunctionMessage::fromEvent(
|
||||
event: $update->getEvent(),
|
||||
params: $update->getParams(),
|
||||
project: $update->getProject(),
|
||||
user: $update->getUser(),
|
||||
userId: $update->getUserId(),
|
||||
payload: $update->getPayload(),
|
||||
platform: $update->getPlatform(),
|
||||
));
|
||||
}
|
||||
|
||||
private function truncate(string $logs): string
|
||||
{
|
||||
$limit = APP_LOG_LENGTH_LIMIT;
|
||||
if (\strlen($logs) <= $limit) {
|
||||
return $logs;
|
||||
}
|
||||
|
||||
return \substr($logs, -$limit);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Usage;
|
||||
|
||||
use Appwrite\Event\Message\Usage as UsageMessage;
|
||||
use Appwrite\Event\Publisher\Usage as UsagePublisher;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Database\Document;
|
||||
|
||||
/**
|
||||
* Emits build usage/billing metrics for a completed deployment. Shared by both
|
||||
* build backends — the executor (Builds worker) and the jobs-service (Jobs
|
||||
* worker) — so a build is counted identically however it was produced.
|
||||
*/
|
||||
final class Build
|
||||
{
|
||||
/**
|
||||
* Record the metrics for a terminal deployment (status 'ready' or 'failed')
|
||||
* onto the usage context and publish them.
|
||||
*/
|
||||
public static function publish(
|
||||
Context $usage,
|
||||
Document $resource,
|
||||
Document $deployment,
|
||||
Document $project,
|
||||
UsagePublisher $publisherForUsage,
|
||||
): void {
|
||||
$spec = Config::getParam('specifications')[$resource->getAttribute('buildSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT)];
|
||||
$cpus = (int) ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT);
|
||||
$memory = (int) ($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT);
|
||||
|
||||
switch ($deployment->getAttribute('status')) {
|
||||
case 'ready':
|
||||
$usage
|
||||
->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project
|
||||
->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_SUCCESS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_SUCCESS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000);
|
||||
break;
|
||||
case 'failed':
|
||||
$usage
|
||||
->addMetric(METRIC_BUILDS_FAILED, 1) // per project
|
||||
->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_FAILED), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_FAILED), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000);
|
||||
break;
|
||||
}
|
||||
|
||||
$usage
|
||||
->addMetric(METRIC_BUILDS, 1) // per project
|
||||
->addMetric(METRIC_BUILDS_STORAGE, $deployment->getAttribute('buildSize', 0))
|
||||
->addMetric(METRIC_BUILDS_COMPUTE, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(METRIC_BUILDS_MB_SECONDS, (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus))
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0))
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus))
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0))
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus));
|
||||
|
||||
if (! $usage->isEmpty()) {
|
||||
$publisherForUsage->enqueue(new UsageMessage(
|
||||
project: $project,
|
||||
metrics: $usage->getMetrics(),
|
||||
reduce: $usage->getReduce()
|
||||
));
|
||||
$usage->reset();
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# End-to-end test of the jobs-service build slice, driven entirely through the
|
||||
# Appwrite CLI's `push function` — which streams the deployment build logs live,
|
||||
# exactly as an end user sees them.
|
||||
#
|
||||
# Self-contained: bootstraps a throwaway console account + project, logs the CLI
|
||||
# in, then pushes a tiny Node function twice — so the second build reuses the
|
||||
# package-manager build cache the first one saved ("Build cache hit").
|
||||
#
|
||||
# Optional env:
|
||||
# ENDPOINT Appwrite endpoint (default: http://localhost/v1)
|
||||
# FUNC_ID function id (default: testbuild)
|
||||
# COMMANDS build command (default: npm install)
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
ENDPOINT="${ENDPOINT:-http://localhost/v1}"
|
||||
FUNC_ID="${FUNC_ID:-testbuild}"
|
||||
COMMANDS="${COMMANDS:-npm install}"
|
||||
EMAIL="tb-$(date +%s)@example.com"
|
||||
PASSWORD="password123"
|
||||
|
||||
id() { python3 -c 'import sys,json;print(json.load(sys.stdin).get("$id",""))'; }
|
||||
console() { curl -s -c "$JAR" -b "$JAR" -H "X-Appwrite-Project: console" -H "Content-Type: application/json" "$@"; }
|
||||
|
||||
# --- Bootstrap a project owned by a fresh console account (REST) -------------
|
||||
JAR="$(mktemp)"
|
||||
console -X POST "$ENDPOINT/account" -d "{\"userId\":\"unique()\",\"email\":\"$EMAIL\",\"password\":\"$PASSWORD\",\"name\":\"Test\"}" >/dev/null
|
||||
console -X POST "$ENDPOINT/account/sessions/email" -d "{\"email\":\"$EMAIL\",\"password\":\"$PASSWORD\"}" >/dev/null
|
||||
TEAM="$(console -X POST "$ENDPOINT/teams" -d '{"teamId":"unique()","name":"t"}' | id)"
|
||||
PROJECT_ID="$(console -X POST "$ENDPOINT/projects" -d "{\"projectId\":\"unique()\",\"name\":\"p\",\"teamId\":\"$TEAM\",\"region\":\"default\"}" | id)"
|
||||
|
||||
# --- Log the CLI in (push uses a session, not an API key) --------------------
|
||||
appwrite login --email "$EMAIL" --password "$PASSWORD" --endpoint "$ENDPOINT" >/dev/null
|
||||
|
||||
# --- Generate a minimal Appwrite project to push -----------------------------
|
||||
WORK="$(mktemp -d)"
|
||||
mkdir -p "$WORK/functions/$FUNC_ID"
|
||||
printf 'module.exports = async ({ res }) => res.json({ ok: true });\n' > "$WORK/functions/$FUNC_ID/main.js"
|
||||
# A real dependency so `npm install` populates a cache worth reusing.
|
||||
printf '{ "name": "fn", "version": "1.0.0", "main": "main.js", "dependencies": { "ms": "2.1.3" } }\n' > "$WORK/functions/$FUNC_ID/package.json"
|
||||
cat > "$WORK/appwrite.json" <<EOF
|
||||
{
|
||||
"projectId": "$PROJECT_ID",
|
||||
"functions": [
|
||||
{
|
||||
"\$id": "$FUNC_ID",
|
||||
"name": "$FUNC_ID",
|
||||
"runtime": "node-22",
|
||||
"execute": ["any"],
|
||||
"entrypoint": "main.js",
|
||||
"commands": "$COMMANDS",
|
||||
"path": "functions/$FUNC_ID"
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
# --- Push + stream the build (real user experience) --------------------------
|
||||
# Push twice against the same function: the first build seeds the build cache,
|
||||
# the second reuses it (watch for "Build cache hit" in the second run's logs).
|
||||
cd "$WORK"
|
||||
echo "Project $PROJECT_ID — first build (cold cache), logs stream below:"
|
||||
echo
|
||||
appwrite push function --function-id "$FUNC_ID" --activate --force
|
||||
|
||||
echo
|
||||
echo "Second build (should hit the build cache seeded above):"
|
||||
echo
|
||||
appwrite push function --function-id "$FUNC_ID" --activate --force
|
||||
Reference in new issue
Block a user