mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Merge pull request #12806 from appwrite/feat/jobs-build-backend
feat(compute): build functions via open-runtimes jobs-service backend
This commit is contained in:
35 files changed
+1701
-170
No files matched your search
@@ -117,6 +117,11 @@ _APP_SITES_TIMEOUT=30
|
||||
_APP_EXECUTOR_HOST=http://exc1/v1
|
||||
_APP_EXECUTOR_SECRET=your-secret-key
|
||||
_APP_EXECUTOR_IMAGES=openruntimes/node:v5-22,openruntimes/static:v5-1
|
||||
_APP_BUILDS_BACKEND=orchestrator
|
||||
_APP_BUILDS_VOLUME=appwrite-builds
|
||||
_APP_JOBS_HOST=http://orchestrator-jobs:8080
|
||||
_APP_JOBS_SECRET=your-secret-key
|
||||
_APP_JOBS_ENDPOINT=http://appwrite
|
||||
_APP_BROWSER_HOST=http://appwrite-browser:3000/v1
|
||||
_APP_DOCKER_HUB_USERNAME=
|
||||
_APP_DOCKER_HUB_PASSWORD=
|
||||
|
||||
@@ -90,6 +90,7 @@ RUN chmod +x /usr/local/bin/doctor && \
|
||||
chmod +x /usr/local/bin/queue-count-processing && \
|
||||
chmod +x /usr/local/bin/queue-count-success && \
|
||||
chmod +x /usr/local/bin/worker-builds && \
|
||||
chmod +x /usr/local/bin/worker-jobs && \
|
||||
chmod +x /usr/local/bin/worker-screenshots && \
|
||||
chmod +x /usr/local/bin/worker-certificates && \
|
||||
chmod +x /usr/local/bin/worker-databases && \
|
||||
|
||||
@@ -979,6 +979,53 @@ return [
|
||||
'question' => '',
|
||||
'filter' => ''
|
||||
],
|
||||
[
|
||||
'name' => '_APP_BUILDS_BACKEND',
|
||||
'description' => 'Backend that builds manual-upload function deployments: "executor" (default; the open-runtimes executor, via the Builds worker) or "orchestrator" (the open-runtimes jobs-service, submitted in the request flow). Other build flows always use the executor.',
|
||||
'introduction' => '1.9.0',
|
||||
'default' => 'executor',
|
||||
'required' => false,
|
||||
'question' => '',
|
||||
'filter' => ''
|
||||
],
|
||||
[
|
||||
'name' => '_APP_BUILDS_VOLUME',
|
||||
'description' => 'The Docker volume (or Kubernetes PersistentVolumeClaim) holding build storage, attached to jobs-service build workers so they write output directly onto it. Must match the storage the "builds" device is backed by. Only used when _APP_BUILDS_BACKEND is "orchestrator".',
|
||||
'introduction' => '1.9.0',
|
||||
'default' => 'appwrite-builds',
|
||||
'required' => false,
|
||||
'question' => '',
|
||||
'filter' => ''
|
||||
],
|
||||
[
|
||||
'name' => '_APP_JOBS_HOST',
|
||||
'description' => 'The host used by Appwrite to communicate with the open-runtimes jobs-service that builds manual-upload function deployments.',
|
||||
'introduction' => '1.9.0',
|
||||
'default' => 'http://orchestrator-jobs:8080',
|
||||
'required' => false,
|
||||
'overwrite' => true,
|
||||
'question' => '',
|
||||
'filter' => ''
|
||||
],
|
||||
[
|
||||
'name' => '_APP_JOBS_SECRET',
|
||||
'description' => 'The secret used to authenticate with the jobs-service and to sign/verify job callback (HMAC) requests. Make sure to change this.',
|
||||
'introduction' => '1.9.0',
|
||||
'default' => 'your-secret-key',
|
||||
'required' => false,
|
||||
'question' => '',
|
||||
'filter' => ''
|
||||
],
|
||||
[
|
||||
'name' => '_APP_JOBS_ENDPOINT',
|
||||
'description' => 'Internal Appwrite endpoint the jobs-service (and the containers it spawns) use to reach the API over the Docker network for presigned artifact + callback URLs.',
|
||||
'introduction' => '1.9.0',
|
||||
'default' => 'http://appwrite',
|
||||
'required' => false,
|
||||
'overwrite' => true,
|
||||
'question' => '',
|
||||
'filter' => ''
|
||||
],
|
||||
[
|
||||
'name' => '_APP_BROWSER_HOST',
|
||||
'description' => 'The host used by Appwrite to communicate with the browser service for screenshots.',
|
||||
|
||||
@@ -9,6 +9,7 @@ use Appwrite\Event\Publisher\Database as DatabasePublisher;
|
||||
use Appwrite\Event\Publisher\Delete as DeletePublisher;
|
||||
use Appwrite\Event\Publisher\Execution as ExecutionPublisher;
|
||||
use Appwrite\Event\Publisher\Func as FunctionPublisher;
|
||||
use Appwrite\Event\Publisher\Jobs as JobsPublisher;
|
||||
use Appwrite\Event\Publisher\Mail as MailPublisher;
|
||||
use Appwrite\Event\Publisher\Messaging as MessagingPublisher;
|
||||
use Appwrite\Event\Publisher\Migration as MigrationPublisher;
|
||||
@@ -18,10 +19,13 @@ use Appwrite\Event\Publisher\StatsResources as StatsResourcesPublisher;
|
||||
use Appwrite\Event\Publisher\Usage as UsagePublisher;
|
||||
use Appwrite\Platform\Modules\Storage\Config\StorageCacheControl;
|
||||
use Executor\Executor;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis;
|
||||
use Utopia\Cache\Adapter\Pool as CachePool;
|
||||
use Utopia\Cache\Adapter\Sharding;
|
||||
use Utopia\Cache\Cache;
|
||||
use Utopia\Client;
|
||||
use Utopia\Client\Adapter\Curl\Client as CurlAdapter;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Console;
|
||||
use Utopia\Database\Document;
|
||||
@@ -66,6 +70,22 @@ $container->set('localeCodes', fn () => array_map(fn ($locale) => $locale['code'
|
||||
|
||||
$container->set('executor', fn () => new Executor(), []);
|
||||
|
||||
$container->set('jobs', function () {
|
||||
$client = (new Client(new CurlAdapter()))
|
||||
->withBearerAuth(System::getEnv('_APP_JOBS_SECRET', ''))
|
||||
->withTimeout(30);
|
||||
|
||||
// No host on executor-only installs: keep the injection resolvable and
|
||||
// fail at call time instead (the client is only used when
|
||||
// _APP_BUILDS_BACKEND=orchestrator, which requires _APP_JOBS_HOST).
|
||||
$host = System::getEnv('_APP_JOBS_HOST', '');
|
||||
if ($host !== '') {
|
||||
$client = $client->withBaseUri($host);
|
||||
}
|
||||
|
||||
return new Jobs($client);
|
||||
}, []);
|
||||
|
||||
$container->set('telemetry', fn () => new NoTelemetry(), []);
|
||||
|
||||
$container->set('authorization', fn () => new Authorization(), []);
|
||||
@@ -131,6 +151,11 @@ $container->set('publisherForBuilds', fn (Publisher $publisher) => new BuildPubl
|
||||
new Queue(System::getEnv('_APP_BUILDS_QUEUE_NAME', Event::BUILDS_QUEUE_NAME))
|
||||
), ['publisher']);
|
||||
|
||||
$container->set('publisherForJobs', fn (Publisher $publisher) => new JobsPublisher(
|
||||
$publisher,
|
||||
new Queue(System::getEnv('_APP_JOBS_QUEUE_NAME', Event::JOBS_QUEUE_NAME))
|
||||
), ['publisher']);
|
||||
|
||||
$container->set('publisherForDatabase', fn (Publisher $publisherDatabases) => new DatabasePublisher(
|
||||
$publisherDatabases,
|
||||
new Queue(System::getEnv('_APP_DATABASE_QUEUE_NAME', Event::DATABASE_QUEUE_NAME))
|
||||
@@ -333,6 +358,8 @@ function getDevice(string $root, string $connection = ''): Device
|
||||
}
|
||||
}
|
||||
|
||||
$container->set('geodb', fn ($register) => $register->get('geodb'), ['register']);
|
||||
|
||||
$container->set('passwordsDictionary', fn ($register) => $register->get('passwordsDictionary'), ['register']);
|
||||
|
||||
$container->set('servers', function () {
|
||||
|
||||
@@ -580,6 +580,12 @@ return function (Container $context): void {
|
||||
if (! \is_string($projectId)) {
|
||||
$projectId = $request->getHeaderLine('x-appwrite-project', '');
|
||||
}
|
||||
// For non-GET requests getParam() reads the body, so a project passed
|
||||
// as a query parameter (e.g. presigned artifact URLs) is only visible
|
||||
// via getQuery().
|
||||
if (empty($projectId)) {
|
||||
$projectId = (string) $request->getQuery('project', '');
|
||||
}
|
||||
|
||||
// Backwards compatibility for new services, originally project resources
|
||||
// These endpoints moved from /v1/projects/:projectId/<resource> to /v1/<resource>
|
||||
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
|
||||
exec php /usr/src/code/app/worker.php jobs "$@"
|
||||
+5
-3
@@ -44,7 +44,7 @@
|
||||
}
|
||||
],
|
||||
"require": {
|
||||
"php": ">=8.3.0",
|
||||
"php": ">=8.4.0",
|
||||
"ext-curl": "*",
|
||||
"ext-imagick": "*",
|
||||
"ext-mbstring": "*",
|
||||
@@ -94,7 +94,7 @@
|
||||
"utopia-php/storage": "2.*",
|
||||
"utopia-php/system": "0.10.*",
|
||||
"utopia-php/telemetry": "^0.4",
|
||||
"utopia-php/vcs": "4.*",
|
||||
"utopia-php/vcs": "^4.4.1",
|
||||
"utopia-php/websocket": "1.0.*",
|
||||
"matomo/device-detector": "6.4.*",
|
||||
"dragonmantank/cron-expression": "3.4.*",
|
||||
@@ -104,7 +104,9 @@
|
||||
"spomky-labs/otphp": "11.*",
|
||||
"webonyx/graphql-php": "15.32.*",
|
||||
"league/csv": "9.14.*",
|
||||
"enshrined/svg-sanitize": "0.22.*"
|
||||
"enshrined/svg-sanitize": "0.22.*",
|
||||
"utopia-php/client": "^0.2.2",
|
||||
"open-runtimes/sdk-for-php": "^0.2.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"ext-fileinfo": "*",
|
||||
|
||||
Generated
+154
-8
@@ -4,7 +4,7 @@
|
||||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "d67184492e8ec8670d0e31f02d10b421",
|
||||
"content-hash": "a453a68b2d3ee2f962af0f040f000a8a",
|
||||
"packages": [
|
||||
{
|
||||
"name": "adhocore/jwt",
|
||||
@@ -1224,6 +1224,49 @@
|
||||
],
|
||||
"time": "2023-11-08T09:30:43+00:00"
|
||||
},
|
||||
{
|
||||
"name": "open-runtimes/sdk-for-php",
|
||||
"version": "0.2.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/open-runtimes/sdk-for-php.git",
|
||||
"reference": "e5cf7852630b69b419f8705482d8ff663ab7ab42"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/open-runtimes/sdk-for-php/zipball/e5cf7852630b69b419f8705482d8ff663ab7ab42",
|
||||
"reference": "e5cf7852630b69b419f8705482d8ff663ab7ab42",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"ext-curl": "*",
|
||||
"ext-json": "*",
|
||||
"php": ">=8.4",
|
||||
"utopia-php/client": "^0.2.2"
|
||||
},
|
||||
"require-dev": {
|
||||
"laravel/pint": "^1.22",
|
||||
"phpstan/phpstan": "^2.1",
|
||||
"phpunit/phpunit": "^13.1",
|
||||
"rector/rector": "^2.4"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"OpenRuntimes\\Orchestrator\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"description": "PHP SDK for the Open Runtimes orchestrator Jobs API.",
|
||||
"support": {
|
||||
"issues": "https://github.com/open-runtimes/sdk-for-php/issues",
|
||||
"source": "https://github.com/open-runtimes/sdk-for-php/tree/0.2.0"
|
||||
},
|
||||
"time": "2026-07-08T11:28:50+00:00"
|
||||
},
|
||||
{
|
||||
"name": "open-telemetry/api",
|
||||
"version": "1.9.0",
|
||||
@@ -3539,6 +3582,63 @@
|
||||
},
|
||||
"time": "2026-06-10T16:25:51+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/client",
|
||||
"version": "0.2.2",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/client.git",
|
||||
"reference": "a5e1ff4e76003ba91be8b76208695469481cf633"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/client/zipball/a5e1ff4e76003ba91be8b76208695469481cf633",
|
||||
"reference": "a5e1ff4e76003ba91be8b76208695469481cf633",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=8.4",
|
||||
"psr/http-client": "^1.0",
|
||||
"psr/http-factory": "^1.0",
|
||||
"psr/http-message": "^1.1 || ^2.0",
|
||||
"utopia-php/pools": "^1.0",
|
||||
"utopia-php/psr7": "^0.1",
|
||||
"utopia-php/span": "^1.1 || ^3.0 || ^4.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"swoole/ide-helper": "^6.0"
|
||||
},
|
||||
"suggest": {
|
||||
"ext-curl": "Required to use the cURL HTTP client adapter.",
|
||||
"ext-simplexml": "Required to decode XML responses with Response::xml().",
|
||||
"ext-swoole": "Required to use the Swoole coroutine HTTP client adapter."
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Utopia\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"description": "A lightweight PSR-18 HTTP client with cURL and Swoole coroutine backends",
|
||||
"keywords": [
|
||||
"client",
|
||||
"curl",
|
||||
"http",
|
||||
"php",
|
||||
"psr-18",
|
||||
"swoole",
|
||||
"utopia"
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/utopia-php/client/issues",
|
||||
"source": "https://github.com/utopia-php/client/tree/0.2.2"
|
||||
},
|
||||
"time": "2026-07-06T11:47:31+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/compression",
|
||||
"version": "0.1.5",
|
||||
@@ -4676,6 +4776,52 @@
|
||||
},
|
||||
"time": "2020-10-24T07:04:59+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/psr7",
|
||||
"version": "0.1.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/psr7.git",
|
||||
"reference": "079dd7f9a51a769ab560d8b6057e276ecd5dae41"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/psr7/zipball/079dd7f9a51a769ab560d8b6057e276ecd5dae41",
|
||||
"reference": "079dd7f9a51a769ab560d8b6057e276ecd5dae41",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=8.4",
|
||||
"psr/http-factory": "^1.0",
|
||||
"psr/http-message": "^1.1 || ^2.0"
|
||||
},
|
||||
"suggest": {
|
||||
"ext-simplexml": "Required to decode XML responses with Response::xml()."
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Utopia\\Psr7\\": "src/Psr7/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"description": "PSR-7 HTTP message implementations and PSR-17 factories for Utopia",
|
||||
"keywords": [
|
||||
"http",
|
||||
"php",
|
||||
"psr-17",
|
||||
"psr-7",
|
||||
"utopia"
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/utopia-php/psr7/issues",
|
||||
"source": "https://github.com/utopia-php/psr7/tree/0.1.0"
|
||||
},
|
||||
"time": "2026-07-06T11:47:31+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/query",
|
||||
"version": "0.1.1",
|
||||
@@ -5133,16 +5279,16 @@
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/vcs",
|
||||
"version": "4.2.4",
|
||||
"version": "4.4.1",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/vcs.git",
|
||||
"reference": "f19c61daa1e9594734d85ae25b3e90ba2b350d33"
|
||||
"reference": "75f36b9db5c957829a3625a91293b164f74faa0d"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/vcs/zipball/f19c61daa1e9594734d85ae25b3e90ba2b350d33",
|
||||
"reference": "f19c61daa1e9594734d85ae25b3e90ba2b350d33",
|
||||
"url": "https://api.github.com/repos/utopia-php/vcs/zipball/75f36b9db5c957829a3625a91293b164f74faa0d",
|
||||
"reference": "75f36b9db5c957829a3625a91293b164f74faa0d",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
@@ -5176,9 +5322,9 @@
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/utopia-php/vcs/issues",
|
||||
"source": "https://github.com/utopia-php/vcs/tree/4.2.4"
|
||||
"source": "https://github.com/utopia-php/vcs/tree/4.4.1"
|
||||
},
|
||||
"time": "2026-06-25T13:27:19+00:00"
|
||||
"time": "2026-07-08T18:15:35+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/websocket",
|
||||
@@ -8372,7 +8518,7 @@
|
||||
"prefer-stable": true,
|
||||
"prefer-lowest": false,
|
||||
"platform": {
|
||||
"php": ">=8.3.0",
|
||||
"php": ">=8.4.0",
|
||||
"ext-curl": "*",
|
||||
"ext-imagick": "*",
|
||||
"ext-mbstring": "*",
|
||||
|
||||
@@ -113,6 +113,18 @@ services:
|
||||
volumes:
|
||||
- ./app:/usr/src/code/app
|
||||
- ./src:/usr/src/code/src
|
||||
appwrite-worker-jobs:
|
||||
build:
|
||||
context: .
|
||||
target: development
|
||||
args:
|
||||
DEBUG: false
|
||||
TESTING: true
|
||||
VERSION: dev
|
||||
image: appwrite-dev
|
||||
volumes:
|
||||
- ./app:/usr/src/code/app
|
||||
- ./src:/usr/src/code/src
|
||||
appwrite-worker-screenshots:
|
||||
build:
|
||||
context: .
|
||||
|
||||
+77
-1
@@ -170,6 +170,11 @@ services:
|
||||
- _APP_DOMAIN_SITES
|
||||
- _APP_EXECUTOR_SECRET
|
||||
- _APP_EXECUTOR_HOST
|
||||
- _APP_BUILDS_BACKEND
|
||||
- _APP_BUILDS_VOLUME
|
||||
- _APP_JOBS_HOST
|
||||
- _APP_JOBS_SECRET
|
||||
- _APP_JOBS_ENDPOINT
|
||||
- _APP_LOGGING_CONFIG
|
||||
- _APP_LOCKING_ENABLED
|
||||
- _APP_MAINTENANCE_INTERVAL
|
||||
@@ -510,8 +515,54 @@ services:
|
||||
- _APP_STORAGE_WASABI_BUCKET
|
||||
- _APP_DATABASE_SHARED_TABLES
|
||||
- _APP_DOMAIN_SITES
|
||||
- _APP_BUILDS_BACKEND
|
||||
- _APP_BUILDS_VOLUME
|
||||
- _APP_JOBS_HOST
|
||||
- _APP_JOBS_SECRET
|
||||
- _APP_JOBS_ENDPOINT
|
||||
extra_hosts:
|
||||
- host.docker.internal:host-gateway
|
||||
|
||||
appwrite-worker-jobs:
|
||||
entrypoint: worker-jobs
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-file: "5"
|
||||
max-size: 10m
|
||||
container_name: appwrite-worker-jobs
|
||||
image: ${_APP_IMAGE:-appwrite/appwrite}:${_APP_VERSION:-latest}
|
||||
networks:
|
||||
- appwrite
|
||||
volumes:
|
||||
- appwrite-builds:/storage/builds:rw
|
||||
depends_on:
|
||||
- redis
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=8
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
- _APP_JOBS_HOST
|
||||
- _APP_JOBS_SECRET
|
||||
- _APP_REDIS_HOST
|
||||
- _APP_REDIS_PORT
|
||||
- _APP_REDIS_USER
|
||||
- _APP_REDIS_PASS
|
||||
- _APP_DB_ADAPTER
|
||||
- _APP_DB_HOST
|
||||
- _APP_DB_PORT
|
||||
- _APP_DB_SCHEMA
|
||||
- _APP_DB_USER
|
||||
- _APP_DB_PASS
|
||||
- _APP_LOGGING_CONFIG
|
||||
- _APP_DATABASE_SHARED_TABLES
|
||||
extra_hosts:
|
||||
- host.docker.internal:host-gateway
|
||||
|
||||
appwrite-worker-screenshots:
|
||||
entrypoint: worker-screenshots
|
||||
logging:
|
||||
@@ -1188,6 +1239,28 @@ services:
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
|
||||
orchestrator-jobs:
|
||||
container_name: orchestrator-jobs
|
||||
hostname: orchestrator-jobs
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-file: "5"
|
||||
max-size: 10m
|
||||
image: ghcr.io/open-runtimes/orchestrator/jobs-service:0.4.3
|
||||
restart: unless-stopped
|
||||
user: root
|
||||
networks:
|
||||
- appwrite
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
environment:
|
||||
- ORCHESTRATOR_BACKEND=docker
|
||||
- ORCHESTRATOR_NETWORK=appwrite
|
||||
- ARTIFACT_ENDPOINT=http://orchestrator-jobs:8080
|
||||
- JOB_SIDECAR_IMAGE=ghcr.io/open-runtimes/orchestrator/job-sidecar:0.4.3
|
||||
|
||||
mariadb:
|
||||
image: mariadb:10.11
|
||||
container_name: appwrite-mariadb
|
||||
@@ -1354,7 +1427,10 @@ volumes:
|
||||
appwrite-certificates: null
|
||||
appwrite-functions: null
|
||||
appwrite-sites: null
|
||||
appwrite-builds: null
|
||||
# Pinned name (unprefixed) so the jobs-service can attach it to build workers
|
||||
# by a stable name across environments (see _APP_BUILDS_VOLUME).
|
||||
appwrite-builds:
|
||||
name: appwrite-builds
|
||||
appwrite-config: null
|
||||
appwrite-models: null
|
||||
openruntimes-node-modules-cache: null
|
||||
@@ -0,0 +1,234 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Compute;
|
||||
|
||||
use Ahc\Jwt\JWT;
|
||||
use Appwrite\Deployment\Token;
|
||||
use OpenRuntimes\Orchestrator\Enum\CallbackEvent;
|
||||
use OpenRuntimes\Orchestrator\Model\Artifact\DownloadArtifact;
|
||||
use OpenRuntimes\Orchestrator\Model\Artifact\StatArtifact;
|
||||
use OpenRuntimes\Orchestrator\Model\Artifact\UnarchiveArtifact;
|
||||
use OpenRuntimes\Orchestrator\Model\Callback;
|
||||
use OpenRuntimes\Orchestrator\Model\Volume;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\System\System;
|
||||
|
||||
/**
|
||||
* Builds an open-runtimes jobs-service job payload for a function deployment.
|
||||
*
|
||||
* Source crosses the boundary via the artifacts system (presigned GET download
|
||||
* + unarchive, run by the sidecar) — a GET has no request-body cap, so large
|
||||
* sources are fine. The build output and package-manager cache instead go on a
|
||||
* mounted volume: the builds storage volume is attached to the build worker at
|
||||
* its Appwrite path, so build.sh writes code.tar.gz + the cache squashfs
|
||||
* straight onto the volume Appwrite already reads. That keeps the multi-hundred-MB
|
||||
* output off the (capped) HTTP upload path and out of the Appwrite process.
|
||||
*
|
||||
* Covers function deployments whose source is a tarball: manual upload,
|
||||
* duplicate/rebuild, and templates (public GitHub tarball via `$template`).
|
||||
*/
|
||||
final class Job
|
||||
{
|
||||
/**
|
||||
* @return array<string, mixed> Named arguments for OpenRuntimes\Orchestrator\Jobs::create().
|
||||
*/
|
||||
/**
|
||||
* @param ?array{url: string, subdir?: string} $source Remote tarball source
|
||||
* (template codeload URL, or a VCS presigned URL). When null, the source is
|
||||
* the deployment's own uploaded tarball, fetched from Appwrite via a
|
||||
* presigned token (manual upload / duplicate).
|
||||
*/
|
||||
public static function build(
|
||||
Document $project,
|
||||
Document $function,
|
||||
Document $deployment,
|
||||
array $platform,
|
||||
?array $source = null,
|
||||
): array {
|
||||
$projectId = $project->getId();
|
||||
$deploymentId = $deployment->getId();
|
||||
$timeout = (int) System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900);
|
||||
|
||||
$version = $function->getAttribute('version', 'v2');
|
||||
$runtime = self::runtime($function, $version);
|
||||
$spec = Config::getParam('specifications')[$function->getAttribute('buildSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT)];
|
||||
$cpus = (float) ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT);
|
||||
$memory = \max((int) ($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT), 1024);
|
||||
|
||||
// The jobs-service (and the containers it spawns) reach Appwrite over
|
||||
// the internal Docker network, so the presigned + callback URLs use an
|
||||
// internal endpoint when configured, falling back to the public host.
|
||||
$protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https';
|
||||
$endpoint = System::getEnv('_APP_JOBS_ENDPOINT', "$protocol://{$platform['apiHostname']}");
|
||||
|
||||
// Source artifacts, both ending in /mnt/code/source:
|
||||
// - remote tarball ($source): templates (public codeload URL) and VCS
|
||||
// (a short-lived presigned URL). The unarchive auto-strips the
|
||||
// "{repo}-{ref}/" wrapper and, via subdir, extracts just the
|
||||
// rootDirectory.
|
||||
// - otherwise: the deployment's uploaded tarball, fetched from Appwrite
|
||||
// over a presigned GET (manual upload / duplicate).
|
||||
if ($source !== null) {
|
||||
$subdir = \trim($source['subdir'] ?? '', '/');
|
||||
$sourceArtifacts = [
|
||||
new DownloadArtifact(id: 'source', in: $source['url'], out: 'source.tar.gz'),
|
||||
new UnarchiveArtifact(id: 'extract', in: 'source.tar.gz', out: 'source', subdir: $subdir !== '' ? $subdir : null),
|
||||
// Appwrite never sees the remote source (the sidecar fetches it),
|
||||
// so unlike the uploaded-tarball path it can't size it. Stat the
|
||||
// downloaded archive so the orchestrator reports its byte size in
|
||||
// an artifact callback, which the worker records as sourceSize.
|
||||
new StatArtifact(id: 'sourceSize', in: 'source.tar.gz', depends: 'source'),
|
||||
];
|
||||
} else {
|
||||
// Presigned source-download URL (GET, no request-body cap), fetched by
|
||||
// the sidecar. Bound to this deployment + direction; valid for the whole
|
||||
// build window plus transfer slack.
|
||||
$ttl = $timeout + 300;
|
||||
$base = "{$endpoint}/v1/functions/{$function->getId()}/deployments/{$deploymentId}";
|
||||
$sourceUrl = "{$base}/download?" . \http_build_query([
|
||||
'type' => Token::TYPE_SOURCE,
|
||||
'project' => $projectId,
|
||||
'token' => Token::sign($deploymentId, Token::TYPE_SOURCE, $ttl),
|
||||
]);
|
||||
$sourceArtifacts = [
|
||||
new DownloadArtifact(id: 'source', in: $sourceUrl, out: 'source.tar.gz'),
|
||||
new UnarchiveArtifact(id: 'extract', in: 'source.tar.gz', out: 'source'),
|
||||
];
|
||||
}
|
||||
|
||||
// Output + cache live on the mounted builds volume (see class doc): the
|
||||
// build writes code.tar.gz to the deployment's build dir and the cache
|
||||
// squashfs to its per-function keyed path — both directly on the volume
|
||||
// Appwrite reads, so nothing large crosses the HTTP boundary.
|
||||
$cacheKey = self::cacheKey($projectId, $function->getId(), $runtime['image'] ?? '');
|
||||
$command = $deployment->getAttribute('buildCommands', '');
|
||||
$env = self::variables($project, $function, $deployment, $runtime, $cpus, $memory, $endpoint, $timeout) + [
|
||||
'OPEN_RUNTIMES_BUILD_INPUT_DIR' => '/mnt/code/source',
|
||||
'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => \dirname(self::buildPath($projectId, $deploymentId)),
|
||||
'OPEN_RUNTIMES_BUILD_CACHE_ARTIFACT' => self::cachePath($projectId, $cacheKey),
|
||||
];
|
||||
|
||||
return [
|
||||
'id' => self::id($projectId, $deploymentId),
|
||||
'image' => $runtime['image'],
|
||||
'command' => '/usr/local/server/helpers/build.sh ' . \escapeshellarg($command),
|
||||
'cpu' => $cpus,
|
||||
'memory' => $memory,
|
||||
'timeoutSeconds' => $timeout,
|
||||
'workspace' => '/mnt/code',
|
||||
'meta' => [
|
||||
'projectId' => $projectId,
|
||||
'deploymentId' => $deploymentId,
|
||||
'resourceId' => $function->getId(),
|
||||
'resourceType' => 'functions',
|
||||
],
|
||||
// The orchestrator expects environment as a string->string map.
|
||||
'environment' => \array_map('strval', $env),
|
||||
'artifacts' => $sourceArtifacts,
|
||||
// Attach the builds storage volume (Docker volume / K8s PVC named by
|
||||
// _APP_BUILDS_VOLUME) to the worker at its Appwrite path so build.sh
|
||||
// writes output + cache straight onto it.
|
||||
'volumes' => [
|
||||
new Volume(source: System::getEnv('_APP_BUILDS_VOLUME', 'appwrite-builds'), path: APP_STORAGE_BUILDS),
|
||||
],
|
||||
'callback' => new Callback(
|
||||
url: "{$endpoint}/v1/jobs/event?" . \http_build_query(['project' => $projectId]),
|
||||
// Artifact callbacks only carry the source-size stat, which exists
|
||||
// only for remote-source builds (templates / VCS).
|
||||
events: $source !== null
|
||||
? [CallbackEvent::Log, CallbackEvent::Artifact, CallbackEvent::Exit]
|
||||
: [CallbackEvent::Log, CallbackEvent::Exit],
|
||||
key: System::getEnv('_APP_JOBS_SECRET', ''),
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* The jobs-service job id for a deployment build (used to submit and cancel).
|
||||
*/
|
||||
public static function id(string $projectId, string $deploymentId): string
|
||||
{
|
||||
return "{$projectId}-{$deploymentId}-build";
|
||||
}
|
||||
|
||||
/**
|
||||
* The build output path on the builds volume. build.sh writes code.tar.gz
|
||||
* into OPEN_RUNTIMES_BUILD_OUTPUT_DIR (this file's directory); pre-computed
|
||||
* so it can be persisted on the deployment before the job is submitted.
|
||||
*/
|
||||
public static function buildPath(string $projectId, string $deploymentId): string
|
||||
{
|
||||
return APP_STORAGE_BUILDS . "/app-{$projectId}/{$deploymentId}/code.tar.gz";
|
||||
}
|
||||
|
||||
/**
|
||||
* Deterministic build-cache key, shared across a function's deployments so
|
||||
* package-manager caches (npm/yarn/pnpm) survive between builds.
|
||||
*/
|
||||
public static function cacheKey(string $projectId, string $functionId, string $image): string
|
||||
{
|
||||
return \substr(\hash('sha256', "{$projectId}:{$functionId}:{$image}"), 0, 48);
|
||||
}
|
||||
|
||||
public static function cachePath(string $projectId, string $cacheKey): string
|
||||
{
|
||||
return APP_STORAGE_BUILDS . "/app-{$projectId}/cache/{$cacheKey}.sqfs";
|
||||
}
|
||||
|
||||
private static function runtime(Document $function, string $version): array
|
||||
{
|
||||
$runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []);
|
||||
$runtime = $runtimes[$function->getAttribute('runtime')] ?? null;
|
||||
if ($runtime === null) {
|
||||
throw new \Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
|
||||
}
|
||||
|
||||
return $runtime;
|
||||
}
|
||||
|
||||
private static function variables(
|
||||
Document $project,
|
||||
Document $function,
|
||||
Document $deployment,
|
||||
array $runtime,
|
||||
float $cpus,
|
||||
int $memory,
|
||||
string $endpoint,
|
||||
int $timeout,
|
||||
): array {
|
||||
$vars = [];
|
||||
|
||||
foreach ($function->getAttribute('varsProject', []) as $var) {
|
||||
$vars[$var->getAttribute('key')] = $var->getAttribute('value', '');
|
||||
}
|
||||
foreach ($function->getAttribute('vars', []) as $var) {
|
||||
$vars[$var->getAttribute('key')] = $var->getAttribute('value', '');
|
||||
}
|
||||
|
||||
$apiKey = (new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $timeout, 0))->encode([
|
||||
'projectId' => $project->getId(),
|
||||
'scopes' => $function->getAttribute('scopes', []),
|
||||
]);
|
||||
|
||||
return \array_merge($vars, [
|
||||
// Consumed by the open-runtimes build helper (build.sh).
|
||||
'OPEN_RUNTIMES_ENTRYPOINT' => $deployment->getAttribute('entrypoint', ''),
|
||||
'OPEN_RUNTIMES_OUTPUT_DIRECTORY' => $deployment->getAttribute('buildOutput', '') ?: $function->getAttribute('outputDirectory', ''),
|
||||
'APPWRITE_VERSION' => APP_VERSION_STABLE,
|
||||
'APPWRITE_REGION' => $project->getAttribute('region'),
|
||||
'APPWRITE_DEPLOYMENT_TYPE' => $deployment->getAttribute('type', ''),
|
||||
'APPWRITE_FUNCTION_API_ENDPOINT' => "{$endpoint}/v1",
|
||||
'APPWRITE_FUNCTION_API_KEY' => API_KEY_EPHEMERAL . '_' . $apiKey,
|
||||
'APPWRITE_FUNCTION_ID' => $function->getId(),
|
||||
'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'),
|
||||
'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(),
|
||||
'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(),
|
||||
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '',
|
||||
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '',
|
||||
'APPWRITE_FUNCTION_CPUS' => $cpus,
|
||||
'APPWRITE_FUNCTION_MEMORY' => $memory,
|
||||
'OPEN_RUNTIMES_NFT' => System::getEnv('_APP_OPEN_RUNTIMES_NFT', 'enabled'),
|
||||
]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Deployment;
|
||||
|
||||
use Ahc\Jwt\JWT;
|
||||
use Utopia\System\System;
|
||||
|
||||
/**
|
||||
* Short-lived, self-contained presigned token for a deployment artifact.
|
||||
*
|
||||
* Signed with `_APP_OPENSSL_KEY_V1` (HS256), it lets the jobs-service sidecar
|
||||
* hit the deployment source-download endpoint without a session or API key. The
|
||||
* token binds the deployment id and the artifact type ("source"), so a leaked
|
||||
* token cannot be repurposed for another deployment or artifact.
|
||||
*/
|
||||
final class Token
|
||||
{
|
||||
public const TYPE_SOURCE = 'source';
|
||||
|
||||
public static function sign(string $deploymentId, string $type, int $ttl): string
|
||||
{
|
||||
return (new JWT(self::key(), 'HS256', $ttl))->encode([
|
||||
'deploymentId' => $deploymentId,
|
||||
'type' => $type,
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify a token against the expected deployment id and artifact type.
|
||||
* Returns true only when the signature, expiry, deployment id and type
|
||||
* all match.
|
||||
*/
|
||||
public static function verify(string $token, string $deploymentId, string $type): bool
|
||||
{
|
||||
if ($token === '') {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
$payload = (new JWT(self::key(), 'HS256'))->decode($token);
|
||||
} catch (\Throwable) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return ($payload['deploymentId'] ?? '') === $deploymentId
|
||||
&& ($payload['type'] ?? '') === $type;
|
||||
}
|
||||
|
||||
private static function key(): string
|
||||
{
|
||||
return System::getEnv('_APP_OPENSSL_KEY_V1', '');
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,9 @@ class Event
|
||||
public const BUILDS_QUEUE_NAME = 'v1-builds';
|
||||
public const BUILDS_CLASS_NAME = 'BuildsV1';
|
||||
|
||||
public const JOBS_QUEUE_NAME = 'v1-jobs';
|
||||
public const JOBS_CLASS_NAME = 'JobsV1';
|
||||
|
||||
public const SCREENSHOTS_QUEUE_NAME = 'v1-screenshots';
|
||||
public const SCREENSHOTS_CLASS_NAME = 'ScreenshotsV1';
|
||||
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Event\Message;
|
||||
|
||||
use Utopia\Database\Document;
|
||||
|
||||
/**
|
||||
* A single jobs-service callback (CloudEvents envelope) handed off to the jobs
|
||||
* worker. `project` provides the DI context so the worker can resolve
|
||||
* `dbForProject`; `event` is the CloudEvent `type`; `data` is the event `data`
|
||||
* payload; `id` is the CloudEvent id used for de-duplication.
|
||||
*/
|
||||
final class Jobs extends Base
|
||||
{
|
||||
public function __construct(
|
||||
public readonly Document $project,
|
||||
public readonly string $id,
|
||||
public readonly string $event,
|
||||
public readonly array $data = [],
|
||||
) {
|
||||
}
|
||||
|
||||
public function toArray(): array
|
||||
{
|
||||
return [
|
||||
'project' => $this->project->getArrayCopy(),
|
||||
'id' => $this->id,
|
||||
'event' => $this->event,
|
||||
'data' => $this->data,
|
||||
];
|
||||
}
|
||||
|
||||
public static function fromArray(array $data): static
|
||||
{
|
||||
return new self(
|
||||
project: new Document($data['project'] ?? []),
|
||||
id: $data['id'] ?? '',
|
||||
event: $data['event'] ?? '',
|
||||
data: $data['data'] ?? [],
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Event\Publisher;
|
||||
|
||||
use Appwrite\Event\Message\Jobs as JobsMessage;
|
||||
use Utopia\Queue\Publisher;
|
||||
use Utopia\Queue\Queue;
|
||||
|
||||
readonly class Jobs extends Base
|
||||
{
|
||||
public function __construct(
|
||||
Publisher $publisher,
|
||||
protected Queue $queue
|
||||
) {
|
||||
parent::__construct($publisher);
|
||||
}
|
||||
|
||||
public function enqueue(JobsMessage $message, ?Queue $queue = null): string|bool
|
||||
{
|
||||
return $this->publish($queue ?? $this->queue, $message);
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Compute;
|
||||
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Message\Build as BuildMessage;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
use Appwrite\Extend\Exception;
|
||||
@@ -9,6 +10,7 @@ use Appwrite\Filter\BranchDomain as BranchDomainFilter;
|
||||
use Appwrite\Platform\Action;
|
||||
use Appwrite\Platform\Modules\Compute\Validator\Specification as SpecificationValidator;
|
||||
use Appwrite\Platform\Permission as AppwritePermission;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
@@ -67,7 +69,7 @@ class Base extends Action
|
||||
return $allowedSpecifications[0];
|
||||
}
|
||||
|
||||
public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = ''): Document
|
||||
public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, GitHub $github, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null): Document
|
||||
{
|
||||
$deploymentId = ID::unique();
|
||||
$entrypoint = $function->getAttribute('entrypoint', '');
|
||||
@@ -118,6 +120,21 @@ class Base extends Action
|
||||
|
||||
$repositoryUrl = "https://github.com/$owner/$repositoryName";
|
||||
|
||||
// Build a plain (non-template) VCS function deployment on the jobs-service
|
||||
// when the caller opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator.
|
||||
// Sites stay on the executor; template-into-repo pushes go through the
|
||||
// Builds worker (which does the git write, then hands the build to the
|
||||
// jobs-service itself when on orchestrator). When on jobs the deployment
|
||||
// is pre-declared 'waiting' with its buildPath so build.sh writes output
|
||||
// onto the mounted volume.
|
||||
$useJobs = $jobs !== null
|
||||
&& $function->getCollection() === 'functions'
|
||||
&& $template->isEmpty()
|
||||
&& System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
|
||||
$buildFields = $useJobs
|
||||
? ['status' => 'waiting', 'buildPath' => Job::buildPath($project->getId(), $deploymentId)]
|
||||
: [];
|
||||
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
@@ -149,16 +166,26 @@ class Base extends Action
|
||||
'providerBranch' => $providerBranch,
|
||||
'providerRootDirectory' => $function->getAttribute('providerRootDirectory', ''),
|
||||
'activate' => $activate,
|
||||
...$buildFields,
|
||||
]));
|
||||
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
template: $template,
|
||||
platform: $platform,
|
||||
));
|
||||
if ($useJobs) {
|
||||
$ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch');
|
||||
$source = [
|
||||
'url' => $github->getRepositoryPresignedUrl($owner, $repositoryName, $ref),
|
||||
'subdir' => $function->getAttribute('providerRootDirectory', ''),
|
||||
];
|
||||
$jobs->create(...Job::build($project, $function, $deployment, $platform, $source));
|
||||
} else {
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
template: $template,
|
||||
platform: $platform,
|
||||
));
|
||||
}
|
||||
|
||||
return $deployment;
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments;
|
||||
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Build as BuildMessage;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
@@ -12,6 +13,7 @@ use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\MethodType;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Response;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
@@ -91,6 +93,7 @@ class Create extends Action
|
||||
->inject('deviceForFunctions')
|
||||
->inject('deviceForLocal')
|
||||
->inject('publisherForBuilds')
|
||||
->inject('jobs')
|
||||
->inject('plan')
|
||||
->inject('authorization')
|
||||
->inject('platform')
|
||||
@@ -112,6 +115,7 @@ class Create extends Action
|
||||
Device $deviceForFunctions,
|
||||
Device $deviceForLocal,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
Jobs $jobs,
|
||||
array $plan,
|
||||
Authorization $authorization,
|
||||
array $platform,
|
||||
@@ -287,7 +291,7 @@ class Create extends Action
|
||||
}
|
||||
|
||||
try {
|
||||
$locks($lockKey, 600, function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $functionId, $path, &$metadata, $mergeUploadMetadata, $platform, $project, $publisherForBuilds, $queueForEvents, $response, $type): void {
|
||||
$locks($lockKey, 600, function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $functionId, $path, &$metadata, $mergeUploadMetadata, $platform, $project, $publisherForBuilds, $jobs, $queueForEvents, $response, $type): void {
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
$uploaded = 0;
|
||||
|
||||
@@ -328,6 +332,21 @@ class Create extends Action
|
||||
|
||||
$fileSize = $deviceForFunctions->getFileSize($path);
|
||||
|
||||
// Build backend for manual-upload function deployments:
|
||||
// 'orchestrator' (open-runtimes jobs-service, submitted below
|
||||
// in the request flow) or 'executor' (default; enqueued to the
|
||||
// Builds worker). Selected by _APP_BUILDS_BACKEND.
|
||||
$useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
|
||||
|
||||
// Fields marking the deployment as queued. The build worker
|
||||
// promotes 'waiting' → 'building' on the first callback. The
|
||||
// jobs path also pre-declares buildPath so build.sh writes the
|
||||
// output straight onto the mounted builds volume.
|
||||
$buildFields = ['status' => 'waiting'];
|
||||
if ($useJobs) {
|
||||
$buildFields['buildPath'] = Job::buildPath($project->getId(), $deploymentId);
|
||||
}
|
||||
|
||||
if ($deployment->isEmpty()) {
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
@@ -349,7 +368,8 @@ class Create extends Action
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'activate' => $activate,
|
||||
'sourceMetadata' => $metadata,
|
||||
'type' => $type
|
||||
'type' => $type,
|
||||
...$buildFields,
|
||||
]));
|
||||
|
||||
} else {
|
||||
@@ -357,17 +377,22 @@ class Create extends Action
|
||||
'sourceSize' => $fileSize,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'sourceMetadata' => $metadata,
|
||||
...$buildFields,
|
||||
]));
|
||||
}
|
||||
|
||||
// Start the build
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
platform: $platform,
|
||||
));
|
||||
if ($useJobs) {
|
||||
$jobs->create(...Job::build($project, $function, $deployment, $platform));
|
||||
} else {
|
||||
// Default: build on the executor via the Builds worker.
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
platform: $platform,
|
||||
));
|
||||
}
|
||||
} else {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
@@ -392,4 +417,5 @@ class Create extends Action
|
||||
throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Deployment upload is busy. Try again.');
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Download;
|
||||
|
||||
use Appwrite\Deployment\Token;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\Platform\Action;
|
||||
use Appwrite\SDK\AuthType;
|
||||
@@ -9,13 +10,16 @@ use Appwrite\SDK\ContentType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\MethodType;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Database\Documents\User;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Database\Validator\UID;
|
||||
use Utopia\Http\Adapter\Swoole\Request;
|
||||
use Utopia\Platform\Enum;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\Storage\Device;
|
||||
use Utopia\Validator\Text;
|
||||
use Utopia\Validator\WhiteList;
|
||||
|
||||
class Get extends Action
|
||||
@@ -35,7 +39,10 @@ class Get extends Action
|
||||
->httpAlias('/v1/functions/:functionId/deployments/:deploymentId/build/download')
|
||||
->groups(['api', 'functions'])
|
||||
->desc('Get deployment download')
|
||||
->label('scope', 'functions.read')
|
||||
// Reachable by privileged callers (console/CLI via functions.read)
|
||||
// and by guests holding a valid presigned token (public); the
|
||||
// action enforces which one actually applies.
|
||||
->label('scope', ['public', 'functions.read'])
|
||||
->label('usage.resource', 'function/{request.functionId}')
|
||||
->label('resourceType', RESOURCE_TYPE_FUNCTIONS)
|
||||
->label('sdk', new Method(
|
||||
@@ -59,11 +66,14 @@ class Get extends Action
|
||||
->param('functionId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Function ID.', false, ['dbForProject'])
|
||||
->param('deploymentId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Deployment ID.', false, ['dbForProject'])
|
||||
->param('type', 'source', new WhiteList(['source', 'output']), 'Deployment file to download. Can be: "source", "output".', true, enum: new Enum(name: 'DeploymentDownloadType'))
|
||||
->param('token', '', new Text(2048), 'Presigned source-download token for accessing this deployment without a session (jobs-service).', true)
|
||||
->inject('response')
|
||||
->inject('request')
|
||||
->inject('dbForProject')
|
||||
->inject('deviceForFunctions')
|
||||
->inject('deviceForBuilds')
|
||||
->inject('user')
|
||||
->inject('authorization')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
@@ -71,18 +81,32 @@ class Get extends Action
|
||||
string $functionId,
|
||||
string $deploymentId,
|
||||
string $type,
|
||||
string $token,
|
||||
Response $response,
|
||||
Request $request,
|
||||
Database $dbForProject,
|
||||
Device $deviceForFunctions,
|
||||
Device $deviceForBuilds
|
||||
Device $deviceForBuilds,
|
||||
User $user,
|
||||
Authorization $authorization,
|
||||
) {
|
||||
$function = $dbForProject->getDocument('functions', $functionId);
|
||||
// Access is granted either to a privileged caller (console/CLI via
|
||||
// session or API key) or to a valid presigned token bound to this
|
||||
// deployment + artifact type (used by the jobs-service).
|
||||
$isPrivileged = $user->isPrivileged($authorization->getRoles()) || $user->isKey($authorization->getRoles());
|
||||
if (! $isPrivileged && ! Token::verify($token, $deploymentId, $type)) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED);
|
||||
}
|
||||
|
||||
// Access has been authorized at the endpoint level (privileged caller or
|
||||
// presigned token), so read the artifact documents without re-applying
|
||||
// document-level permissions (a token holder is not an authed user).
|
||||
$function = $authorization->skip(fn () => $dbForProject->getDocument('functions', $functionId));
|
||||
if ($function->isEmpty()) {
|
||||
throw new Exception(Exception::FUNCTION_NOT_FOUND);
|
||||
}
|
||||
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
$deployment = $authorization->skip(fn () => $dbForProject->getDocument('deployments', $deploymentId));
|
||||
if ($deployment->isEmpty()) {
|
||||
throw new Exception(Exception::DEPLOYMENT_NOT_FOUND);
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Duplicate;
|
||||
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Build as BuildMessage;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
@@ -10,6 +11,7 @@ use Appwrite\SDK\AuthType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Response;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
@@ -17,6 +19,7 @@ use Utopia\Database\Validator\UID;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\Storage\Device;
|
||||
use Utopia\System\System;
|
||||
|
||||
class Create extends Action
|
||||
{
|
||||
@@ -64,6 +67,7 @@ class Create extends Action
|
||||
->inject('dbForProject')
|
||||
->inject('queueForEvents')
|
||||
->inject('publisherForBuilds')
|
||||
->inject('jobs')
|
||||
->inject('deviceForFunctions')
|
||||
->inject('project')
|
||||
->inject('platform')
|
||||
@@ -78,6 +82,7 @@ class Create extends Action
|
||||
Database $dbForProject,
|
||||
Event $queueForEvents,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
Jobs $jobs,
|
||||
Device $deviceForFunctions,
|
||||
Document $project,
|
||||
array $platform
|
||||
@@ -100,6 +105,11 @@ class Create extends Action
|
||||
|
||||
$deploymentId = ID::unique();
|
||||
|
||||
// Build backend for the rebuild: 'orchestrator' (jobs-service, submitted
|
||||
// here) or 'executor' (default; Builds worker). The jobs path pre-declares
|
||||
// buildPath so build.sh writes output straight onto the mounted volume.
|
||||
$useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
|
||||
|
||||
$destination = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
|
||||
$deviceForFunctions->transfer($path, $destination, $deviceForFunctions);
|
||||
|
||||
@@ -117,17 +127,21 @@ class Create extends Action
|
||||
'buildDuration' => null,
|
||||
'buildSize' => null,
|
||||
'status' => 'waiting',
|
||||
'buildPath' => '',
|
||||
'buildPath' => $useJobs ? Job::buildPath($project->getId(), $deploymentId) : '',
|
||||
'buildLogs' => '',
|
||||
]));
|
||||
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
platform: $platform,
|
||||
));
|
||||
if ($useJobs) {
|
||||
$jobs->create(...Job::build($project, $function, $deployment, $platform));
|
||||
} else {
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
platform: $platform,
|
||||
));
|
||||
}
|
||||
|
||||
$queueForEvents
|
||||
->setParam('functionId', $function->getId())
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Status;
|
||||
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\SDK\AuthType;
|
||||
@@ -9,6 +10,7 @@ use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Executor\Executor;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
@@ -61,6 +63,7 @@ class Update extends Action
|
||||
->inject('project')
|
||||
->inject('queueForEvents')
|
||||
->inject('executor')
|
||||
->inject('jobs')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
@@ -71,7 +74,8 @@ class Update extends Action
|
||||
Database $dbForProject,
|
||||
Document $project,
|
||||
Event $queueForEvents,
|
||||
Executor $executor
|
||||
Executor $executor,
|
||||
Jobs $jobs,
|
||||
) {
|
||||
$function = $dbForProject->getDocument('functions', $functionId);
|
||||
|
||||
@@ -94,11 +98,7 @@ class Update extends Action
|
||||
$duration = $endTime->getTimestamp() - $startTime->getTimestamp();
|
||||
|
||||
try {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
|
||||
'buildEndedAt' => DateTime::now(),
|
||||
'buildDuration' => $duration,
|
||||
'status' => 'canceled'
|
||||
]));
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration)));
|
||||
} catch (TransactionException) {
|
||||
$deployment = $dbForProject->getDocument('deployments', $deployment->getId());
|
||||
|
||||
@@ -111,18 +111,20 @@ class Update extends Action
|
||||
}
|
||||
|
||||
if ($deployment->getAttribute('status') !== 'canceled') {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
|
||||
'buildEndedAt' => DateTime::now(),
|
||||
'buildDuration' => $duration,
|
||||
'status' => 'canceled'
|
||||
]));
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($this->cancel($deployment, $duration)));
|
||||
}
|
||||
}
|
||||
|
||||
// Best-effort cleanup on both backends — the deployment is already
|
||||
// marked 'canceled', and only one backend actually holds the build.
|
||||
try {
|
||||
$executor->deleteRuntime($project->getId(), $deploymentId . "-build");
|
||||
} catch (\Throwable) {
|
||||
// Best-effort cleanup — deployment status is already 'canceled'
|
||||
}
|
||||
|
||||
try {
|
||||
$jobs->delete(Job::id($project->getId(), $deploymentId));
|
||||
} catch (\Throwable) {
|
||||
}
|
||||
|
||||
$queueForEvents
|
||||
@@ -131,4 +133,28 @@ class Update extends Action
|
||||
|
||||
$response->dynamic($deployment, Response::MODEL_DEPLOYMENT);
|
||||
}
|
||||
|
||||
/**
|
||||
* The sparse update marking a build canceled. Jobs-backed builds (identified
|
||||
* by a buildPath set at submission) have no cancel worker to write the
|
||||
* closing log line the executor's Builds worker adds, so it is appended here;
|
||||
* executor deployments get it from their worker.
|
||||
*
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
private function cancel(Document $deployment, int $duration): array
|
||||
{
|
||||
$update = [
|
||||
'buildEndedAt' => DateTime::now(),
|
||||
'buildDuration' => $duration,
|
||||
'status' => 'canceled',
|
||||
];
|
||||
|
||||
if ($deployment->getAttribute('buildPath', '') !== '') {
|
||||
$logs = $deployment->getAttribute('buildLogs', '') . "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[33m Build has been canceled. \033[0m\n";
|
||||
$update['buildLogs'] = \substr($logs, -APP_LOG_LENGTH_LIMIT);
|
||||
}
|
||||
|
||||
return $update;
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Template;
|
||||
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Build as BuildMessage;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
@@ -12,6 +13,7 @@ use Appwrite\SDK\AuthType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Response;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
@@ -22,6 +24,7 @@ use Utopia\Database\Validator\UID;
|
||||
use Utopia\Http\Adapter\Swoole\Request;
|
||||
use Utopia\Platform\Enum;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\System\System;
|
||||
use Utopia\Validator\Boolean;
|
||||
use Utopia\Validator\Text;
|
||||
use Utopia\Validator\WhiteList;
|
||||
@@ -80,6 +83,7 @@ class Create extends Base
|
||||
->inject('queueForEvents')
|
||||
->inject('project')
|
||||
->inject('publisherForBuilds')
|
||||
->inject('jobs')
|
||||
->inject('gitHub')
|
||||
->inject('authorization')
|
||||
->inject('platform')
|
||||
@@ -101,6 +105,7 @@ class Create extends Base
|
||||
Event $queueForEvents,
|
||||
Document $project,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
Jobs $jobs,
|
||||
GitHub $github,
|
||||
Authorization $authorization,
|
||||
array $platform
|
||||
@@ -124,6 +129,10 @@ class Create extends Base
|
||||
]);
|
||||
|
||||
if (!empty($function->getAttribute('providerRepositoryId'))) {
|
||||
// VCS-connected function: the Builds worker merges the template into
|
||||
// the user's repo, pushes it as a commit, then builds that commit —
|
||||
// on the executor itself, or by submitting a job when
|
||||
// _APP_BUILDS_BACKEND=orchestrator.
|
||||
$installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId'));
|
||||
|
||||
$deployment = $this->redeployVcsFunction(
|
||||
@@ -152,6 +161,11 @@ class Create extends Base
|
||||
return;
|
||||
}
|
||||
|
||||
// Backend for the template build: 'orchestrator' (jobs-service, which
|
||||
// pulls the public GitHub tarball via artifacts) or 'executor' (default;
|
||||
// the Builds worker clones the repo). The jobs path pre-declares buildPath.
|
||||
$useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
|
||||
|
||||
$deploymentId = ID::unique();
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
@@ -173,18 +187,43 @@ class Create extends Base
|
||||
'providerBranch' => $type == GitHub::CLONE_TYPE_BRANCH ? $reference : '',
|
||||
'type' => 'vcs',
|
||||
'activate' => $activate,
|
||||
'status' => 'waiting',
|
||||
'buildPath' => $useJobs ? Job::buildPath($project->getId(), $deploymentId) : '',
|
||||
]));
|
||||
|
||||
$this->updateEmptyManualRule($project, $function, $deployment, $dbForPlatform, $authorization);
|
||||
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
template: $template,
|
||||
platform: $platform,
|
||||
));
|
||||
if ($useJobs) {
|
||||
// Templates can pin a version range (e.g. "0.3.*"); codeload only
|
||||
// takes a concrete ref, so resolve the range to the highest matching
|
||||
// tag (mirrors the executor's `git ls-remote --tags | tail -1`).
|
||||
$ref = $reference;
|
||||
if ($type === GitHub::CLONE_TYPE_TAG && \str_contains($reference, '*')) {
|
||||
try {
|
||||
$tags = $github->listTags($owner, $repository, $reference);
|
||||
$ref = \end($tags) ?: $reference;
|
||||
} catch (\Throwable) {
|
||||
// Fall back to the raw reference; the build surfaces a bad ref.
|
||||
}
|
||||
}
|
||||
|
||||
// Public template: pull the source straight from GitHub's codeload
|
||||
// tarball; unarchive strips the "{repo}-{ref}/" wrapper + the rootDirectory.
|
||||
$source = [
|
||||
'url' => "https://codeload.github.com/{$owner}/{$repository}/tar.gz/{$ref}",
|
||||
'subdir' => $rootDirectory,
|
||||
];
|
||||
$jobs->create(...Job::build($project, $function, $deployment, $platform, $source));
|
||||
} else {
|
||||
$publisherForBuilds->enqueue(new BuildMessage(
|
||||
project: $project,
|
||||
resource: $function,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
template: $template,
|
||||
platform: $platform,
|
||||
));
|
||||
}
|
||||
|
||||
$queueForEvents
|
||||
->setParam('functionId', $function->getId())
|
||||
|
||||
@@ -11,6 +11,7 @@ use Appwrite\SDK\AuthType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Response;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Validator\UID;
|
||||
@@ -73,6 +74,7 @@ class Create extends Base
|
||||
->inject('project')
|
||||
->inject('queueForEvents')
|
||||
->inject('publisherForBuilds')
|
||||
->inject('jobs')
|
||||
->inject('gitHub')
|
||||
->inject('platform')
|
||||
->callback($this->action(...));
|
||||
@@ -90,6 +92,7 @@ class Create extends Base
|
||||
Document $project,
|
||||
Event $queueForEvents,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
Jobs $jobs,
|
||||
GitHub $github,
|
||||
array $platform,
|
||||
) {
|
||||
@@ -119,7 +122,8 @@ class Create extends Base
|
||||
activate: $activate,
|
||||
platform: $platform,
|
||||
reference: $reference,
|
||||
referenceType: $type
|
||||
referenceType: $type,
|
||||
jobs: $jobs
|
||||
);
|
||||
|
||||
$queueForEvents
|
||||
|
||||
@@ -20,6 +20,7 @@ use Appwrite\Task\Validator\Cron;
|
||||
use Appwrite\Utopia\Database\Validator\CustomId;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Appwrite\Utopia\Response\Model\Rule;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Abuse\Abuse;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Database\Database;
|
||||
@@ -123,6 +124,7 @@ class Create extends Base
|
||||
->inject('project')
|
||||
->inject('queueForEvents')
|
||||
->inject('publisherForBuilds')
|
||||
->inject('jobs')
|
||||
->inject('queueForRealtime')
|
||||
->inject('queueForWebhooks')
|
||||
->inject('publisherForFunctions')
|
||||
@@ -167,6 +169,7 @@ class Create extends Base
|
||||
Document $project,
|
||||
Event $queueForEvents,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
Jobs $jobs,
|
||||
Realtime $queueForRealtime,
|
||||
Webhook $queueForWebhooks,
|
||||
FunctionPublisher $publisherForFunctions,
|
||||
@@ -343,6 +346,7 @@ class Create extends Base
|
||||
installation: $installation,
|
||||
dbForProject: $dbForProject,
|
||||
publisherForBuilds: $publisherForBuilds,
|
||||
jobs: $jobs,
|
||||
template: $template,
|
||||
github: $github,
|
||||
activate: true,
|
||||
|
||||
@@ -14,6 +14,7 @@ use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Task\Validator\Cron;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Executor\Executor;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\DateTime;
|
||||
@@ -111,6 +112,7 @@ class Update extends Base
|
||||
->inject('project')
|
||||
->inject('queueForEvents')
|
||||
->inject('publisherForBuilds')
|
||||
->inject('jobs')
|
||||
->inject('dbForPlatform')
|
||||
->inject('gitHub')
|
||||
->inject('executor')
|
||||
@@ -148,6 +150,7 @@ class Update extends Base
|
||||
Document $project,
|
||||
Event $queueForEvents,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
Jobs $jobs,
|
||||
Database $dbForPlatform,
|
||||
GitHub $github,
|
||||
Executor $executor,
|
||||
@@ -309,7 +312,7 @@ class Update extends Base
|
||||
|
||||
// Redeploy logic
|
||||
if (!$isConnected && !empty($providerRepositoryId)) {
|
||||
$this->redeployVcsFunction($request, $function, $project, $installation, $dbForProject, $publisherForBuilds, new Document(), $github, true, $platform);
|
||||
$this->redeployVcsFunction($request, $function, $project, $installation, $dbForProject, $publisherForBuilds, new Document(), $github, true, $platform, jobs: $jobs);
|
||||
}
|
||||
|
||||
// Inform scheduler if function is still active
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Http\Jobs\Event;
|
||||
|
||||
use Appwrite\Event\Message\Jobs as JobsMessage;
|
||||
use Appwrite\Event\Publisher\Jobs as JobsPublisher;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\Platform\Action;
|
||||
use Appwrite\Utopia\Response;
|
||||
use OpenRuntimes\Orchestrator\Callback\Signature;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Http\Adapter\Swoole\Request;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\System\System;
|
||||
|
||||
/**
|
||||
* Ingests an open-runtimes jobs-service CloudEvents callback. Verifies the
|
||||
* HMAC signature, then hands the event off to the jobs worker which applies
|
||||
* it to the deployment. Kept thin on purpose — the request returns
|
||||
* immediately so the jobs-service delivery loop isn't blocked.
|
||||
*/
|
||||
class Create extends Action
|
||||
{
|
||||
use HTTP;
|
||||
|
||||
public static function getName()
|
||||
{
|
||||
return 'createJobEvent';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->setHttpMethod(Action::HTTP_REQUEST_METHOD_POST)
|
||||
->setHttpPath('/v1/jobs/event')
|
||||
->groups(['api'])
|
||||
->desc('Create job event')
|
||||
->label('scope', 'public')
|
||||
->inject('request')
|
||||
->inject('response')
|
||||
->inject('publisherForJobs')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
public function action(
|
||||
Request $request,
|
||||
Response $response,
|
||||
JobsPublisher $publisherForJobs,
|
||||
): void {
|
||||
$body = $request->getRawPayload();
|
||||
$secret = System::getEnv('_APP_JOBS_SECRET', '');
|
||||
$signature = $request->getHeaderLine('x-signature-256');
|
||||
|
||||
if ($secret === '' || ! Signature::verify($body, $signature, $secret)) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid jobs signature.');
|
||||
}
|
||||
|
||||
$event = \json_decode($body, true) ?? [];
|
||||
|
||||
// The project isn't a request-scoped resource here (the jobs-service is
|
||||
// the caller), so resolve it from the job meta Appwrite authored. The
|
||||
// worker reloads the full project document from this id.
|
||||
$projectId = $event['data']['meta']['projectId'] ?? '';
|
||||
|
||||
$publisherForJobs->enqueue(new JobsMessage(
|
||||
project: new Document(['$id' => $projectId]),
|
||||
id: $event['id'] ?? '',
|
||||
event: $event['type'] ?? '',
|
||||
data: $event['data'] ?? [],
|
||||
));
|
||||
|
||||
$response->noContent();
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ use Appwrite\Platform\Modules\Functions\Http\Functions\Deployment\Update as Upda
|
||||
use Appwrite\Platform\Modules\Functions\Http\Functions\Get as GetFunction;
|
||||
use Appwrite\Platform\Modules\Functions\Http\Functions\Update as UpdateFunction;
|
||||
use Appwrite\Platform\Modules\Functions\Http\Functions\XList as ListFunctions;
|
||||
use Appwrite\Platform\Modules\Functions\Http\Jobs\Event\Create as CreateJobEvent;
|
||||
use Appwrite\Platform\Modules\Functions\Http\Runtimes\XList as ListRuntimes;
|
||||
use Appwrite\Platform\Modules\Functions\Http\Specifications\XList as ListSpecifications;
|
||||
use Appwrite\Platform\Modules\Functions\Http\Templates\Get as GetTemplate;
|
||||
@@ -63,6 +64,9 @@ class Http extends Service
|
||||
$this->addAction(CreateDuplicateDeployment::getName(), new CreateDuplicateDeployment());
|
||||
$this->addAction(UpdateDeploymentStatus::getName(), new UpdateDeploymentStatus());
|
||||
|
||||
// Jobs (open-runtimes jobs-service build backend)
|
||||
$this->addAction(CreateJobEvent::getName(), new CreateJobEvent());
|
||||
|
||||
// Executions
|
||||
$this->addAction(CreateExecution::getName(), new CreateExecution());
|
||||
$this->addAction(GetExecution::getName(), new GetExecution());
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
namespace Appwrite\Platform\Modules\Functions\Services;
|
||||
|
||||
use Appwrite\Platform\Modules\Functions\Workers\Builds;
|
||||
use Appwrite\Platform\Modules\Functions\Workers\Jobs;
|
||||
use Appwrite\Platform\Modules\Functions\Workers\Screenshots;
|
||||
use Utopia\Platform\Service;
|
||||
|
||||
@@ -12,6 +13,7 @@ class Workers extends Service
|
||||
{
|
||||
$this->type = Service::TYPE_WORKER;
|
||||
$this->addAction(Builds::getName(), new Builds());
|
||||
$this->addAction(Jobs::getName(), new Jobs());
|
||||
$this->addAction(Screenshots::getName(), new Screenshots());
|
||||
}
|
||||
}
|
||||
@@ -3,9 +3,9 @@
|
||||
namespace Appwrite\Platform\Modules\Functions\Workers;
|
||||
|
||||
use Ahc\Jwt\JWT;
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Func as FunctionMessage;
|
||||
use Appwrite\Event\Message\Usage as UsageMessage;
|
||||
use Appwrite\Event\Publisher\Func as FunctionPublisher;
|
||||
use Appwrite\Event\Publisher\Screenshot;
|
||||
use Appwrite\Event\Publisher\Usage as UsagePublisher;
|
||||
@@ -13,6 +13,7 @@ use Appwrite\Event\Realtime;
|
||||
use Appwrite\Event\Webhook;
|
||||
use Appwrite\Extend\Exception as AppwriteException;
|
||||
use Appwrite\Filter\BranchDomain as BranchDomainFilter;
|
||||
use Appwrite\Usage\Build as BuildUsage;
|
||||
use Appwrite\Usage\Context;
|
||||
use Appwrite\Utopia\Response\Model\Deployment;
|
||||
use Appwrite\Vcs\Comment;
|
||||
@@ -20,6 +21,7 @@ use Exception;
|
||||
use Executor\Exception as ExecutorException;
|
||||
use Executor\Exception\Timeout as ExecutorTimeout;
|
||||
use Executor\Executor;
|
||||
use OpenRuntimes\Orchestrator\Jobs as JobsClient;
|
||||
use Swoole\Coroutine as Co;
|
||||
use Utopia\Cache\Cache;
|
||||
use Utopia\Config\Config;
|
||||
@@ -98,6 +100,7 @@ class Builds extends Action
|
||||
->inject('deviceForFiles')
|
||||
->inject('log')
|
||||
->inject('executor')
|
||||
->inject('jobs')
|
||||
->inject('plan')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
@@ -124,6 +127,7 @@ class Builds extends Action
|
||||
Device $deviceForFiles,
|
||||
Log $log,
|
||||
Executor $executor,
|
||||
JobsClient $jobs,
|
||||
array $plan
|
||||
): void {
|
||||
$payload = $message->getPayload();
|
||||
@@ -168,6 +172,7 @@ class Builds extends Action
|
||||
$getIsResourceBlocked,
|
||||
$log,
|
||||
$executor,
|
||||
$jobs,
|
||||
$plan,
|
||||
$platform,
|
||||
(int) ($payload['timeout'] ?? System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900))
|
||||
@@ -204,6 +209,7 @@ class Builds extends Action
|
||||
callable $getIsResourceBlocked,
|
||||
Log $log,
|
||||
Executor $executor,
|
||||
JobsClient $jobs,
|
||||
array $plan,
|
||||
array $platform,
|
||||
int $timeout
|
||||
@@ -518,6 +524,29 @@ class Builds extends Action
|
||||
->trigger();
|
||||
}
|
||||
|
||||
// On the jobs backend the only build reaching this worker is the
|
||||
// template-into-repo push above (a git *write* the jobs-service
|
||||
// artifact system has no primitive for). With the commit pushed,
|
||||
// hand the build to the jobs-service like any other VCS commit:
|
||||
// source from the presigned tarball, output onto the mounted
|
||||
// builds volume, progress via callbacks to the jobs worker.
|
||||
if ($resource->getCollection() === 'functions'
|
||||
&& System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator') {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
|
||||
'buildPath' => Job::buildPath($project->getId(), $deploymentId),
|
||||
]));
|
||||
|
||||
$ref = $deployment->getAttribute('providerCommitHash') ?: $branchName;
|
||||
$jobs->create(...Job::build($project, $resource, $deployment, $platform, [
|
||||
'url' => $github->getRepositoryPresignedUrl($cloneOwner, $cloneRepository, $ref),
|
||||
'subdir' => $resource->getAttribute('providerRootDirectory', ''),
|
||||
]));
|
||||
|
||||
Console::execute('rm -rf ' . \escapeshellarg('/tmp/builds/' . $deploymentId), '', $stdout, $stderr);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$tmpPath = '/tmp/builds/' . $deploymentId;
|
||||
$tmpPathFile = $tmpPath . '/code.tar.gz';
|
||||
$localDevice = new Local();
|
||||
@@ -1259,55 +1288,7 @@ class Builds extends Action
|
||||
|
||||
protected function sendUsage(Document $resource, Document $deployment, Document $project, Context $usage, UsagePublisher $publisherForUsage): void
|
||||
{
|
||||
$spec = Config::getParam('specifications')[$resource->getAttribute('buildSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT)];
|
||||
$cpus = (int) ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT);
|
||||
$memory = (int) ($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT);
|
||||
|
||||
switch ($deployment->getAttribute('status')) {
|
||||
case 'ready':
|
||||
$usage
|
||||
->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project
|
||||
->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_SUCCESS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_SUCCESS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000);
|
||||
break;
|
||||
case 'failed':
|
||||
$usage
|
||||
->addMetric(METRIC_BUILDS_FAILED, 1) // per project
|
||||
->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_FAILED), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_FAILED), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000);
|
||||
break;
|
||||
}
|
||||
|
||||
$usage
|
||||
->addMetric(METRIC_BUILDS, 1) // per project
|
||||
->addMetric(METRIC_BUILDS_STORAGE, $deployment->getAttribute('buildSize', 0))
|
||||
->addMetric(METRIC_BUILDS_COMPUTE, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(METRIC_BUILDS_MB_SECONDS, (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus))
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0))
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus))
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0))
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus));
|
||||
|
||||
// Publish usage metrics
|
||||
if (! $usage->isEmpty()) {
|
||||
$message = new UsageMessage(
|
||||
project: $project,
|
||||
metrics: $usage->getMetrics(),
|
||||
reduce: $usage->getReduce()
|
||||
);
|
||||
$publisherForUsage->enqueue($message);
|
||||
$usage->reset();
|
||||
}
|
||||
BuildUsage::publish($usage, $resource, $deployment, $project, $publisherForUsage);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,459 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\Functions\Workers;
|
||||
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Func as FunctionMessage;
|
||||
use Appwrite\Event\Message\Jobs as JobsMessage;
|
||||
use Appwrite\Event\Publisher\Func as FunctionPublisher;
|
||||
use Appwrite\Event\Publisher\Usage as UsagePublisher;
|
||||
use Appwrite\Event\Realtime;
|
||||
use Appwrite\Event\Webhook;
|
||||
use Appwrite\Usage\Build as BuildUsage;
|
||||
use Appwrite\Usage\Context as UsageContext;
|
||||
use Appwrite\Utopia\Response\Model\Deployment;
|
||||
use Utopia\Cache\Cache;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\DateTime;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Queue\Message;
|
||||
use Utopia\Storage\Device;
|
||||
use Utopia\System\System;
|
||||
use Utopia\VCS\Adapter\Git\GitHub;
|
||||
|
||||
/**
|
||||
* Applies open-runtimes jobs-service callbacks to a deployment. The API
|
||||
* callback endpoint verifies + enqueues the CloudEvent; this worker owns the
|
||||
* deployment state transitions (log streaming and the ready/failed outcome)
|
||||
* that the executor path performs inline in the Builds worker.
|
||||
*
|
||||
* Callbacks arrive concurrently and out of order (one log line each), so
|
||||
* processing for a deployment is serialized under a per-deployment lock — that
|
||||
* keeps its buildLogs a clean, monotonic append while letting different
|
||||
* deployments build in parallel. Callbacks are at-least-once, so events are
|
||||
* de-duplicated on the CloudEvent id (inside the lock, so dedup + apply is
|
||||
* atomic and a lock timeout retries cleanly).
|
||||
*/
|
||||
class Jobs extends Action
|
||||
{
|
||||
private const DEDUPE_TTL = 3600;
|
||||
private const LOCK_TTL = 30;
|
||||
private const LOCK_TIMEOUT = 10.0;
|
||||
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'jobs';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->desc('Jobs worker')
|
||||
->groups(['jobs'])
|
||||
->inject('message')
|
||||
->inject('project')
|
||||
->inject('dbForProject')
|
||||
->inject('dbForPlatform')
|
||||
->inject('queueForRealtime')
|
||||
->inject('queueForEvents')
|
||||
->inject('queueForWebhooks')
|
||||
->inject('publisherForFunctions')
|
||||
->inject('publisherForUsage')
|
||||
->inject('usage')
|
||||
->inject('deviceForBuilds')
|
||||
->inject('gitHub')
|
||||
->inject('cache')
|
||||
->inject('locks')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
public function action(
|
||||
Message $message,
|
||||
Document $project,
|
||||
Database $dbForProject,
|
||||
Database $dbForPlatform,
|
||||
Realtime $queueForRealtime,
|
||||
Event $queueForEvents,
|
||||
Webhook $queueForWebhooks,
|
||||
FunctionPublisher $publisherForFunctions,
|
||||
UsagePublisher $publisherForUsage,
|
||||
UsageContext $usage,
|
||||
Device $deviceForBuilds,
|
||||
GitHub $github,
|
||||
Cache $cache,
|
||||
callable $locks,
|
||||
): void {
|
||||
$event = JobsMessage::fromArray($message->getPayload());
|
||||
|
||||
$deploymentId = $event->data['meta']['deploymentId'] ?? '';
|
||||
if ($deploymentId === '') {
|
||||
return;
|
||||
}
|
||||
|
||||
$locks('jobs-deployment:' . $deploymentId, self::LOCK_TTL, function () use ($event, $project, $dbForProject, $dbForPlatform, $queueForRealtime, $queueForEvents, $queueForWebhooks, $publisherForFunctions, $publisherForUsage, $usage, $deviceForBuilds, $github, $cache, $deploymentId): void {
|
||||
if ($event->id !== '') {
|
||||
$key = 'jobs-event-' . $event->id;
|
||||
if ($cache->load($key, self::DEDUPE_TTL) !== false) {
|
||||
return; // already processed
|
||||
}
|
||||
$cache->save($key, true);
|
||||
}
|
||||
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
if ($deployment->isEmpty() || $deployment->getAttribute('status') === 'canceled') {
|
||||
return;
|
||||
}
|
||||
|
||||
// The build writes its output onto the mounted volume before build.sh
|
||||
// exits, so the exit callback is a truthful completion signal.
|
||||
$finalizing = $event->event === 'orchestrator.job.exit';
|
||||
|
||||
$deployment = match ($event->event) {
|
||||
'orchestrator.job.log' => $this->onLog($dbForProject, $deployment, $event->data),
|
||||
'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $deployment, $event->data),
|
||||
'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds, $github),
|
||||
default => $deployment,
|
||||
};
|
||||
|
||||
// Console realtime on every callback (log stream + status).
|
||||
$queueForRealtime
|
||||
->setSubscribers(['console'])
|
||||
->setProject($project)
|
||||
->setEvent('functions.[functionId].deployments.[deploymentId].update')
|
||||
->setParam('functionId', $deployment->getAttribute('resourceId'))
|
||||
->setParam('deploymentId', $deploymentId)
|
||||
->setPayload($deployment->getArrayCopy())
|
||||
->trigger();
|
||||
|
||||
// On a real terminal outcome (not a concurrently-canceled build),
|
||||
// notify webhooks + event-triggered functions of the deployment
|
||||
// update (mirrors the executor Builds worker).
|
||||
if ($finalizing && \in_array($deployment->getAttribute('status'), ['ready', 'failed'], true)) {
|
||||
$this->dispatchUpdate($queueForEvents, $queueForWebhooks, $publisherForFunctions, $project, $deployment);
|
||||
}
|
||||
}, self::LOCK_TIMEOUT);
|
||||
}
|
||||
|
||||
private function onLog(Database $dbForProject, Document $deployment, array $data): Document
|
||||
{
|
||||
$lines = $data['lines'] ?? [];
|
||||
$chunk = \is_array($lines) ? \implode("\n", $lines) : (string) $lines;
|
||||
if ($chunk === '') {
|
||||
return $deployment;
|
||||
}
|
||||
|
||||
$logs = $this->truncate($deployment->getAttribute('buildLogs', '') . $chunk . "\n");
|
||||
$update = ['buildLogs' => $logs];
|
||||
|
||||
// First build output means the build is running: promote the queued
|
||||
// deployment to 'building' and stamp its start (mirrors the executor).
|
||||
// 'processing' occurs when the Builds worker prepared the source (the
|
||||
// template-into-repo push) before handing the build to the jobs-service.
|
||||
if (\in_array($deployment->getAttribute('status'), ['waiting', 'processing'], true)) {
|
||||
$update['status'] = 'building';
|
||||
$update['buildStartedAt'] = DateTime::now();
|
||||
}
|
||||
|
||||
return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document($update));
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a reported artifact size. Remote-source builds (templates / VCS)
|
||||
* fetch the source in the sidecar, so Appwrite can't size it the way the
|
||||
* uploaded-tarball path does — the job stats the downloaded archive and the
|
||||
* orchestrator reports its size here, which becomes the deployment's
|
||||
* sourceSize. This arrives before the exit callback, so finalize folds it
|
||||
* into totalSize.
|
||||
*/
|
||||
private function onArtifact(Database $dbForProject, Document $deployment, array $data): Document
|
||||
{
|
||||
if (($data['artifactId'] ?? '') !== 'sourceSize' || ($data['status'] ?? '') !== 'success') {
|
||||
return $deployment;
|
||||
}
|
||||
|
||||
// A stat artifact reports the file's byte size as its 'content'.
|
||||
$size = (int) ($data['content'] ?? 0);
|
||||
if ($size <= 0) {
|
||||
return $deployment;
|
||||
}
|
||||
|
||||
// Recompute totalSize too: this callback can land either side of exit
|
||||
// (they arrive out of order), and finalize likewise derives totalSize
|
||||
// from the current sourceSize — so whichever runs last, the two agree.
|
||||
return $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([
|
||||
'sourceSize' => $size,
|
||||
'totalSize' => $size + (int) $deployment->getAttribute('buildSize', 0),
|
||||
]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the build-command exit into a terminal outcome. The output is
|
||||
* already on the mounted builds volume, so a zero exit enforces the size
|
||||
* limit against the on-disk artifact before readying; a non-zero exit fails.
|
||||
*/
|
||||
private function onExit(
|
||||
Database $dbForProject,
|
||||
Database $dbForPlatform,
|
||||
Document $project,
|
||||
Document $deployment,
|
||||
int $exitCode,
|
||||
UsageContext $usage,
|
||||
UsagePublisher $publisherForUsage,
|
||||
Device $deviceForBuilds,
|
||||
GitHub $github,
|
||||
): Document {
|
||||
if ($exitCode !== 0) {
|
||||
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $github);
|
||||
}
|
||||
|
||||
$path = Job::buildPath($project->getId(), $deployment->getId());
|
||||
$size = $deviceForBuilds->exists($path) ? $deviceForBuilds->getFileSize($path) : 0;
|
||||
|
||||
$limit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000');
|
||||
if ($limit !== 0 && $size > $limit) {
|
||||
$deviceForBuilds->delete($path);
|
||||
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build size should be less than ' . \number_format($limit / (1000 * 1000), 2) . ' MBs.', $usage, $publisherForUsage, $github);
|
||||
}
|
||||
|
||||
return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, true, '', $usage, $publisherForUsage, $github, $size);
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a terminal outcome (ready or failed) to the deployment. Owns
|
||||
* activation, usage and the latestDeployment pointer, mirroring the executor
|
||||
* Builds worker.
|
||||
*/
|
||||
private function finalize(
|
||||
Database $dbForProject,
|
||||
Database $dbForPlatform,
|
||||
Document $project,
|
||||
Document $deployment,
|
||||
bool $success,
|
||||
string $message,
|
||||
UsageContext $usage,
|
||||
UsagePublisher $publisherForUsage,
|
||||
GitHub $github,
|
||||
int $buildSize = 0,
|
||||
): Document {
|
||||
$function = $dbForProject->getDocument('functions', $deployment->getAttribute('resourceId'));
|
||||
|
||||
$logs = $deployment->getAttribute('buildLogs', '');
|
||||
$startedAt = \strtotime($deployment->getAttribute('buildStartedAt', '') ?: 'now') ?: \time();
|
||||
$trailer = $success
|
||||
? "\033[90m[" . \date('H:i:s') . "] \033[90m[\033[0mappwrite\033[90m]\033[32m Deployment finished. \033[0m\n"
|
||||
: "\n" . ($message !== '' ? $message : 'Build failed.') . "\n";
|
||||
$update = [
|
||||
'status' => $success ? 'ready' : 'failed',
|
||||
'buildEndedAt' => DateTime::now(),
|
||||
'buildDuration' => \max(0, \time() - $startedAt),
|
||||
'buildLogs' => $this->truncate($logs . $trailer),
|
||||
];
|
||||
if ($success) {
|
||||
$update['buildSize'] = $buildSize;
|
||||
$update['totalSize'] = $buildSize + (int) $deployment->getAttribute('sourceSize', 0);
|
||||
}
|
||||
|
||||
// Guard against a concurrent cancel (updateDeploymentStatus): only
|
||||
// transition if the build wasn't canceled between the read and this
|
||||
// write, so a late outcome can't resurrect a canceled build.
|
||||
$applied = $dbForProject->updateDocuments('deployments', new Document($update), [
|
||||
Query::equal('$id', [$deployment->getId()]),
|
||||
Query::notEqual('status', 'canceled'),
|
||||
]);
|
||||
$deployment = $dbForProject->getDocument('deployments', $deployment->getId());
|
||||
|
||||
if ($applied > 0 && $success && $deployment->getAttribute('activate') === true && ! $function->isEmpty()) {
|
||||
$this->activate($dbForProject, $dbForPlatform, $project, $function, $deployment);
|
||||
}
|
||||
|
||||
// Count the build for usage/billing once it reached a terminal outcome
|
||||
// (mirrors the executor Builds worker); never for a concurrently-canceled
|
||||
// build (the guard above left it 'canceled').
|
||||
if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'], true) && ! $function->isEmpty()) {
|
||||
BuildUsage::publish($usage, $function, $deployment, $project, $publisherForUsage);
|
||||
}
|
||||
|
||||
// Keep the function's "latest deployment" pointer + status current, and
|
||||
// (re)activate its schedule so the scheduler enqueues cron executions —
|
||||
// both mirror the executor Builds worker on build completion.
|
||||
if (! $function->isEmpty()) {
|
||||
$this->updateLatestDeployment($dbForProject, $function);
|
||||
$this->schedule($dbForProject, $dbForPlatform, $function);
|
||||
}
|
||||
|
||||
// Report the terminal outcome as a VCS commit status (jobs-built VCS
|
||||
// deployments only; best-effort). "pending" at build start is a follow-up.
|
||||
$status = $deployment->getAttribute('status');
|
||||
if (\in_array($status, ['ready', 'failed'], true) && ! $function->isEmpty()) {
|
||||
$this->gitStatus($github, $dbForPlatform, $project, $function, $deployment, $status === 'ready' ? 'success' : 'failure');
|
||||
}
|
||||
|
||||
return $deployment;
|
||||
}
|
||||
|
||||
/**
|
||||
* Post a GitHub commit status for a VCS deployment (no-op for non-VCS builds
|
||||
* or silent mode). Best-effort — a failed status update never fails the build.
|
||||
*/
|
||||
private function gitStatus(GitHub $github, Database $dbForPlatform, Document $project, Document $function, Document $deployment, string $state): void
|
||||
{
|
||||
$commitHash = $deployment->getAttribute('providerCommitHash', '');
|
||||
if ($commitHash === '' || $function->getAttribute('providerSilentMode', false) === true) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
$installation = $dbForPlatform->getDocument('installations', $function->getAttribute('installationId', ''));
|
||||
$providerInstallationId = $installation->getAttribute('providerInstallationId', '');
|
||||
if ($providerInstallationId === '') {
|
||||
return;
|
||||
}
|
||||
$github->initializeVariables($providerInstallationId, System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY', ''), System::getEnv('_APP_VCS_GITHUB_APP_ID', ''));
|
||||
|
||||
$protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') === 'disabled' ? 'http' : 'https';
|
||||
$hostname = System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN', ''));
|
||||
$region = $project->getAttribute('region', 'default');
|
||||
$targetUrl = "{$protocol}://{$hostname}/console/project-{$region}-{$project->getId()}/functions/function-{$function->getId()}";
|
||||
$message = $state === 'success' ? 'Build succeeded.' : 'Build failed.';
|
||||
$name = $function->getAttribute('name', '') . ' (' . $project->getAttribute('name', '') . ')';
|
||||
|
||||
$github->updateCommitStatus(
|
||||
$deployment->getAttribute('providerRepositoryName', ''),
|
||||
$commitHash,
|
||||
$deployment->getAttribute('providerRepositoryOwner', ''),
|
||||
$state,
|
||||
$message,
|
||||
$targetUrl,
|
||||
$name,
|
||||
);
|
||||
} catch (\Throwable) {
|
||||
// Best-effort; the build outcome stands regardless of the status update.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Point the function at this deployment (auto-activate). Mirrors the
|
||||
* essential function activation from the Builds worker.
|
||||
*/
|
||||
private function activate(Database $dbForProject, Database $dbForPlatform, Document $project, Document $function, Document $deployment): void
|
||||
{
|
||||
$function = $dbForProject->updateDocument('functions', $function->getId(), new Document([
|
||||
'live' => true,
|
||||
'deploymentId' => $deployment->getId(),
|
||||
'deploymentInternalId' => $deployment->getSequence(),
|
||||
'deploymentCreatedAt' => $deployment->getCreatedAt(),
|
||||
]));
|
||||
|
||||
$dbForPlatform->forEach('rules', function (Document $rule) use ($dbForPlatform, $deployment) {
|
||||
$dbForPlatform->updateDocument('rules', $rule->getId(), new Document([
|
||||
'deploymentId' => $deployment->getId(),
|
||||
'deploymentInternalId' => $deployment->getSequence(),
|
||||
]));
|
||||
}, [
|
||||
Query::equal('projectInternalId', [$project->getSequence()]),
|
||||
Query::equal('type', ['deployment']),
|
||||
Query::equal('deploymentResourceInternalId', [$function->getSequence()]),
|
||||
Query::equal('deploymentResourceType', ['function']),
|
||||
Query::equal('trigger', ['manual']),
|
||||
Query::equal('deploymentVcsProviderBranch', ['']),
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refresh the function's latestDeployment* fields from its newest
|
||||
* deployment. Mirrors the Builds worker so the console reflects the current
|
||||
* build status.
|
||||
*/
|
||||
private function updateLatestDeployment(Database $dbForProject, Document $function): void
|
||||
{
|
||||
$latest = $dbForProject->findOne('deployments', [
|
||||
Query::equal('resourceType', ['functions']),
|
||||
Query::equal('resourceInternalId', [$function->getSequence()]),
|
||||
Query::orderDesc('$createdAt'),
|
||||
]);
|
||||
|
||||
if ($latest->isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
$dbForProject->updateDocument('functions', $function->getId(), new Document([
|
||||
'latestDeploymentId' => $latest->getId(),
|
||||
'latestDeploymentInternalId' => $latest->getSequence(),
|
||||
'latestDeploymentCreatedAt' => $latest->getCreatedAt(),
|
||||
'latestDeploymentStatus' => $latest->getAttribute('status', ''),
|
||||
]));
|
||||
}
|
||||
|
||||
/**
|
||||
* (Re)activate the function's schedule document so the scheduler enqueues
|
||||
* cron executions. Mirrors the executor Builds worker: a schedule is active
|
||||
* only when the function has both a cron expression and an active
|
||||
* deployment. Re-reads the function so it sees a deploymentId just set by
|
||||
* activate().
|
||||
*/
|
||||
private function schedule(Database $dbForProject, Database $dbForPlatform, Document $function): void
|
||||
{
|
||||
$scheduleId = $function->getAttribute('scheduleId', '');
|
||||
if ($scheduleId === '') {
|
||||
return;
|
||||
}
|
||||
|
||||
$function = $dbForProject->getDocument('functions', $function->getId());
|
||||
$schedule = $dbForPlatform->getDocument('schedules', $scheduleId);
|
||||
if ($schedule->isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
$dbForPlatform->updateDocument('schedules', $schedule->getId(), new Document([
|
||||
'resourceUpdatedAt' => DateTime::now(),
|
||||
'schedule' => $function->getAttribute('schedule', ''),
|
||||
'active' => ! empty($function->getAttribute('schedule')) && ! empty($function->getAttribute('deploymentId')),
|
||||
]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify project webhooks and event-triggered functions of a deployment
|
||||
* update, mirroring the executor Builds worker's fan-out. queueForEvents
|
||||
* only builds the event; realtime is triggered separately in action().
|
||||
*/
|
||||
private function dispatchUpdate(
|
||||
Event $queueForEvents,
|
||||
Webhook $queueForWebhooks,
|
||||
FunctionPublisher $publisherForFunctions,
|
||||
Document $project,
|
||||
Document $deployment,
|
||||
): void {
|
||||
$model = new Deployment();
|
||||
$update = $queueForEvents
|
||||
->setProject($project)
|
||||
->setEvent('functions.[functionId].deployments.[deploymentId].update')
|
||||
->setParam('functionId', $deployment->getAttribute('resourceId'))
|
||||
->setParam('deploymentId', $deployment->getId())
|
||||
->setPayload($deployment->getArrayCopy(\array_keys($model->getRules())));
|
||||
|
||||
$queueForWebhooks->from($update)->trigger();
|
||||
|
||||
$publisherForFunctions->enqueue(FunctionMessage::fromEvent(
|
||||
event: $update->getEvent(),
|
||||
params: $update->getParams(),
|
||||
project: $update->getProject(),
|
||||
user: $update->getUser(),
|
||||
userId: $update->getUserId(),
|
||||
payload: $update->getPayload(),
|
||||
platform: $update->getPlatform(),
|
||||
));
|
||||
}
|
||||
|
||||
private function truncate(string $logs): string
|
||||
{
|
||||
$limit = APP_LOG_LENGTH_LIMIT;
|
||||
if (\strlen($logs) <= $limit) {
|
||||
return $logs;
|
||||
}
|
||||
|
||||
return \substr($logs, -$limit);
|
||||
}
|
||||
}
|
||||
+4
-1
@@ -10,6 +10,7 @@ use Appwrite\SDK\AuthType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Response;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
@@ -61,6 +62,7 @@ class Update extends Action
|
||||
->inject('authorization')
|
||||
->inject('getProjectDB')
|
||||
->inject('publisherForBuilds')
|
||||
->inject('jobs')
|
||||
->inject('platform')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
@@ -76,6 +78,7 @@ class Update extends Action
|
||||
Authorization $authorization,
|
||||
callable $getProjectDB,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
Jobs $jobs,
|
||||
array $platform
|
||||
) {
|
||||
$installation = $dbForPlatform->getDocument('installations', $installationId);
|
||||
@@ -137,7 +140,7 @@ class Update extends Action
|
||||
...array_filter(array_column($prFiles, 'previous_filename'))
|
||||
];
|
||||
|
||||
$this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, true, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform);
|
||||
$this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, true, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs);
|
||||
|
||||
$response->noContent();
|
||||
}
|
||||
|
||||
@@ -2,12 +2,14 @@
|
||||
|
||||
namespace Appwrite\Platform\Modules\VCS\Http\GitHub;
|
||||
|
||||
use Appwrite\Compute\Job;
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Build as BuildMessage;
|
||||
use Appwrite\Event\Publisher\Build as BuildPublisher;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\Filter\BranchDomain as BranchDomainFilter;
|
||||
use Appwrite\Vcs\Comment;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Console;
|
||||
use Utopia\Database\Database;
|
||||
@@ -50,6 +52,7 @@ trait Deployment
|
||||
BuildPublisher $publisherForBuilds,
|
||||
callable $getProjectDB,
|
||||
array $platform,
|
||||
?Jobs $jobs = null,
|
||||
) {
|
||||
$errors = [];
|
||||
foreach ($repositories as $repository) {
|
||||
@@ -360,6 +363,17 @@ trait Deployment
|
||||
$commands[] = $resource->getAttribute('commands', '');
|
||||
}
|
||||
|
||||
// Build function deployments on the jobs-service when the caller
|
||||
// opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator. Sites stay
|
||||
// on the executor. On jobs the deployment is pre-declared 'waiting'
|
||||
// with its buildPath so build.sh writes onto the mounted volume.
|
||||
$useJobs = $jobs !== null
|
||||
&& $resourceCollection === 'functions'
|
||||
&& System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator';
|
||||
$buildFields = $useJobs
|
||||
? ['status' => 'waiting', 'buildPath' => Job::buildPath($projectId, $deploymentId)]
|
||||
: [];
|
||||
|
||||
$deployment = $authorization->skip(fn () => $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
@@ -367,6 +381,7 @@ trait Deployment
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
],
|
||||
...$buildFields,
|
||||
'resourceId' => $resourceId,
|
||||
'resourceInternalId' => $resourceInternalId,
|
||||
'resourceType' => $resourceCollection,
|
||||
@@ -557,18 +572,26 @@ trait Deployment
|
||||
$github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'pending', $message, $providerTargetUrl, $name);
|
||||
}
|
||||
|
||||
$queueName = $this->getBuildQueueName($project, $dbForPlatform, $authorization);
|
||||
if ($useJobs) {
|
||||
$source = [
|
||||
'url' => $github->getRepositoryPresignedUrl($providerRepositoryOwner, $providerRepositoryName, $providerCommitHash),
|
||||
'subdir' => $resource->getAttribute('providerRootDirectory', ''),
|
||||
];
|
||||
$jobs->create(...Job::build($project, $resource, $deployment, $platform, $source));
|
||||
} else {
|
||||
$queueName = $this->getBuildQueueName($project, $dbForPlatform, $authorization);
|
||||
|
||||
$publisherForBuilds->enqueue(
|
||||
new BuildMessage(
|
||||
project: $project,
|
||||
resource: $resource,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
platform: $platform,
|
||||
),
|
||||
new \Utopia\Queue\Queue($queueName)
|
||||
);
|
||||
$publisherForBuilds->enqueue(
|
||||
new BuildMessage(
|
||||
project: $project,
|
||||
resource: $resource,
|
||||
deployment: $deployment,
|
||||
type: BUILD_TYPE_DEPLOYMENT,
|
||||
platform: $platform,
|
||||
),
|
||||
new \Utopia\Queue\Queue($queueName)
|
||||
);
|
||||
}
|
||||
|
||||
Span::add("{$logBase}.build.triggered", 'true');
|
||||
//TODO: Add event?
|
||||
|
||||
@@ -8,6 +8,7 @@ use Appwrite\Platform\Action;
|
||||
use Appwrite\Platform\Modules\VCS\Http\GitHub\Deployment;
|
||||
use Appwrite\Utopia\Request;
|
||||
use Appwrite\Utopia\Response;
|
||||
use OpenRuntimes\Orchestrator\Jobs;
|
||||
use Utopia\Console;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
@@ -43,6 +44,7 @@ class Create extends Action
|
||||
->inject('authorization')
|
||||
->inject('getProjectDB')
|
||||
->inject('publisherForBuilds')
|
||||
->inject('jobs')
|
||||
->inject('platform')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
@@ -55,6 +57,7 @@ class Create extends Action
|
||||
Authorization $authorization,
|
||||
callable $getProjectDB,
|
||||
BuildPublisher $publisherForBuilds,
|
||||
Jobs $jobs,
|
||||
array $platform
|
||||
) {
|
||||
$this->preprocessEvent($request);
|
||||
@@ -79,8 +82,8 @@ class Create extends Action
|
||||
|
||||
match ($event) {
|
||||
$github::EVENT_INSTALLATION => $this->handleInstallationEvent($parsedPayload, $dbForPlatform, $authorization, $getProjectDB),
|
||||
$github::EVENT_PUSH => $this->handlePushEvent($parsedPayload, $githubAppId, $privateKey, $github, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform),
|
||||
$github::EVENT_PULL_REQUEST => $this->handlePullRequestEvent($parsedPayload, $privateKey, $githubAppId, $github, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform),
|
||||
$github::EVENT_PUSH => $this->handlePushEvent($parsedPayload, $githubAppId, $privateKey, $github, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs),
|
||||
$github::EVENT_PULL_REQUEST => $this->handlePullRequestEvent($parsedPayload, $privateKey, $githubAppId, $github, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs),
|
||||
default => null,
|
||||
};
|
||||
|
||||
@@ -187,6 +190,7 @@ class Create extends Action
|
||||
BuildPublisher $publisherForBuilds,
|
||||
callable $getProjectDB,
|
||||
array $platform,
|
||||
?Jobs $jobs = null,
|
||||
) {
|
||||
$providerBranchDeleted = $parsedPayload["branchDeleted"] ?? false;
|
||||
$providerBranch = $parsedPayload["branch"] ?? '';
|
||||
@@ -219,7 +223,7 @@ class Create extends Action
|
||||
// Create new deployment only on push (not committed by us) and not when branch is deleted
|
||||
if ($providerCommitAuthorEmail !== APP_VCS_GITHUB_EMAIL && !$providerBranchDeleted) {
|
||||
$providerAffectedFiles = $parsedPayload['affectedFiles'] ?? [];
|
||||
$this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthorName, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', $providerAffectedFiles, false, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform);
|
||||
$this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthorName, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', $providerAffectedFiles, false, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -233,6 +237,7 @@ class Create extends Action
|
||||
BuildPublisher $publisherForBuilds,
|
||||
callable $getProjectDB,
|
||||
array $platform,
|
||||
?Jobs $jobs = null,
|
||||
) {
|
||||
$action = $parsedPayload["action"] ?? '';
|
||||
|
||||
@@ -283,7 +288,7 @@ class Create extends Action
|
||||
Query::orderDesc('$createdAt')
|
||||
]));
|
||||
|
||||
$this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, $external, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform);
|
||||
$this->createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, $external, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs);
|
||||
} elseif ($action == "closed") {
|
||||
// Allowed external contributions cleanup
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Usage;
|
||||
|
||||
use Appwrite\Event\Message\Usage as UsageMessage;
|
||||
use Appwrite\Event\Publisher\Usage as UsagePublisher;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Database\Document;
|
||||
|
||||
/**
|
||||
* Emits build usage/billing metrics for a completed deployment. Shared by both
|
||||
* build backends — the executor (Builds worker) and the jobs-service (Jobs
|
||||
* worker) — so a build is counted identically however it was produced.
|
||||
*/
|
||||
final class Build
|
||||
{
|
||||
/**
|
||||
* Record the metrics for a terminal deployment (status 'ready' or 'failed')
|
||||
* onto the usage context and publish them.
|
||||
*/
|
||||
public static function publish(
|
||||
Context $usage,
|
||||
Document $resource,
|
||||
Document $deployment,
|
||||
Document $project,
|
||||
UsagePublisher $publisherForUsage,
|
||||
): void {
|
||||
$spec = Config::getParam('specifications')[$resource->getAttribute('buildSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT)];
|
||||
$cpus = (int) ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT);
|
||||
$memory = (int) ($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT);
|
||||
|
||||
switch ($deployment->getAttribute('status')) {
|
||||
case 'ready':
|
||||
$usage
|
||||
->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project
|
||||
->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_SUCCESS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_SUCCESS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000);
|
||||
break;
|
||||
case 'failed':
|
||||
$usage
|
||||
->addMetric(METRIC_BUILDS_FAILED, 1) // per project
|
||||
->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_FAILED), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_FAILED), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000);
|
||||
break;
|
||||
}
|
||||
|
||||
$usage
|
||||
->addMetric(METRIC_BUILDS, 1) // per project
|
||||
->addMetric(METRIC_BUILDS_STORAGE, $deployment->getAttribute('buildSize', 0))
|
||||
->addMetric(METRIC_BUILDS_COMPUTE, (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(METRIC_BUILDS_MB_SECONDS, (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus))
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0))
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus))
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS), 1) // per function
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0))
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000)
|
||||
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus));
|
||||
|
||||
if (! $usage->isEmpty()) {
|
||||
$publisherForUsage->enqueue(new UsageMessage(
|
||||
project: $project,
|
||||
metrics: $usage->getMetrics(),
|
||||
reduce: $usage->getReduce()
|
||||
));
|
||||
$usage->reset();
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# End-to-end test of the jobs-service build slice, driven entirely through the
|
||||
# Appwrite CLI's `push function` — which streams the deployment build logs live,
|
||||
# exactly as an end user sees them.
|
||||
#
|
||||
# Self-contained: bootstraps a throwaway console account + project, logs the CLI
|
||||
# in, then pushes a tiny Node function twice — so the second build reuses the
|
||||
# package-manager build cache the first one saved ("Build cache hit").
|
||||
#
|
||||
# Optional env:
|
||||
# ENDPOINT Appwrite endpoint (default: http://localhost/v1)
|
||||
# FUNC_ID function id (default: testbuild)
|
||||
# COMMANDS build command (default: npm install)
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
ENDPOINT="${ENDPOINT:-http://localhost/v1}"
|
||||
FUNC_ID="${FUNC_ID:-testbuild}"
|
||||
COMMANDS="${COMMANDS:-npm install}"
|
||||
EMAIL="tb-$(date +%s)@example.com"
|
||||
PASSWORD="password123"
|
||||
|
||||
id() { python3 -c 'import sys,json;print(json.load(sys.stdin).get("$id",""))'; }
|
||||
console() { curl -s -c "$JAR" -b "$JAR" -H "X-Appwrite-Project: console" -H "Content-Type: application/json" "$@"; }
|
||||
|
||||
# --- Bootstrap a project owned by a fresh console account (REST) -------------
|
||||
JAR="$(mktemp)"
|
||||
console -X POST "$ENDPOINT/account" -d "{\"userId\":\"unique()\",\"email\":\"$EMAIL\",\"password\":\"$PASSWORD\",\"name\":\"Test\"}" >/dev/null
|
||||
console -X POST "$ENDPOINT/account/sessions/email" -d "{\"email\":\"$EMAIL\",\"password\":\"$PASSWORD\"}" >/dev/null
|
||||
TEAM="$(console -X POST "$ENDPOINT/teams" -d '{"teamId":"unique()","name":"t"}' | id)"
|
||||
PROJECT_ID="$(console -X POST "$ENDPOINT/projects" -d "{\"projectId\":\"unique()\",\"name\":\"p\",\"teamId\":\"$TEAM\",\"region\":\"default\"}" | id)"
|
||||
|
||||
# --- Log the CLI in (push uses a session, not an API key) --------------------
|
||||
appwrite login --email "$EMAIL" --password "$PASSWORD" --endpoint "$ENDPOINT" >/dev/null
|
||||
|
||||
# --- Generate a minimal Appwrite project to push -----------------------------
|
||||
WORK="$(mktemp -d)"
|
||||
mkdir -p "$WORK/functions/$FUNC_ID"
|
||||
printf 'module.exports = async ({ res }) => res.json({ ok: true });\n' > "$WORK/functions/$FUNC_ID/main.js"
|
||||
# A real dependency so `npm install` populates a cache worth reusing.
|
||||
printf '{ "name": "fn", "version": "1.0.0", "main": "main.js", "dependencies": { "ms": "2.1.3" } }\n' > "$WORK/functions/$FUNC_ID/package.json"
|
||||
cat > "$WORK/appwrite.json" <<EOF
|
||||
{
|
||||
"projectId": "$PROJECT_ID",
|
||||
"functions": [
|
||||
{
|
||||
"\$id": "$FUNC_ID",
|
||||
"name": "$FUNC_ID",
|
||||
"runtime": "node-22",
|
||||
"execute": ["any"],
|
||||
"entrypoint": "main.js",
|
||||
"commands": "$COMMANDS",
|
||||
"path": "functions/$FUNC_ID"
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
# --- Push + stream the build (real user experience) --------------------------
|
||||
# Push twice against the same function: the first build seeds the build cache,
|
||||
# the second reuses it (watch for "Build cache hit" in the second run's logs).
|
||||
cd "$WORK"
|
||||
echo "Project $PROJECT_ID — first build (cold cache), logs stream below:"
|
||||
echo
|
||||
appwrite push function --function-id "$FUNC_ID" --activate --force
|
||||
|
||||
echo
|
||||
echo "Second build (should hit the build cache seeded above):"
|
||||
echo
|
||||
appwrite push function --function-id "$FUNC_ID" --activate --force
|
||||
@@ -1086,19 +1086,15 @@ final class RealtimeConsoleClientTest extends Scope
|
||||
$this->assertEquals("waiting", $response['data']['payload']['status']);
|
||||
$this->assertContains("functions.{$functionId}.deployments.{$deploymentId}.create", $response['data']['events']);
|
||||
|
||||
$response = json_decode($client->receive(), true);
|
||||
$this->assertContains("functions.{$functionId}.deployments.{$deploymentId}.update", $response['data']['events']);
|
||||
$this->assertContains('console', $response['data']['channels']);
|
||||
$this->assertContains("projects.{$projectId}", $response['data']['channels']);
|
||||
$this->assertEquals("processing", $response['data']['payload']['status']);
|
||||
|
||||
$response = json_decode($client->receive(), true);
|
||||
$this->assertContains("functions.{$functionId}.deployments.{$deploymentId}.update", $response['data']['events']);
|
||||
$this->assertContains('console', $response['data']['channels']);
|
||||
$this->assertContains("projects.{$projectId}", $response['data']['channels']);
|
||||
$this->assertEquals("building", $response['data']['payload']['status']);
|
||||
|
||||
// Consume the build lifecycle updates. The status model is
|
||||
// waiting → building → ready/failed (the jobs backend has no intermediate
|
||||
// "processing" phase and may interleave extra update events), so assert
|
||||
// the meaningful milestones tolerantly rather than a fixed positional
|
||||
// sequence: the build reaches "building" with a growing log, then a
|
||||
// terminal "ready" carrying the build metadata.
|
||||
$sawBuilding = false;
|
||||
$previousBuildLogs = null;
|
||||
$payload = null;
|
||||
while (true) {
|
||||
$response = json_decode($client->receive(), true);
|
||||
$this->assertContains("functions.{$functionId}.deployments.{$deploymentId}.update", $response['data']['events']);
|
||||
@@ -1106,37 +1102,28 @@ final class RealtimeConsoleClientTest extends Scope
|
||||
$this->assertContains("projects.{$projectId}", $response['data']['channels']);
|
||||
$this->assertArrayHasKey('buildLogs', $response['data']['payload']);
|
||||
|
||||
if (!empty($response['data']['payload']['buildSize'])) {
|
||||
$this->assertNotEmpty($response['data']['payload']['buildStartedAt']);
|
||||
$this->assertNotEmpty($response['data']['payload']['buildPath']);
|
||||
$this->assertNotEmpty($response['data']['payload']['buildSize']);
|
||||
$this->assertNotEmpty($response['data']['payload']['totalSize']);
|
||||
$this->assertNotEmpty($response['data']['payload']['buildLogs']);
|
||||
$payload = $response['data']['payload'];
|
||||
$this->assertNotEquals('failed', $payload['status']);
|
||||
|
||||
if ($payload['status'] === 'building') {
|
||||
$sawBuilding = true;
|
||||
if ($previousBuildLogs !== null) {
|
||||
$this->assertNotEquals($previousBuildLogs, $payload['buildLogs']);
|
||||
}
|
||||
$previousBuildLogs = $payload['buildLogs'];
|
||||
}
|
||||
|
||||
if ($payload['status'] === 'ready' && !empty($payload['buildDuration']) && !empty($payload['buildEndedAt'])) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Ignore comparison for first payload
|
||||
if ($previousBuildLogs !== null) {
|
||||
$this->assertNotEquals($previousBuildLogs, $response['data']['payload']['buildLogs']);
|
||||
}
|
||||
|
||||
$previousBuildLogs = $response['data']['payload']['buildLogs'];
|
||||
|
||||
$this->assertEquals('building', $response['data']['payload']['status']);
|
||||
}
|
||||
|
||||
$response = json_decode($client->receive(), true);
|
||||
$this->assertContains("functions.{$functionId}.deployments.{$deploymentId}.update", $response['data']['events']);
|
||||
$this->assertContains('console', $response['data']['channels']);
|
||||
$this->assertContains("projects.{$projectId}", $response['data']['channels']);
|
||||
$this->assertEquals("ready", $response['data']['payload']['status']);
|
||||
|
||||
$response = json_decode($client->receive(), true);
|
||||
$this->assertContains("functions.{$functionId}.deployments.{$deploymentId}.update", $response['data']['events']);
|
||||
$this->assertContains('console', $response['data']['channels']);
|
||||
$this->assertContains("projects.{$projectId}", $response['data']['channels']);
|
||||
$this->assertNotEmpty($response['data']['payload']['buildDuration']);
|
||||
$this->assertNotEmpty($response['data']['payload']['buildEndedAt']);
|
||||
$this->assertTrue($sawBuilding);
|
||||
$this->assertNotEmpty($payload['buildStartedAt']);
|
||||
$this->assertNotEmpty($payload['buildPath']);
|
||||
$this->assertNotEmpty($payload['buildSize']);
|
||||
$this->assertNotEmpty($payload['totalSize']);
|
||||
$this->assertNotEmpty($payload['buildLogs']);
|
||||
|
||||
$client->close();
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user