diff --git a/.claude/skills/patch-release-checklist/SKILL.md b/.claude/skills/patch-release-checklist/SKILL.md index d052640f23..000fcd065a 100644 --- a/.claude/skills/patch-release-checklist/SKILL.md +++ b/.claude/skills/patch-release-checklist/SKILL.md @@ -6,8 +6,9 @@ When bumping a patch version (e.g., `1.9.0` -> `1.9.1`), follow this checklist. ### Bump console image -Update the console Docker image tag: -- [ ] `docker-compose.yml` -- update `image: appwrite/vibes:X.Y.Z` (the `appwrite-console` service) +Update the console Docker image tag in both files: +- [ ] `docker-compose.yml` -- update `image: appwrite/console:X.Y.Z` +- [ ] `app/views/install/compose.phtml` -- update `image: /console:X.Y.Z` ### Bump Appwrite version diff --git a/.claude/skills/pr-workflow/SKILL.md b/.claude/skills/pr-workflow/SKILL.md deleted file mode 100644 index f2e04477e5..0000000000 --- a/.claude/skills/pr-workflow/SKILL.md +++ /dev/null @@ -1,31 +0,0 @@ -# PR Workflow for Appwrite - -## Branch Targeting - -**Important:** Appwrite does NOT use `main` as the primary development branch. - -PRs should target the **current version branch** (e.g., `1.8.x`, `1.9.x`), not `main`. - -### How to determine the correct target branch - -1. Check which version branch you're currently on: `git branch --show-current` -2. Look for branches matching the pattern `X.Y.x` (e.g., `1.8.x`, `1.9.x`) -3. The current active development branch is typically the highest version number with the `.x` suffix - -### When creating PRs - -Always use the version branch as the base: - -```bash -# Correct - targets the version branch -gh pr create --base 1.8.x --title "Your PR title" --body "..." - -# Wrong - do not target main -gh pr create --base main ... -``` - -### Branch naming convention - -- `X.Y.x` branches (e.g., `1.8.x`) - Active development branches for each minor version -- `main` - Not used for regular PRs; reserved for release management -- Feature branches should be created from and merged back into the current version branch diff --git a/.env b/.env index ed524c5c25..eed895eb1e 100644 --- a/.env +++ b/.env @@ -36,9 +36,9 @@ _APP_CONSOLE_COUNTRIES_DENYLIST=AQ _APP_CONSOLE_WHITELIST_EMAILS= _APP_CONSOLE_WHITELIST_IPS= _APP_CONSOLE_WHITELIST_ROOT=disabled -_APP_DB_ADAPTER=mongodb -_APP_DB_HOST=mongodb -_APP_DB_PORT=27017 +_APP_DB_ADAPTER=postgresql +_APP_DB_HOST=postgresql +_APP_DB_PORT=5432 _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 793b4b2144..2021359541 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -252,7 +252,7 @@ jobs: return; } - core.setOutput('databases', JSON.stringify(['MongoDB'])); + core.setOutput('databases', JSON.stringify(['PostgreSQL'])); core.setOutput('modes', JSON.stringify(['dedicated'])); build: diff --git a/.gitignore b/.gitignore index 99ad534f93..6846e19aa7 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,3 @@ docker-compose.web-installer.yml .env.web-installer docker-compose.web-installer.yml.**.backup tests/playwright/screenshots -.playwright-mcp/ diff --git a/app/config/errors.php b/app/config/errors.php index cff9a6367c..cb40810077 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -347,6 +347,11 @@ return [ 'description' => 'OAuth2 provider returned some error.', 'code' => 424, ], + Exception::USER_OAUTH2_PROVIDER_FAILURE => [ + 'name' => Exception::USER_OAUTH2_PROVIDER_FAILURE, + 'description' => '%s couldn\'t complete sign-in (%s). Please try again.', + 'code' => 424, + ], Exception::USER_EMAIL_NOT_VERIFIED => [ 'name' => Exception::USER_EMAIL_NOT_VERIFIED, 'description' => 'User email is not verified', diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index ed9a8046e6..ac2eabdece 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -76,8 +76,8 @@ use Utopia\Validator\Range; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; -$oauthDefaultSuccess = '/auth/oauth2/success'; -$oauthDefaultFailure = '/auth/oauth2/failure'; +$oauthDefaultSuccess = '/console/auth/oauth2/success'; +$oauthDefaultFailure = '/console/auth/oauth2/failure'; $createSession = function (string $userId, string $secret, Request $request, Response $response, User $user, Database $dbForProject, Document $project, array $platform, Locale $locale, GeoRecord $geoRecord, Event $queueForEvents, Bus $bus, Store $store, ProofsToken $proofForToken, ProofsCode $proofForCode, bool $domainVerification, ?string $cookieDomain, Authorization $authorization) { @@ -1570,8 +1570,8 @@ Http::get('/v1/account/sessions/oauth2/:provider/redirect') $failure = URLParser::parse($state['failure']); } - $failureRedirect = (function (string $type, ?string $message = null, ?int $code = null) use ($failure, $response) { - $exception = new Exception($type, $message, $code); + $failureRedirect = (function (string $type, ?string $message = null, ?int $code = null, ?\Throwable $previous = null, array $params = []) use ($failure, $response) { + $exception = new Exception($type, $message, $code, $previous, params: $params); if (!empty($failure)) { $query = URLParser::parseQuery($failure['query']); $query['error'] = json_encode([ @@ -1617,11 +1617,12 @@ Http::get('/v1/account/sessions/oauth2/:provider/redirect') $accessTokenExpiry = $oauth2->getAccessTokenExpiry($code); } catch (OAuth2Exception $ex) { + $providerError = $ex->getError() ?: $ex->getMessage(); $failureRedirect( - $ex->getType(), - 'Failed to obtain access token. The ' . $providerName . ' OAuth2 provider returned an error: ' . $ex->getMessage(), - $ex->getCode(), + Exception::USER_OAUTH2_PROVIDER_FAILURE, + previous: $ex, + params: [$providerName, $providerError], ); } @@ -2395,7 +2396,7 @@ Http::post('/v1/account/tokens/magic-url') } elseif ($protocol === 'http' && $port !== '80') { $callbackBase .= ':' . $port; } - $url = $callbackBase . '/auth/magic-url'; + $url = $callbackBase . '/console/auth/magic-url'; } $url = Template::parseURL($url); diff --git a/app/controllers/general.php b/app/controllers/general.php index 498d2de3c1..557ef9ac3b 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -125,7 +125,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S if (\str_ends_with($host, $denyDomain)) { $exception = new AppwriteException(AppwriteException::RULE_NOT_FOUND, 'This domain is not connected to any Appwrite resources. Visit domains tab under function/site settings to configure it.', view: $errorView); - $exception->addCTA('Start with this domain', $url); + $exception->addCTA('Start with this domain', $url . '/console'); throw $exception; } } @@ -200,7 +200,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S $resourceId = $rule->getAttribute('deploymentResourceId', ''); $type = ($resourceType === 'site') ? 'sites' : 'functions'; $exception = new AppwriteException(AppwriteException::DEPLOYMENT_NOT_FOUND, view: $errorView); - $exception->addCTA('View deployments', $url . '/projects/' . $projectId . '/' . $type . '/' . $resourceId); + $exception->addCTA('View deployments', $url . '/console/project-' . $project->getAttribute('region', 'default') . '-' . $projectId . '/' . $type . '/' . $resourceType . '-' . $resourceId); throw $exception; } @@ -289,7 +289,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S $response ->addHeader('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') ->addHeader('Pragma', 'no-cache') - ->redirect($url . '/auth/preview?' + ->redirect($url . '/console/auth/preview?' . \http_build_query([ 'projectId' => $projectId, 'origin' => $protocol . '://' . $host, @@ -346,21 +346,22 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S $allowAnyStatus = !\is_null($apiKey) && $apiKey->isDeploymentStatusIgnored(); if (!$allowAnyStatus && $deployment->getAttribute('status') !== 'ready') { $status = $deployment->getAttribute('status'); + $region = $project->getAttribute('region', 'default'); switch ($status) { case 'failed': $exception = new AppwriteException(AppwriteException::BUILD_FAILED, view: $errorView); - $ctaUrl = '/projects/' . $project->getId() . '/sites/' . $resource->getId() . '/deployments/' . $deployment->getId(); + $ctaUrl = '/console/project-' . $region . '-' . $project->getId() . '/sites/site-' . $resource->getId() . '/deployments/deployment-' . $deployment->getId(); $exception->addCTA('View logs', $url . $ctaUrl); break; case 'canceled': $exception = new AppwriteException(AppwriteException::BUILD_CANCELED, view: $errorView); - $ctaUrl = '/projects/' . $project->getId() . '/sites/' . $resource->getId() . '/deployments'; + $ctaUrl = '/console/project-' . $region . '-' . $project->getId() . '/sites/site-' . $resource->getId() . '/deployments'; $exception->addCTA('View deployments', $url . $ctaUrl); break; default: $exception = new AppwriteException(AppwriteException::BUILD_NOT_READY, view: $errorView); - $ctaUrl = '/projects/' . $project->getId() . '/sites/' . $resource->getId() . '/deployments/' . $deployment->getId(); + $ctaUrl = '/console/project-' . $region . '-' . $project->getId() . '/sites/site-' . $resource->getId() . '/deployments/deployment-' . $deployment->getId(); $exception->addCTA('Reload', '/'); $exception->addCTA('View logs', $url . $ctaUrl); break; @@ -372,7 +373,7 @@ function router(Http $utopia, Database $dbForPlatform, callable $getProjectDB, S $permissions = $resource->getAttribute('execute'); if (!(\in_array('any', $permissions)) && !(\in_array('guests', $permissions))) { $exception = new AppwriteException(AppwriteException::FUNCTION_EXECUTE_PERMISSION_MISSING, view: $errorView); - $exception->addCTA('View settings', $url . '/projects/' . $project->getId() . '/functions/' . $resource->getId() . '/settings'); + $exception->addCTA('View settings', $url . '/console/project-' . $project->getAttribute('region', 'default') . '-' . $project->getId() . '/functions/function-' . $resource->getId() . '/settings'); throw $exception; } } diff --git a/app/init/registers.php b/app/init/registers.php index d188a0b614..633a9b0d12 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -151,9 +151,9 @@ $register->set('pools', function () { $group = new Group(); $fallbackForDB = 'db_main=' . AppwriteURL::unparse([ - 'scheme' => System::getEnv('_APP_DB_ADAPTER', 'mongodb'), - 'host' => System::getEnv('_APP_DB_HOST', 'mongodb'), - 'port' => System::getEnv('_APP_DB_PORT', '27017'), + 'scheme' => System::getEnv('_APP_DB_ADAPTER', 'postgresql'), + 'host' => System::getEnv('_APP_DB_HOST', 'postgresql'), + 'port' => System::getEnv('_APP_DB_PORT', '5432'), 'user' => System::getEnv('_APP_DB_USER', ''), 'pass' => System::getEnv('_APP_DB_PASS', ''), 'path' => System::getEnv('_APP_DB_SCHEMA', ''), @@ -408,7 +408,7 @@ $register->set('db', function () { $dbUser = System::getEnv('_APP_DB_USER', ''); $dbPass = System::getEnv('_APP_DB_PASS', ''); $dbSchema = System::getEnv('_APP_DB_SCHEMA', ''); - $dbAdapter = System::getEnv('_APP_DB_ADAPTER', 'mongodb'); + $dbAdapter = System::getEnv('_APP_DB_ADAPTER', 'postgresql'); $dsn = ''; switch ($dbAdapter) { diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 54309c2430..580fa45602 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -5,9 +5,12 @@ use Ahc\Jwt\JWTException; use Appwrite\Auth\Key; use Appwrite\Database\Factory as DatabaseFactory; use Appwrite\Databases\TransactionState; +use Appwrite\Deployment\Backend\Executor as ExecutorBackend; +use Appwrite\Deployment\Backend\Orchestrator; use Appwrite\Event\Context\Audit as AuditContext; use Appwrite\Event\Event; use Appwrite\Event\Message\Func as FunctionMessage; +use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Event\Publisher\Func as FunctionPublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Webhook; @@ -24,6 +27,8 @@ use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; +use Executor\Executor; +use OpenRuntimes\Orchestrator\Jobs; use Utopia\Agents\Adapters\Appwrite as AppwriteAdapter; use Utopia\Agents\Agent; use Utopia\Audit\Adapter\Database as AdapterDatabase; @@ -191,6 +196,18 @@ return function (Container $context): void { $publisher, new Queue(System::getEnv('_APP_FUNCTIONS_QUEUE_NAME', Event::FUNCTIONS_QUEUE_NAME), 'utopia-queue', Event::FUNCTIONS_QUEUE_TTL) ), ['publisher']); + $context->set('deployments', function (BuildPublisher $publisherForBuilds, Jobs $jobs, Database $dbForProject, Document $project, Executor $executor, array $platform, Request $request) { + // The jobs-service orchestrator backend only understands functions + // (its payload builder reads function-shaped attributes like + // `runtime`); sites always build on the executor. Transitional: + // once sites build on the jobs-service too, this URI check and the + // nullable Backend params in the VCS trait can both go. + $isSite = \str_starts_with($request->getURI(), '/v1/sites'); + + return !$isSite && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator' + ? new Orchestrator($jobs, $dbForProject, $project, $platform) + : new ExecutorBackend($publisherForBuilds, $dbForProject, $project, $executor, $platform); + }, ['publisherForBuilds', 'jobs', 'dbForProject', 'project', 'executor', 'platform', 'request']); $context->set('eventProcessor', fn () => new EventProcessor(), []); $context->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory( $pools, diff --git a/app/init/worker/message.php b/app/init/worker/message.php index a5838e9257..bf2d2f6cb1 100644 --- a/app/init/worker/message.php +++ b/app/init/worker/message.php @@ -1,12 +1,14 @@ getId())); }, ['project', 'telemetry']); + // Workers only ever build functions on the jobs-service backend (sites + // always build on the executor via the Builds worker directly), so + // unlike the HTTP-side 'deployments' resource this doesn't need to pick + // a backend per-request. + $container->set('deployments', function (Jobs $jobs, Database $dbForProject, Document $project, array $platform) { + return new Orchestrator($jobs, $dbForProject, $project, $platform); + }, ['jobs', 'dbForProject', 'project', 'platform']); + $container->set('logError', function (Registry $register, Document $project) { return function (Throwable $error, string $namespace, string $action, ?array $extras = null) use ($register, $project) { $logger = $register->get('logger'); diff --git a/composer.json b/composer.json index 9e78d91c65..daa1ab4483 100644 --- a/composer.json +++ b/composer.json @@ -106,7 +106,7 @@ "league/csv": "9.14.*", "enshrined/svg-sanitize": "0.22.*", "utopia-php/client": "^0.2.2", - "open-runtimes/sdk-for-php": "^0.2.0" + "open-runtimes/sdk-for-php": "^0.4.0" }, "require-dev": { "ext-fileinfo": "*", diff --git a/composer.lock b/composer.lock index 5492e2482d..5436edf6c3 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "71622f20d2344f54090357f8275af661", + "content-hash": "aaa69508ad474daafe2261e01a65f27c", "packages": [ { "name": "adhocore/jwt", @@ -1147,16 +1147,16 @@ }, { "name": "open-runtimes/sdk-for-php", - "version": "0.2.0", + "version": "0.4.0", "source": { "type": "git", "url": "https://github.com/open-runtimes/sdk-for-php.git", - "reference": "e5cf7852630b69b419f8705482d8ff663ab7ab42" + "reference": "02c01ec6c807f43602d93534252e96c9451ecd80" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/open-runtimes/sdk-for-php/zipball/e5cf7852630b69b419f8705482d8ff663ab7ab42", - "reference": "e5cf7852630b69b419f8705482d8ff663ab7ab42", + "url": "https://api.github.com/repos/open-runtimes/sdk-for-php/zipball/02c01ec6c807f43602d93534252e96c9451ecd80", + "reference": "02c01ec6c807f43602d93534252e96c9451ecd80", "shasum": "" }, "require": { @@ -1184,9 +1184,9 @@ "description": "PHP SDK for the Open Runtimes orchestrator Jobs API.", "support": { "issues": "https://github.com/open-runtimes/sdk-for-php/issues", - "source": "https://github.com/open-runtimes/sdk-for-php/tree/0.2.0" + "source": "https://github.com/open-runtimes/sdk-for-php/tree/0.4.0" }, - "time": "2026-07-08T11:28:50+00:00" + "time": "2026-07-13T21:55:56+00:00" }, { "name": "open-telemetry/api", @@ -4437,16 +4437,16 @@ }, { "name": "utopia-php/migration", - "version": "1.17.2", + "version": "1.17.3", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "80f7f49e79ad57a0b5200b043e2b1bfeade8a190" + "reference": "daf1008800633416a186c18069e3a5784de88d1c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/80f7f49e79ad57a0b5200b043e2b1bfeade8a190", - "reference": "80f7f49e79ad57a0b5200b043e2b1bfeade8a190", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/daf1008800633416a186c18069e3a5784de88d1c", + "reference": "daf1008800633416a186c18069e3a5784de88d1c", "shasum": "" }, "require": { @@ -4486,22 +4486,22 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/1.17.2" + "source": "https://github.com/utopia-php/migration/tree/1.17.3" }, - "time": "2026-07-09T19:53:43+00:00" + "time": "2026-07-14T14:04:00+00:00" }, { "name": "utopia-php/mongo", - "version": "1.3.0", + "version": "1.3.1", "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "44eeb4db9bfcfddccd6e757668a519d9541e65c9" + "reference": "f6ef5f1cecf80fb0378eda13250d6c14096d7620" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/44eeb4db9bfcfddccd6e757668a519d9541e65c9", - "reference": "44eeb4db9bfcfddccd6e757668a519d9541e65c9", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/f6ef5f1cecf80fb0378eda13250d6c14096d7620", + "reference": "f6ef5f1cecf80fb0378eda13250d6c14096d7620", "shasum": "" }, "require": { @@ -4547,9 +4547,9 @@ ], "support": { "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/1.3.0" + "source": "https://github.com/utopia-php/mongo/tree/1.3.1" }, - "time": "2026-07-08T04:09:01+00:00" + "time": "2026-07-14T15:22:52+00:00" }, { "name": "utopia-php/platform", diff --git a/docker-compose.yml b/docker-compose.yml index ae7b889dcf..37677abd2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,7 +74,7 @@ services: - appwrite-builds:/storage/builds:rw depends_on: - redis - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} - appwrite-embedding - appwrite-geo environment: @@ -224,22 +224,20 @@ services: max-file: "5" max-size: 10m container_name: appwrite-console - image: appwrite/vibes:0.3.16 + image: appwrite/console:8.7.11 restart: unless-stopped networks: - appwrite - environment: - - VITE_CONSOLE_PROFILE=self-hosted labels: - traefik.enable=true - traefik.constraint-label-stack=appwrite - traefik.docker.network=appwrite - - traefik.http.services.appwrite_console.loadbalancer.server.port=3000 + - traefik.http.services.appwrite_console.loadbalancer.server.port=80 - traefik.http.routers.appwrite_console_http.entrypoints=appwrite_web - - traefik.http.routers.appwrite_console_http.rule=(Host(`${_APP_DOMAIN:-localhost}`) || Host(`${_APP_CONSOLE_DOMAIN:-localhost}`)) && !PathPrefix(`/v1`) && !PathPrefix(`/.well-known`) && !HeaderRegexp(`x-appwrite-hostname`, `.+`) + - traefik.http.routers.appwrite_console_http.rule=PathPrefix(`/console`) - traefik.http.routers.appwrite_console_http.service=appwrite_console - traefik.http.routers.appwrite_console_https.entrypoints=appwrite_websecure - - traefik.http.routers.appwrite_console_https.rule=(Host(`${_APP_DOMAIN:-localhost}`) || Host(`${_APP_CONSOLE_DOMAIN:-localhost}`)) && !PathPrefix(`/v1`) && !PathPrefix(`/.well-known`) && !HeaderRegexp(`x-appwrite-hostname`, `.+`) + - traefik.http.routers.appwrite_console_https.rule=PathPrefix(`/console`) - traefik.http.routers.appwrite_console_https.service=appwrite_console - traefik.http.routers.appwrite_console_https.tls=true appwrite-realtime: @@ -267,7 +265,7 @@ services: networks: - appwrite depends_on: - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} - redis - appwrite-embedding environment: @@ -311,7 +309,7 @@ services: - appwrite depends_on: - redis - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} - appwrite-embedding environment: - _APP_ENV @@ -347,7 +345,7 @@ services: - appwrite depends_on: - redis - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} - appwrite-embedding volumes: - appwrite-uploads:/storage/uploads:rw @@ -415,7 +413,7 @@ services: - appwrite depends_on: - redis - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} - appwrite-embedding environment: - _APP_ENV @@ -460,7 +458,7 @@ services: - appwrite-builds:/storage/builds:rw depends_on: - redis - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} - appwrite-embedding environment: - _APP_ENV @@ -550,7 +548,7 @@ services: - appwrite-builds:/storage/builds:rw depends_on: - redis - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} environment: - _APP_ENV - _APP_WORKERS_NUM=1 @@ -590,7 +588,7 @@ services: - appwrite-uploads:/storage/uploads:rw depends_on: - redis - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} environment: - _APP_BROWSER_HOST - _APP_WORKER_SCREENSHOTS_ROUTER @@ -650,7 +648,7 @@ services: - appwrite depends_on: - redis - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} - appwrite-embedding volumes: - appwrite-config:/storage/config:rw @@ -696,7 +694,7 @@ services: networks: - appwrite depends_on: - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} - redis environment: - _APP_ENV @@ -891,7 +889,7 @@ services: - appwrite-imports:/storage/imports:rw - appwrite-uploads:/storage/uploads:rw depends_on: - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} environment: - _APP_ENV - _APP_WORKERS_NUM=1 @@ -935,7 +933,7 @@ services: networks: - appwrite depends_on: - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} - redis environment: - _APP_ENV @@ -982,7 +980,7 @@ services: networks: - appwrite depends_on: - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} - redis environment: - _APP_ENV @@ -1024,7 +1022,7 @@ services: networks: - appwrite depends_on: - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} - redis - appwrite-embedding environment: @@ -1063,7 +1061,7 @@ services: networks: - appwrite depends_on: - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} - redis - appwrite-embedding environment: @@ -1102,7 +1100,7 @@ services: networks: - appwrite depends_on: - - ${_APP_DB_HOST:-mongodb} + - ${_APP_DB_HOST:-postgresql} - redis - appwrite-embedding environment: @@ -1228,7 +1226,7 @@ services: options: max-file: "5" max-size: 10m - image: ghcr.io/open-runtimes/orchestrator/jobs-service:0.4.3 + image: ghcr.io/open-runtimes/orchestrator/jobs-service:0.6.0 restart: unless-stopped user: root networks: @@ -1239,7 +1237,7 @@ services: - ORCHESTRATOR_BACKEND=docker - ORCHESTRATOR_NETWORK=appwrite - ARTIFACT_ENDPOINT=http://orchestrator-jobs:8080 - - JOB_SIDECAR_IMAGE=ghcr.io/open-runtimes/orchestrator/job-sidecar:0.4.3 + - JOB_SIDECAR_IMAGE=ghcr.io/open-runtimes/orchestrator/job-sidecar:0.6.0 mariadb: image: mariadb:10.11 diff --git a/src/Appwrite/Auth/OAuth2/Github.php b/src/Appwrite/Auth/OAuth2/Github.php index d5d3b07918..f80c067daa 100644 --- a/src/Appwrite/Auth/OAuth2/Github.php +++ b/src/Appwrite/Auth/OAuth2/Github.php @@ -56,7 +56,7 @@ class Github extends OAuth2 $response = $this->request( 'POST', 'https://github.com/login/oauth/access_token', - [], + ['Accept: application/json'], \http_build_query([ 'client_id' => $this->appID, 'redirect_uri' => $this->callback, @@ -65,9 +65,7 @@ class Github extends OAuth2 ]) ); - $output = []; - \parse_str($response, $output); - $this->tokens = $output; + $this->tokens = $this->parseTokens($response); } return $this->tokens; @@ -83,7 +81,7 @@ class Github extends OAuth2 $response = $this->request( 'POST', 'https://github.com/login/oauth/access_token', - [], + ['Accept: application/json'], \http_build_query([ 'client_id' => $this->appID, 'client_secret' => $this->appSecret, @@ -92,9 +90,7 @@ class Github extends OAuth2 ]) ); - $output = []; - \parse_str($response, $output); - $this->tokens = $output; + $this->tokens = $this->parseTokens($response); if (empty($this->tokens['refresh_token'])) { $this->tokens['refresh_token'] = $refreshToken; @@ -103,6 +99,35 @@ class Github extends OAuth2 return $this->tokens; } + /** + * @return array + */ + private function parseTokens(string $response): array + { + $tokens = \json_decode($response, true); + + if (!\is_array($tokens)) { + $tokens = []; + \parse_str($response, $tokens); + } + + if (isset($tokens['error'])) { + throw new Exception(\json_encode( + $tokens, + JSON_INVALID_UTF8_SUBSTITUTE | JSON_THROW_ON_ERROR, + ), 400); + } + + if (empty($tokens['access_token'])) { + throw new Exception(\json_encode([ + 'error' => 'access_token_missing', + 'error_description' => 'GitHub did not return an access token.', + ]), 400); + } + + return $tokens; + } + /** * @param string $accessToken * diff --git a/src/Appwrite/Deployment/Backend.php b/src/Appwrite/Deployment/Backend.php new file mode 100644 index 0000000000..c2f03faa2e --- /dev/null +++ b/src/Appwrite/Deployment/Backend.php @@ -0,0 +1,143 @@ +getSequence() === null) { + return $this->dbForProject->createDocument('deployments', new Document([ + '$permissions' => self::permissions(), + ...self::resourceFields($resource), + ...$deployment->getArrayCopy(), + ])); + } + + return $this->dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + } + + /** + * Finalizes the deployment document (see upload() for what `$deployment` + * should carry) and dispatches it for building from its own uploaded + * source. Deactivates any other active deployment for $resource, marks + * it queued (and writes whatever else the backend needs, e.g. + * buildPath). Returns the persisted, updated deployment. + */ + abstract public function createFromUpload(Document $resource, Document $deployment): Document; + + /** + * Same as createFromUpload(), but builds from a public git reference + * (a template's repository) instead of the deployment's own uploaded + * source. $reference is already resolved to a concrete commit/branch/tag + * — resolving a version range (e.g. "0.3.*") is the caller's job, since + * only it holds the GitHub client that can do so. + */ + abstract public function createFromRef( + Document $resource, + Document $deployment, + string $owner, + string $repository, + string $type, + string $reference, + string $rootDirectory = '', + ): Document; + + /** + * Same as createFromUpload(), but builds from a remote tarball at $url + * (a VCS presigned URL) instead of the deployment's own uploaded source. + */ + abstract public function createFromUrl( + Document $resource, + Document $deployment, + string $url, + string $rootDirectory = '', + ): Document; + + /** + * Best-effort cancel of an in-flight build. The deployment is already + * marked canceled by the caller; this only needs to stop the backend from + * still writing to it. + */ + abstract public function cancel(string $deploymentId): void; + + /** + * Deactivates any other active deployment for $resource before this one + * goes live. Shared by both backends, called right before they mark the + * deployment queued. + */ + protected function deactivateOthers(Document $resource, Document $deployment): void + { + if (!$deployment->getAttribute('activate', false)) { + return; + } + + $others = $this->dbForProject->find('deployments', [ + Query::equal('activate', [true]), + Query::equal('resourceId', [$resource->getId()]), + Query::equal('resourceType', [$resource->getCollection()]), + Query::notEqual('$id', $deployment->getId()), + ]); + + foreach ($others as $other) { + $this->dbForProject->updateDocument('deployments', $other->getId(), new Document([ + 'activate' => false, + ])); + } + } + + /** + * @return array + */ + private static function permissions(): array + { + return [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ]; + } + + /** + * @return array + */ + private static function resourceFields(Document $resource): array + { + return [ + 'resourceInternalId' => $resource->getSequence(), + 'resourceId' => $resource->getId(), + 'resourceType' => $resource->getCollection(), + ]; + } +} diff --git a/src/Appwrite/Deployment/Backend/Executor.php b/src/Appwrite/Deployment/Backend/Executor.php new file mode 100644 index 0000000000..1039c24272 --- /dev/null +++ b/src/Appwrite/Deployment/Backend/Executor.php @@ -0,0 +1,86 @@ +dispatch($resource, $deployment, null); + } + + public function createFromRef( + Document $resource, + Document $deployment, + string $owner, + string $repository, + string $type, + string $reference, + string $rootDirectory = '', + ): Document { + // No presigned URL to fetch here — the Builds worker clones the + // public repo itself via these raw git coordinates. + return $this->dispatch($resource, $deployment, new Document([ + 'ownerName' => $owner, + 'repositoryName' => $repository, + 'referenceType' => $type, + 'referenceValue' => $reference, + 'rootDirectory' => $rootDirectory, + ])); + } + + public function createFromUrl( + Document $resource, + Document $deployment, + string $url, + string $rootDirectory = '', + ): Document { + // The Builds worker clones the repo via the deployment's own + // installationId/providerRepositoryId instead of $url. + return $this->dispatch($resource, $deployment, null); + } + + private function dispatch(Document $resource, Document $deployment, ?Document $template): Document + { + $deployment = $this->upload($resource, $deployment); + $this->deactivateOthers($resource, $deployment); + + // The Builds worker promotes 'waiting' → 'building' on pickup. + $deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ + 'status' => 'waiting', + ])); + + $this->publisherForBuilds->enqueue(new BuildMessage( + project: $this->project, + resource: $resource, + deployment: $deployment, + type: BUILD_TYPE_DEPLOYMENT, + template: $template, + platform: $this->platform, + )); + + return $deployment; + } + + public function cancel(string $deploymentId): void + { + $this->executor->deleteRuntime($this->project->getId(), $deploymentId . '-build'); + } +} diff --git a/src/Appwrite/Compute/Job.php b/src/Appwrite/Deployment/Backend/Orchestrator.php similarity index 59% rename from src/Appwrite/Compute/Job.php rename to src/Appwrite/Deployment/Backend/Orchestrator.php index ac8d2ce84e..0288224953 100644 --- a/src/Appwrite/Compute/Job.php +++ b/src/Appwrite/Deployment/Backend/Orchestrator.php @@ -1,45 +1,108 @@ submit($resource, $deployment, null); + } + + public function createFromRef( + Document $resource, + Document $deployment, + string $owner, + string $repository, + string $type, + string $reference, + string $rootDirectory = '', + ): Document { + // The jobs-service has no GitHub client of its own — it only fetches + // tarballs — so $reference must already be a concrete commit/branch/ + // tag; codeload only understands one ref per tarball, not a range. + $url = "https://codeload.github.com/{$owner}/{$repository}/tar.gz/{$reference}"; + + return $this->submit($resource, $deployment, ['url' => $url, 'subdir' => $rootDirectory]); + } + + public function createFromUrl( + Document $resource, + Document $deployment, + string $url, + string $rootDirectory = '', + ): Document { + return $this->submit($resource, $deployment, ['url' => $url, 'subdir' => $rootDirectory]); + } + + private function submit(Document $resource, Document $deployment, ?array $source): Document + { + $deployment = $this->upload($resource, $deployment); + $this->deactivateOthers($resource, $deployment); + + // Pre-declare buildPath so build.sh writes output straight onto the + // mounted builds volume, at the path this deployment expects. + $deployment = $this->dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ + 'status' => 'waiting', + 'buildPath' => static::buildPath($this->project->getId(), $deployment->getId()), + ])); + + $this->jobs->create(...static::payload($this->project, $resource, $deployment, $this->platform, $source)); + + return $deployment; + } + + public function cancel(string $deploymentId): void + { + $this->jobs->delete(static::id($this->project->getId(), $deploymentId)); + } + /** * @return array Named arguments for OpenRuntimes\Orchestrator\Jobs::create(). */ - /** - * @param ?array{url: string, subdir?: string} $source Remote tarball source - * (template codeload URL, or a VCS presigned URL). When null, the source is - * the deployment's own uploaded tarball, fetched from Appwrite via a - * presigned token (manual upload / duplicate). - */ - public static function build( + protected static function payload( Document $project, Document $function, Document $deployment, @@ -97,20 +160,18 @@ final class Job ]; } - // Output + cache live on the mounted builds volume (see class doc): the - // build writes code.tar.gz to the deployment's build dir and the cache - // squashfs to its per-function keyed path — both directly on the volume - // Appwrite reads, so nothing large crosses the HTTP boundary. - $cacheKey = self::cacheKey($projectId, $function->getId(), $runtime['image'] ?? ''); + // Where output + cache land is a swappable strategy (see storage()) — + // the default mounts the shared builds volume; nothing else here cares + // which strategy is active. + $output = static::storage($project, $function, $deployment); + $command = $deployment->getAttribute('buildCommands', ''); $env = self::variables($project, $function, $deployment, $runtime, $cpus, $memory, $endpoint, $timeout) + [ 'OPEN_RUNTIMES_BUILD_INPUT_DIR' => '/mnt/code/source', - 'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => \dirname(self::buildPath($projectId, $deploymentId)), - 'OPEN_RUNTIMES_BUILD_CACHE_ARTIFACT' => self::cachePath($projectId, $cacheKey), - ]; + ] + $output['environment']; return [ - 'id' => self::id($projectId, $deploymentId), + 'id' => static::id($projectId, $deploymentId), 'image' => $runtime['image'], 'command' => '/usr/local/server/helpers/build.sh ' . \escapeshellarg($command), 'cpu' => $cpus, @@ -125,20 +186,19 @@ final class Job ], // The orchestrator expects environment as a string->string map. 'environment' => \array_map('strval', $env), - 'artifacts' => $sourceArtifacts, - // Attach the builds storage volume (Docker volume / K8s PVC named by - // _APP_BUILDS_VOLUME) to the worker at its Appwrite path so build.sh - // writes output + cache straight onto it. - 'volumes' => [ - new Volume(source: System::getEnv('_APP_BUILDS_VOLUME', 'appwrite-builds'), path: APP_STORAGE_BUILDS), - ], + 'artifacts' => [...$sourceArtifacts, ...$output['artifacts']], + 'volumes' => $output['volumes'], 'callback' => new Callback( url: "{$endpoint}/v1/jobs/event?" . \http_build_query(['project' => $projectId]), - // Artifact callbacks only carry the source-size stat, which exists - // only for remote-source builds (templates / VCS). + // Two terminal callbacks: exit carries the code (fires on + // command exit, before post-job artifacts), complete confirms + // artifact delivery (carries only jobId + meta) — the worker + // joins them, so readiness holds on any storage strategy. + // Artifact callbacks only carry the source-size stat, which + // exists only for remote-source builds (templates / VCS). events: $source !== null - ? [CallbackEvent::Log, CallbackEvent::Artifact, CallbackEvent::Exit] - : [CallbackEvent::Log, CallbackEvent::Exit], + ? [CallbackEvent::Log, CallbackEvent::Artifact, CallbackEvent::Exit, CallbackEvent::Complete] + : [CallbackEvent::Log, CallbackEvent::Exit, CallbackEvent::Complete], key: System::getEnv('_APP_JOBS_SECRET', ''), ), ]; @@ -176,7 +236,45 @@ final class Job return APP_STORAGE_BUILDS . "/app-{$projectId}/cache/{$cacheKey}.sqfs"; } - private static function runtime(Document $function, string $version): array + /** + * Where build.sh's output (code.tar.gz) and package-manager cache + * (a squashfs) land, and what the job needs to get them there. The + * default mounts the shared builds volume at buildPath()/cachePath(); + * build.sh only cares that OPEN_RUNTIMES_BUILD_OUTPUT_DIR/_CACHE_ARTIFACT + * point somewhere on its local filesystem, volume-backed or not — so a + * strategy without a shared volume (e.g. S3) instead points them at a + * local tmp path and moves things in/out via 'artifacts': + * - cache pull, before the build: a plain DownloadArtifact (no + * `depends`, so it runs before the command) into the local cache path. + * - cache push and output upload, after the build: an UploadArtifact + * with `depends: 'job'` — 'job' is the orchestrator's sentinel id for + * "after the build command finishes", not an id of another artifact. + * + * @return array{volumes: array, artifacts: array, environment: array} + */ + protected static function storage(Document $project, Document $resource, Document $deployment): array + { + $projectId = $project->getId(); + $deploymentId = $deployment->getId(); + $runtime = self::runtime($resource, $resource->getAttribute('version', 'v2')); + $cacheKey = static::cacheKey($projectId, $resource->getId(), $runtime['image'] ?? ''); + + return [ + // Docker volume / K8s PVC named by _APP_BUILDS_VOLUME, attached + // to the worker at its Appwrite path so build.sh writes output + + // cache straight onto it. + 'volumes' => [ + new Volume(source: System::getEnv('_APP_BUILDS_VOLUME', 'appwrite-builds'), path: APP_STORAGE_BUILDS), + ], + 'artifacts' => [], + 'environment' => [ + 'OPEN_RUNTIMES_BUILD_OUTPUT_DIR' => \dirname(static::buildPath($projectId, $deploymentId)), + 'OPEN_RUNTIMES_BUILD_CACHE_ARTIFACT' => static::cachePath($projectId, $cacheKey), + ], + ]; + } + + protected static function runtime(Document $function, string $version): array { $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); $runtime = $runtimes[$function->getAttribute('runtime')] ?? null; diff --git a/src/Appwrite/Docker/Compose/Generator.php b/src/Appwrite/Docker/Compose/Generator.php index 50f227af9f..2baed3f2f6 100644 --- a/src/Appwrite/Docker/Compose/Generator.php +++ b/src/Appwrite/Docker/Compose/Generator.php @@ -55,6 +55,7 @@ class Generator 'condition' => 'service_healthy', ], 'placeholders' => [ + '${_APP_DB_HOST:-postgresql}', '${_APP_DB_HOST:-mongodb}', '${_APP_DB_HOST:-mariadb}', ], diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index d490848037..6d5160b879 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -108,6 +108,7 @@ class Exception extends \Exception public const string USER_OAUTH2_BAD_REQUEST = 'user_oauth2_bad_request'; public const string USER_OAUTH2_UNAUTHORIZED = 'user_oauth2_unauthorized'; public const string USER_OAUTH2_PROVIDER_ERROR = 'user_oauth2_provider_error'; + public const string USER_OAUTH2_PROVIDER_FAILURE = 'user_oauth2_provider_failure'; public const string USER_EMAIL_ALREADY_VERIFIED = 'user_email_already_verified'; public const string USER_PHONE_ALREADY_VERIFIED = 'user_phone_already_verified'; public const string USER_DELETION_PROHIBITED = 'user_deletion_prohibited'; diff --git a/src/Appwrite/Platform/Modules/Compute/Base.php b/src/Appwrite/Platform/Modules/Compute/Base.php index b9dccfdca2..60591096ce 100644 --- a/src/Appwrite/Platform/Modules/Compute/Base.php +++ b/src/Appwrite/Platform/Modules/Compute/Base.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Compute; -use Appwrite\Compute\Job; +use Appwrite\Deployment\Backend; use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -10,7 +10,6 @@ use Appwrite\Filter\BranchDomain as BranchDomainFilter; use Appwrite\Platform\Action; use Appwrite\Platform\Modules\Compute\Validator\Specification as SpecificationValidator; use Appwrite\Platform\Permission as AppwritePermission; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; @@ -69,7 +68,7 @@ class Base extends Action return $allowedSpecifications[0]; } - public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Jobs $jobs = null): Document + public function redeployVcsFunction(Request $request, Document $function, Document $project, Document $installation, Database $dbForProject, BuildPublisher $publisherForBuilds, Document $template, Git $vcs, bool $activate, array $platform = [], string $referenceType = 'branch', string $reference = '', ?Backend $deployments = null): Document { $deploymentId = ID::unique(); $entrypoint = $function->getAttribute('entrypoint', ''); @@ -117,28 +116,8 @@ class Base extends Action $repositoryUrl = $vcs->getRepositoryUrl($owner, $repositoryName); - // Build a plain (non-template) VCS function deployment on the jobs-service - // when the caller opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator. - // Sites stay on the executor; template-into-repo pushes go through the - // Builds worker (which does the git write, then hands the build to the - // jobs-service itself when on orchestrator). When on jobs the deployment - // is pre-declared 'waiting' with its buildPath so build.sh writes output - // onto the mounted volume. - $useJobs = $jobs !== null - && $function->getCollection() === 'functions' - && $template->isEmpty() - && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'; - $buildFields = $useJobs - ? ['status' => 'waiting', 'buildPath' => Job::buildPath($project->getId(), $deploymentId)] - : []; - - $deployment = $dbForProject->createDocument('deployments', new Document([ + $deployment = new Document([ '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], 'resourceId' => $function->getId(), 'resourceInternalId' => $function->getSequence(), 'resourceType' => 'functions', @@ -163,17 +142,33 @@ class Base extends Action 'providerBranch' => $providerBranch, 'providerRootDirectory' => $function->getAttribute('providerRootDirectory', ''), 'activate' => $activate, - ...$buildFields, - ])); + ]); - if ($useJobs) { + // Build a plain (non-template) VCS function deployment through + // $deployments (executor or jobs-service, decided by + // _APP_BUILDS_BACKEND) when the caller opts in. Sites stay on the + // executor; template-into-repo pushes go through the Builds worker + // (which does the git write, then hands the build to the + // jobs-service itself when on orchestrator). + if ($deployments !== null && $function->getCollection() === 'functions' && $template->isEmpty()) { $ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch'); - $source = [ - 'url' => $vcs->getRepositoryPresignedUrl($owner, $repositoryName, $ref), - 'subdir' => $function->getAttribute('providerRootDirectory', ''), - ]; - $jobs->create(...Job::build($project, $function, $deployment, $platform, $source)); + $deployment = $deployments->createFromUrl( + $function, + $deployment, + $vcs->getRepositoryPresignedUrl($owner, $repositoryName, $ref), + $function->getAttribute('providerRootDirectory', ''), + ); } else { + $deployment = $dbForProject->createDocument('deployments', new Document([ + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ...$deployment->getArrayCopy(), + 'status' => 'waiting', + ])); + $publisherForBuilds->enqueue(new BuildMessage( project: $project, resource: $function, diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php index f6e4c59e44..8df9759afc 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php @@ -2,10 +2,8 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments; -use Appwrite\Compute\Job; +use Appwrite\Deployment\Backend; use Appwrite\Event\Event; -use Appwrite\Event\Message\Build as BuildMessage; -use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -13,13 +11,9 @@ use Appwrite\SDK\Method; use Appwrite\SDK\MethodType; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; -use Utopia\Database\Helpers\Permission; -use Utopia\Database\Helpers\Role; -use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Http\Adapter\Swoole\Request; @@ -92,11 +86,9 @@ class Create extends Action ->inject('project') ->inject('deviceForFunctions') ->inject('deviceForLocal') - ->inject('publisherForBuilds') - ->inject('jobs') + ->inject('deployments') ->inject('plan') ->inject('authorization') - ->inject('platform') ->inject('locks') ->callback($this->action(...)); } @@ -114,11 +106,9 @@ class Create extends Action Document $project, Device $deviceForFunctions, Device $deviceForLocal, - BuildPublisher $publisherForBuilds, - Jobs $jobs, + Backend $deployments, array $plan, Authorization $authorization, - array $platform, callable $locks ) { $activate = \strval($activate) === 'true' || \strval($activate) === '1'; @@ -227,7 +217,7 @@ class Create extends Action $type = $request->getHeaderLine('x-sdk-language') === 'cli' ? 'cli' : 'manual'; try { - $locks($lockKey, 600, function () use ($activate, &$chunks, $commands, $contentRange, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, &$metadata, $path, $type, &$completed, $response): void { + $locks($lockKey, 600, function () use ($activate, &$chunks, $commands, $contentRange, $dbForProject, $deploymentId, $deployments, $deviceForFunctions, $entrypoint, $fileSize, &$function, &$metadata, $path, $type, &$completed, $response): void { $deployment = $dbForProject->getDocument('deployments', $deploymentId); if (!$deployment->isEmpty()) { @@ -249,16 +239,8 @@ class Create extends Action $deviceForFunctions->prepareUpload($path, $metadata['content_type'] ?? '', $chunks, $metadata); if (!empty($contentRange)) { - $deployment = $dbForProject->createDocument('deployments', new Document([ + $deployment = $deployments->upload($function, $deployment->setAttributes([ '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceInternalId' => $function->getSequence(), - 'resourceId' => $function->getId(), - 'resourceType' => 'functions', 'entrypoint' => $entrypoint, 'buildCommands' => $commands, 'startCommand' => $function->getAttribute('startCommand', ''), @@ -269,7 +251,7 @@ class Create extends Action 'sourceChunksUploaded' => 0, 'activate' => $activate, 'sourceMetadata' => $metadata, - 'type' => $type + 'type' => $type, ])); } } @@ -291,7 +273,7 @@ class Create extends Action } try { - $locks($lockKey, 600, function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $functionId, $path, &$metadata, $mergeUploadMetadata, $platform, $project, $publisherForBuilds, $jobs, $queueForEvents, $response, $type): void { + $locks($lockKey, 600, function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $path, &$metadata, $mergeUploadMetadata, $deployments, $queueForEvents, $response, $type): void { $deployment = $dbForProject->getDocument('deployments', $deploymentId); $uploaded = 0; @@ -315,86 +297,24 @@ class Create extends Action if ($chunksUploaded === $chunks && $uploaded < $chunks) { $deviceForFunctions->finalizeUpload($path, $chunks, $metadata); - if ($activate) { - // Remove deploy for all other deployments. - $activeDeployments = $dbForProject->find('deployments', [ - Query::equal('activate', [true]), - Query::equal('resourceId', [$functionId]), - Query::equal('resourceType', ['functions']) - ]); - - foreach ($activeDeployments as $activeDeployment) { - $dbForProject->updateDocument('deployments', $activeDeployment->getId(), new Document([ - 'activate' => false, - ])); - } - } - $fileSize = $deviceForFunctions->getFileSize($path); - // Build backend for manual-upload function deployments: - // 'orchestrator' (open-runtimes jobs-service, submitted below - // in the request flow) or 'executor' (default; enqueued to the - // Builds worker). Selected by _APP_BUILDS_BACKEND. - $useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'; - - // Fields marking the deployment as queued. The build worker - // promotes 'waiting' → 'building' on the first callback. The - // jobs path also pre-declares buildPath so build.sh writes the - // output straight onto the mounted builds volume. - $buildFields = ['status' => 'waiting']; - if ($useJobs) { - $buildFields['buildPath'] = Job::buildPath($project->getId(), $deploymentId); - } - - if ($deployment->isEmpty()) { - $deployment = $dbForProject->createDocument('deployments', new Document([ - '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceInternalId' => $function->getSequence(), - 'resourceId' => $function->getId(), - 'resourceType' => 'functions', - 'entrypoint' => $entrypoint, - 'buildCommands' => $commands, - 'startCommand' => $function->getAttribute('startCommand', ''), - 'sourcePath' => $path, - 'sourceSize' => $fileSize, - 'totalSize' => $fileSize, - 'sourceChunksTotal' => $chunks, - 'sourceChunksUploaded' => $chunksUploaded, - 'activate' => $activate, - 'sourceMetadata' => $metadata, - 'type' => $type, - ...$buildFields, - ])); - - } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ - 'sourceSize' => $fileSize, - 'sourceChunksUploaded' => $chunksUploaded, - 'sourceMetadata' => $metadata, - ...$buildFields, - ])); - } - - if ($useJobs) { - $jobs->create(...Job::build($project, $function, $deployment, $platform)); - } else { - // Default: build on the executor via the Builds worker. - $publisherForBuilds->enqueue(new BuildMessage( - project: $project, - resource: $function, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - platform: $platform, - )); - } + $deployment = $deployments->createFromUpload($function, $deployment->setAttributes([ + '$id' => $deploymentId, + 'entrypoint' => $entrypoint, + 'buildCommands' => $commands, + 'startCommand' => $function->getAttribute('startCommand', ''), + 'sourcePath' => $path, + 'sourceSize' => $fileSize, + 'totalSize' => $fileSize, + 'sourceChunksTotal' => $chunks, + 'sourceChunksUploaded' => $chunksUploaded, + 'activate' => $activate, + 'sourceMetadata' => $metadata, + 'type' => $type, + ])); } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ + $deployment = $deployments->upload($function, $deployment->setAttributes([ 'sourceChunksUploaded' => $chunksUploaded, 'sourceMetadata' => $metadata, ])); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php index 1e50fcd214..44d6cdd68b 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php @@ -2,24 +2,21 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Duplicate; -use Appwrite\Compute\Job; +use Appwrite\Deployment\Backend; use Appwrite\Event\Event; -use Appwrite\Event\Message\Build as BuildMessage; -use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Database\Database; -use Utopia\Database\Document; use Utopia\Database\Helpers\ID; use Utopia\Database\Validator\UID; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; use Utopia\Storage\Device; use Utopia\System\System; +use Utopia\VCS\Adapter\Git\GitHub; class Create extends Action { @@ -65,12 +62,11 @@ class Create extends Action ->param('buildId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Build unique ID.', true, ['dbForProject']) // added as optional param for backward compatibility ->inject('response') ->inject('dbForProject') + ->inject('dbForPlatform') ->inject('queueForEvents') - ->inject('publisherForBuilds') - ->inject('jobs') + ->inject('deployments') ->inject('deviceForFunctions') - ->inject('project') - ->inject('platform') + ->inject('gitHub') ->callback($this->action(...)); } @@ -80,12 +76,11 @@ class Create extends Action string $buildId, Response $response, Database $dbForProject, + Database $dbForPlatform, Event $queueForEvents, - BuildPublisher $publisherForBuilds, - Jobs $jobs, + Backend $deployments, Device $deviceForFunctions, - Document $project, - array $platform + GitHub $github, ) { $function = $dbForProject->getDocument('functions', $functionId); @@ -98,27 +93,38 @@ class Create extends Action throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } + // Remote-source deployments (templates / VCS) on the jobs-service + // backend never store a source tarball — the build sidecar fetches + // it — so a duplicate re-fetches the same source from the + // coordinates persisted on the deployment. $path = $deployment->getAttribute('sourcePath'); - if (empty($path) || !$deviceForFunctions->exists($path)) { + $hasSource = ! empty($path) && $deviceForFunctions->exists($path); + $installationId = $deployment->getAttribute('installationId', ''); + $owner = $deployment->getAttribute('providerRepositoryOwner', ''); + $repository = $deployment->getAttribute('providerRepositoryName', ''); + + if (! $hasSource && ($owner === '' || $repository === '')) { throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } $deploymentId = ID::unique(); - // Build backend for the rebuild: 'orchestrator' (jobs-service, submitted - // here) or 'executor' (default; Builds worker). The jobs path pre-declares - // buildPath so build.sh writes output straight onto the mounted volume. - $useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'; - - $destination = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); - $deviceForFunctions->transfer($path, $destination, $deviceForFunctions); + $destination = ''; + if ($hasSource) { + $destination = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); + $deviceForFunctions->transfer($path, $destination, $deviceForFunctions); + } + // Cloning the source deployment's attributes onto the new one, with + // its own $id and no $sequence, tells the service to create it fresh + // rather than update the deployment being duplicated. A re-fetched + // source starts unsized; its stat artifact reports the size. $deployment->removeAttribute('$sequence'); - - $deployment = $dbForProject->createDocument('deployments', $deployment->setAttributes([ + $deployment->setAttributes([ '$id' => $deploymentId, 'sourcePath' => $destination, - 'totalSize' => $deployment->getAttribute('sourceSize', 0), + 'sourceSize' => $hasSource ? $deployment->getAttribute('sourceSize', 0) : 0, + 'totalSize' => $hasSource ? $deployment->getAttribute('sourceSize', 0) : 0, 'entrypoint' => $function->getAttribute('entrypoint'), 'buildCommands' => $function->getAttribute('commands', ''), 'startCommand' => $function->getAttribute('startCommand', ''), @@ -126,21 +132,43 @@ class Create extends Action 'buildEndedAt' => null, 'buildDuration' => null, 'buildSize' => null, - 'status' => 'waiting', - 'buildPath' => $useJobs ? Job::buildPath($project->getId(), $deploymentId) : '', + 'buildPath' => '', 'buildLogs' => '', - ])); + ]); - if ($useJobs) { - $jobs->create(...Job::build($project, $function, $deployment, $platform)); + if ($hasSource) { + $deployment = $deployments->createFromUpload($function, $deployment); + } elseif ($installationId !== '') { + $installation = $dbForPlatform->getDocument('installations', $installationId); + if ($installation->isEmpty()) { + throw new Exception(Exception::INSTALLATION_NOT_FOUND); + } + + $github->initializeVariables( + $installation->getAttribute('providerInstallationId', ''), + System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY'), + System::getEnv('_APP_VCS_GITHUB_APP_ID'), + ); + + $ref = $deployment->getAttribute('providerCommitHash') ?: $deployment->getAttribute('providerBranch'); + $deployment = $deployments->createFromUrl( + $function, + $deployment, + $github->getRepositoryPresignedUrl($owner, $repository, $ref), + $deployment->getAttribute('providerRootDirectory', ''), + ); } else { - $publisherForBuilds->enqueue(new BuildMessage( - project: $project, - resource: $function, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - platform: $platform, - )); + // Public template repo: providerBranch holds the resolved ref + // (branch, tag, or commit — see Deployments/Template/Create). + $deployment = $deployments->createFromRef( + $function, + $deployment, + $owner, + $repository, + GitHub::CLONE_TYPE_BRANCH, + $deployment->getAttribute('providerBranch', ''), + $deployment->getAttribute('providerRootDirectory', ''), + ); } $queueForEvents diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php index fb354f799d..961e3182b6 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php @@ -2,15 +2,13 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Status; -use Appwrite\Compute\Job; +use Appwrite\Deployment\Backend; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response; -use Executor\Executor; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; @@ -60,10 +58,8 @@ class Update extends Action ->param('deploymentId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Deployment ID.', false, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('project') ->inject('queueForEvents') - ->inject('executor') - ->inject('jobs') + ->inject('deployments') ->callback($this->action(...)); } @@ -72,10 +68,8 @@ class Update extends Action string $deploymentId, Response $response, Database $dbForProject, - Document $project, Event $queueForEvents, - Executor $executor, - Jobs $jobs, + Backend $deployments, ) { $function = $dbForProject->getDocument('functions', $functionId); @@ -115,15 +109,9 @@ class Update extends Action } } - // Best-effort cleanup on both backends — the deployment is already - // marked 'canceled', and only one backend actually holds the build. + // Best-effort cleanup — the deployment is already marked 'canceled'. try { - $executor->deleteRuntime($project->getId(), $deploymentId . "-build"); - } catch (\Throwable) { - } - - try { - $jobs->delete(Job::id($project->getId(), $deploymentId)); + $deployments->cancel($deploymentId); } catch (\Throwable) { } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php index 146c75c11f..699425ce39 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php @@ -2,9 +2,8 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Template; -use Appwrite\Compute\Job; +use Appwrite\Deployment\Backend; use Appwrite\Event\Event; -use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; @@ -14,22 +13,18 @@ use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response; use Appwrite\Vcs\Factory as VcsFactory; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; -use Utopia\Database\Helpers\Permission; -use Utopia\Database\Helpers\Role; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Http\Adapter\Swoole\Request; use Utopia\Platform\Enum; use Utopia\Platform\Scope\HTTP; -use Utopia\System\System; use Utopia\Validator\Boolean; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; -use Utopia\VCS\Adapter\Git; +use Utopia\VCS\Adapter\Git\GitHub; class Create extends Base { @@ -59,7 +54,7 @@ class Create extends Base name: 'createTemplateDeployment', description: <<inject('project') ->inject('publisherForBuilds') ->inject('vcsFactory') - ->inject('jobs') + ->inject('deployments') + ->inject('gitHub') ->inject('authorization') ->inject('platform') ->callback($this->action(...)); @@ -107,7 +103,8 @@ class Create extends Base Document $project, BuildPublisher $publisherForBuilds, VcsFactory $vcsFactory, - Jobs $jobs, + Backend $deployments, + GitHub $github, Authorization $authorization, array $platform ) { @@ -146,6 +143,7 @@ class Create extends Base template: $template, vcs: $vcsFactory->fromInstallation($installation), activate: $activate, + deployments: $deployments, platform: $platform, referenceType: $type, reference: $reference @@ -162,72 +160,56 @@ class Create extends Base return; } - // Backend for the template build: 'orchestrator' (jobs-service, which - // pulls the public GitHub tarball via artifacts) or 'executor' (default; - // the Builds worker clones the repo). The jobs path pre-declares buildPath. - $useJobs = System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'; - $deploymentId = ID::unique(); - $deployment = $dbForProject->createDocument('deployments', new Document([ - '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceId' => $function->getId(), - 'resourceInternalId' => $function->getSequence(), - 'resourceType' => 'functions', - 'entrypoint' => $function->getAttribute('entrypoint', ''), - 'buildCommands' => $function->getAttribute('commands', ''), - 'startCommand' => $function->getAttribute('startCommand', ''), - 'providerRepositoryName' => $repository, - 'providerRepositoryOwner' => $owner, - 'providerRepositoryUrl' => $repositoryUrl, - 'providerBranchUrl' => $branchUrl, - 'providerBranch' => $type == Git::CLONE_TYPE_BRANCH ? $reference : '', - 'type' => 'vcs', - 'activate' => $activate, - 'status' => 'waiting', - 'buildPath' => $useJobs ? Job::buildPath($project->getId(), $deploymentId) : '', - ])); + + // Templates can pin a version range (e.g. "0.3.*"); codeload only + // takes a concrete ref, so resolve the range to the highest matching + // tag (mirrors the executor's `git ls-remote --tags | tail -1`). + // Templates are public github.com repositories regardless of the + // function's own provider, so this always uses the GitHub adapter. + $ref = $reference; + if ($type === GitHub::CLONE_TYPE_TAG && \str_contains($reference, '*')) { + try { + $tags = $github->listTags($owner, $repository, $reference); + $ref = \end($tags) ?: $reference; + } catch (\Throwable) { + // Fall back to the raw reference; the build surfaces a bad ref. + } + } + + // Public template: pull the source straight from GitHub's public repo + // (codeload tarball on the jobs backend, a plain git clone on the + // executor); unarchive/checkout strips down to the rootDirectory. + $deployment = $deployments->createFromRef( + $function, + new Document([ + '$id' => $deploymentId, + 'entrypoint' => $function->getAttribute('entrypoint', ''), + 'buildCommands' => $function->getAttribute('commands', ''), + 'startCommand' => $function->getAttribute('startCommand', ''), + 'providerRepositoryName' => $repository, + 'providerRepositoryOwner' => $owner, + 'providerRepositoryUrl' => $repositoryUrl, + 'providerBranchUrl' => $branchUrl, + // The resolved concrete ref (branch, tag, or commit — codeload + // serves them all through the same URL form), so a duplicate + // can re-fetch the exact same source later; likewise the root + // directory. Remote-source builds never store a tarball, so + // these coordinates are all a redeploy has. + 'providerBranch' => $ref, + 'providerRootDirectory' => $rootDirectory, + 'type' => 'vcs', + 'activate' => $activate, + ]), + $owner, + $repository, + $type, + $ref, + $rootDirectory, + ); $this->updateEmptyManualRule($project, $function, $deployment, $dbForPlatform, $authorization); - if ($useJobs) { - // Templates can pin a version range (e.g. "0.3.*"); codeload only - // takes a concrete ref, so resolve the range to the highest matching - // tag (mirrors the executor's `git ls-remote --tags | tail -1`). - $ref = $reference; - if ($type === Git::CLONE_TYPE_TAG && \str_contains($reference, '*')) { - try { - // Templates are public github.com repositories regardless of - // the function's own provider. - $tags = $vcsFactory->fromProvider('github')->listTags($owner, $repository, $reference); - $ref = \end($tags) ?: $reference; - } catch (\Throwable) { - // Fall back to the raw reference; the build surfaces a bad ref. - } - } - - // Public template: pull the source straight from GitHub's codeload - // tarball; unarchive strips the "{repo}-{ref}/" wrapper + the rootDirectory. - $source = [ - 'url' => "https://codeload.github.com/{$owner}/{$repository}/tar.gz/{$ref}", - 'subdir' => $rootDirectory, - ]; - $jobs->create(...Job::build($project, $function, $deployment, $platform, $source)); - } else { - $publisherForBuilds->enqueue(new BuildMessage( - project: $project, - resource: $function, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - template: $template, - platform: $platform, - )); - } - $queueForEvents ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php index 75ade638e9..f539373048 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Vcs/Create.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Deployments\Vcs; +use Appwrite\Deployment\Backend; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; @@ -12,7 +13,6 @@ use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response; use Appwrite\Vcs\Factory as VcsFactory; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Validator\UID; @@ -75,7 +75,7 @@ class Create extends Base ->inject('queueForEvents') ->inject('publisherForBuilds') ->inject('vcsFactory') - ->inject('jobs') + ->inject('deployments') ->inject('platform') ->callback($this->action(...)); } @@ -93,7 +93,7 @@ class Create extends Base Event $queueForEvents, BuildPublisher $publisherForBuilds, VcsFactory $vcsFactory, - Jobs $jobs, + Backend $deployments, array $platform, ) { $function = $dbForProject->getDocument('functions', $functionId); @@ -123,7 +123,7 @@ class Create extends Base platform: $platform, reference: $reference, referenceType: $type, - jobs: $jobs + deployments: $deployments ); $queueForEvents diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php index 453dd000d1..d486ce2e8b 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Functions; +use Appwrite\Deployment\Backend; use Appwrite\Event\Event; use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Message\Func as FunctionMessage; @@ -22,7 +23,6 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model\Rule; use Appwrite\Vcs\Factory as VcsFactory; use Appwrite\Vcs\RepositoryWebhooks; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Abuse\Abuse; use Utopia\Config\Config; use Utopia\Database\Database; @@ -126,7 +126,7 @@ class Create extends Base ->inject('project') ->inject('queueForEvents') ->inject('publisherForBuilds') - ->inject('jobs') + ->inject('deployments') ->inject('queueForRealtime') ->inject('queueForWebhooks') ->inject('publisherForFunctions') @@ -172,7 +172,7 @@ class Create extends Base Document $project, Event $queueForEvents, BuildPublisher $publisherForBuilds, - Jobs $jobs, + Backend $deployments, Realtime $queueForRealtime, Webhook $queueForWebhooks, FunctionPublisher $publisherForFunctions, @@ -362,7 +362,7 @@ class Create extends Base installation: $installation, dbForProject: $dbForProject, publisherForBuilds: $publisherForBuilds, - jobs: $jobs, + deployments: $deployments, template: $template, vcs: $vcsFactory->fromInstallation($installation), activate: true, diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php index 62eb1c46b4..b44c8a7ef8 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Http\Functions; +use Appwrite\Deployment\Backend; use Appwrite\Event\Event; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Event\Validator\FunctionEvent; @@ -16,7 +17,6 @@ use Appwrite\Utopia\Response; use Appwrite\Vcs\Factory as VcsFactory; use Appwrite\Vcs\RepositoryWebhooks; use Executor\Executor; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -101,12 +101,12 @@ class Update extends Base System::getEnv('_APP_COMPUTE_MEMORY', 0), 'buildSpecifications' )), 'Build specification for the function deployments.', true, ['plan']) - ->param('runtimeSpecification', fn (array $plan) => $this->getDefaultSpecification($plan), fn (array $plan) => new Specification( + ->param('runtimeSpecification', null, fn (array $plan) => new Nullable(new Specification( $plan, Config::getParam('specifications', []), System::getEnv('_APP_COMPUTE_CPUS', 0), System::getEnv('_APP_COMPUTE_MEMORY', 0) - ), 'Runtime specification for the function executions.', true, ['plan']) + )), 'Runtime specification for the function executions.', true, ['plan']) ->param('deploymentRetention', 0, new Range(0, APP_COMPUTE_DEPLOYMENT_MAX_RETENTION), 'Days to keep non-active deployments before deletion. Value 0 means all deployments will be kept.', true) ->inject('request') ->inject('response') @@ -114,7 +114,7 @@ class Update extends Base ->inject('project') ->inject('queueForEvents') ->inject('publisherForBuilds') - ->inject('jobs') + ->inject('deployments') ->inject('dbForPlatform') ->inject('vcsFactory') ->inject('repositoryWebhooks') @@ -145,7 +145,7 @@ class Update extends Base ?array $providerBranches, ?array $providerPaths, ?string $buildSpecification, - string $runtimeSpecification, + ?string $runtimeSpecification, int $deploymentRetention, Request $request, Response $response, @@ -153,7 +153,7 @@ class Update extends Base Document $project, Event $queueForEvents, BuildPublisher $publisherForBuilds, - Jobs $jobs, + Backend $deployments, Database $dbForPlatform, VcsFactory $vcsFactory, RepositoryWebhooks $repositoryWebhooks, @@ -183,6 +183,7 @@ class Update extends Base } $buildSpecification ??= $function->getAttribute('buildSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT); + $runtimeSpecification ??= $function->getAttribute('runtimeSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT); $repositoryId = $function->getAttribute('repositoryId', ''); $repositoryInternalId = $function->getAttribute('repositoryInternalId', ''); @@ -328,7 +329,7 @@ class Update extends Base // Redeploy logic if (!$isConnected && !empty($providerRepositoryId)) { - $this->redeployVcsFunction($request, $function, $project, $installation, $dbForProject, $publisherForBuilds, new Document(), $vcsFactory->fromInstallation($installation), true, $platform, jobs: $jobs); + $this->redeployVcsFunction($request, $function, $project, $installation, $dbForProject, $publisherForBuilds, new Document(), $vcsFactory->fromInstallation($installation), true, $platform, deployments: $deployments); } // Inform scheduler if function is still active diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 17e0e03234..3b2daf976b 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -3,7 +3,7 @@ namespace Appwrite\Platform\Modules\Functions\Workers; use Ahc\Jwt\JWT; -use Appwrite\Compute\Job; +use Appwrite\Deployment\Backend; use Appwrite\Event\Event; use Appwrite\Event\Message\Func as FunctionMessage; use Appwrite\Event\Publisher\Func as FunctionPublisher; @@ -22,7 +22,6 @@ use Exception; use Executor\Exception as ExecutorException; use Executor\Exception\Timeout as ExecutorTimeout; use Executor\Executor; -use OpenRuntimes\Orchestrator\Jobs as JobsClient; use Swoole\Coroutine as Co; use Utopia\Config\Config; use Utopia\Console; @@ -100,7 +99,7 @@ class Builds extends Action ->inject('deviceForFiles') ->inject('log') ->inject('executor') - ->inject('jobs') + ->inject('deployments') ->inject('plan') ->callback($this->action(...)); } @@ -127,7 +126,7 @@ class Builds extends Action Device $deviceForFiles, Log $log, Executor $executor, - JobsClient $jobs, + Backend $deployments, array $plan ): void { $payload = $message->getPayload(); @@ -173,7 +172,7 @@ class Builds extends Action $getIsResourceBlocked, $log, $executor, - $jobs, + $deployments, $plan, $platform, (int) ($payload['timeout'] ?? System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900)) @@ -211,7 +210,7 @@ class Builds extends Action callable $getIsResourceBlocked, Log $log, Executor $executor, - JobsClient $jobs, + Backend $deployments, array $plan, array $platform, int $timeout @@ -528,20 +527,17 @@ class Builds extends Action // On the jobs backend the only build reaching this worker is the // template-into-repo push above (a git *write* the jobs-service // artifact system has no primitive for). With the commit pushed, - // hand the build to the jobs-service like any other VCS commit: - // source from the presigned tarball, output onto the mounted - // builds volume, progress via callbacks to the jobs worker. + // hand the build to the jobs-service like any other VCS commit, + // via the same Deployments service the HTTP endpoints use. if ($resource->getCollection() === 'functions' && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator') { - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), new Document([ - 'buildPath' => Job::buildPath($project->getId(), $deploymentId), - ])); - $ref = $deployment->getAttribute('providerCommitHash') ?: $branchName; - $jobs->create(...Job::build($project, $resource, $deployment, $platform, [ - 'url' => $providerAdapter->getRepositoryPresignedUrl($cloneOwner, $cloneRepository, $ref), - 'subdir' => $resource->getAttribute('providerRootDirectory', ''), - ])); + $deployments->createFromUrl( + $resource, + $deployment, + $providerAdapter->getRepositoryPresignedUrl($cloneOwner, $cloneRepository, $ref), + $resource->getAttribute('providerRootDirectory', ''), + ); Console::execute('rm -rf ' . \escapeshellarg('/tmp/builds/' . $deploymentId), '', $stdout, $stderr); diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php index b4ced5bce2..95703988cf 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Jobs.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Functions\Workers; -use Appwrite\Compute\Job; +use Appwrite\Deployment\Backend\Orchestrator; use Appwrite\Event\Event; use Appwrite\Event\Message\Func as FunctionMessage; use Appwrite\Event\Message\Jobs as JobsMessage; @@ -107,14 +107,13 @@ class Jobs extends Action return; } - // The build writes its output onto the mounted volume before build.sh - // exits, so the exit callback is a truthful completion signal. - $finalizing = $event->event === 'orchestrator.job.exit'; + $statusBefore = $deployment->getAttribute('status'); $deployment = match ($event->event) { 'orchestrator.job.log' => $this->onLog($dbForProject, $deployment, $event->data), 'orchestrator.job.artifact' => $this->onArtifact($dbForProject, $deployment, $event->data), - 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds, $vcsFactory), + 'orchestrator.job.exit' => $this->onExit($dbForProject, $dbForPlatform, $project, $deployment, (int) ($event->data['exitCode'] ?? 0), $usage, $publisherForUsage, $deviceForBuilds, $vcsFactory, $cache), + 'orchestrator.job.complete' => $this->onComplete($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $deviceForBuilds, $vcsFactory, $cache), default => $deployment, }; @@ -130,8 +129,10 @@ class Jobs extends Action // On a real terminal outcome (not a concurrently-canceled build), // notify webhooks + event-triggered functions of the deployment - // update (mirrors the executor Builds worker). - if ($finalizing && \in_array($deployment->getAttribute('status'), ['ready', 'failed'], true)) { + // update (mirrors the executor Builds worker). The transition can + // land on either the exit (failures) or the complete callback + // (success), so key off the status change rather than the event. + if ($statusBefore !== $deployment->getAttribute('status') && \in_array($deployment->getAttribute('status'), ['ready', 'failed'], true)) { $this->dispatchUpdate($queueForEvents, $queueForWebhooks, $publisherForFunctions, $project, $deployment); } }, self::LOCK_TIMEOUT); @@ -190,9 +191,13 @@ class Jobs extends Action } /** - * Resolve the build-command exit into a terminal outcome. The output is - * already on the mounted builds volume, so a zero exit enforces the size - * limit against the on-disk artifact before readying; a non-zero exit fails. + * A successful build needs both terminal callbacks before it can ready: + * exit carries the code but fires before post-job artifacts are + * delivered; complete confirms delivery but carries no code. Failures + * short-circuit at exit — no output is needed to fail. The success join + * is order-independent: each of exit(0)/complete leaves a marker, and + * whichever finds the other's marker finalizes (markers live in cache, + * applied under the per-deployment lock). */ private function onExit( Database $dbForProject, @@ -204,17 +209,70 @@ class Jobs extends Action UsagePublisher $publisherForUsage, Device $deviceForBuilds, VcsFactory $vcsFactory, + Cache $cache, ): Document { if ($exitCode !== 0) { return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, "Build failed with exit code {$exitCode}.", $usage, $publisherForUsage, $vcsFactory); } - $path = Job::buildPath($project->getId(), $deployment->getId()); + $cache->save('jobs-exit-' . $deployment->getId(), true); + if ($cache->load('jobs-complete-' . $deployment->getId(), self::DEDUPE_TTL) !== false) { + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $deviceForBuilds, $vcsFactory); + } + + return $deployment; + } + + /** + * The delivery half of the success join — see onExit. Emitted by the + * jobs-service once the job's post-job artifacts have run, so on every + * storage strategy the output is already where Appwrite reads it. + */ + private function onComplete( + Database $dbForProject, + Database $dbForPlatform, + Document $project, + Document $deployment, + UsageContext $usage, + UsagePublisher $publisherForUsage, + Device $deviceForBuilds, + VcsFactory $vcsFactory, + Cache $cache, + ): Document { + if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'], true)) { + return $deployment; // already finalized (a non-zero exit fails at the exit callback) + } + + if ($cache->load('jobs-exit-' . $deployment->getId(), self::DEDUPE_TTL) !== false) { + return $this->ready($dbForProject, $dbForPlatform, $project, $deployment, $usage, $publisherForUsage, $deviceForBuilds, $vcsFactory); + } + + $cache->save('jobs-complete-' . $deployment->getId(), true); + + return $deployment; + } + + /** + * Finalize a successful build: enforce the size limit against the stored + * output, then mark ready. + */ + private function ready( + Database $dbForProject, + Database $dbForPlatform, + Document $project, + Document $deployment, + UsageContext $usage, + UsagePublisher $publisherForUsage, + Device $deviceForBuilds, + VcsFactory $vcsFactory, + ): Document { + $path = Orchestrator::buildPath($project->getId(), $deployment->getId()); $size = $deviceForBuilds->exists($path) ? $deviceForBuilds->getFileSize($path) : 0; $limit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000'); if ($limit !== 0 && $size > $limit) { $deviceForBuilds->delete($path); + return $this->finalize($dbForProject, $dbForPlatform, $project, $deployment, false, 'Build size should be less than ' . \number_format($limit / (1000 * 1000), 2) . ' MBs.', $usage, $publisherForUsage, $vcsFactory); } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php index 79164c4999..8f3f0ba17e 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php @@ -2,9 +2,8 @@ namespace Appwrite\Platform\Modules\Sites\Http\Deployments; +use Appwrite\Deployment\Backend; use Appwrite\Event\Event; -use Appwrite\Event\Message\Build as BuildMessage; -use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -15,9 +14,6 @@ use Appwrite\Utopia\Response; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Helpers\ID; -use Utopia\Database\Helpers\Permission; -use Utopia\Database\Helpers\Role; -use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Http\Adapter\Swoole\Request; @@ -88,7 +84,7 @@ class Create extends Action ->inject('queueForEvents') ->inject('deviceForSites') ->inject('deviceForLocal') - ->inject('publisherForBuilds') + ->inject('deployments') ->inject('plan') ->inject('authorization') ->inject('platform') @@ -111,7 +107,7 @@ class Create extends Action Event $queueForEvents, Device $deviceForSites, Device $deviceForLocal, - BuildPublisher $publisherForBuilds, + Backend $deployments, array $plan, Authorization $authorization, array $platform, @@ -231,7 +227,7 @@ class Create extends Action } try { - $locks($lockKey, 600, function () use ($activate, $authorization, &$chunks, $commands, $contentRange, $dbForPlatform, $dbForProject, $deploymentId, $deviceForSites, $fileSize, &$metadata, $outputDirectory, $path, $platform, $project, &$site, $type, &$completed, $response): void { + $locks($lockKey, 600, function () use ($activate, $authorization, &$chunks, $commands, $contentRange, $dbForPlatform, $dbForProject, $deploymentId, $deployments, $deviceForSites, $fileSize, &$metadata, $outputDirectory, $path, $platform, $project, &$site, $type, &$completed, $response): void { $deployment = $dbForProject->getDocument('deployments', $deploymentId); if (!$deployment->isEmpty()) { @@ -253,16 +249,8 @@ class Create extends Action $deviceForSites->prepareUpload($path, $metadata['content_type'] ?? '', $chunks, $metadata); if (!empty($contentRange)) { - $deployment = $dbForProject->createDocument('deployments', new Document([ + $deployment = $deployments->upload($site, $deployment->setAttributes([ '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceInternalId' => $site->getSequence(), - 'resourceId' => $site->getId(), - 'resourceType' => 'sites', 'buildCommands' => \implode(' && ', $commands), 'startCommand' => $site->getAttribute('startCommand', ''), 'buildOutput' => $outputDirectory, @@ -325,7 +313,7 @@ class Create extends Action } try { - $locks($lockKey, 600, function () use ($activate, $authorization, $commands, &$chunks, $chunksUploaded, $dbForPlatform, $dbForProject, $deploymentId, $deviceForSites, $fileSize, &$metadata, $mergeUploadMetadata, $outputDirectory, $path, $platform, $project, $publisherForBuilds, $queueForEvents, $response, &$site, $siteId, $type): void { + $locks($lockKey, 600, function () use ($activate, $authorization, $commands, &$chunks, $chunksUploaded, $dbForPlatform, $dbForProject, $deploymentId, $deployments, $deviceForSites, $fileSize, &$metadata, $mergeUploadMetadata, $outputDirectory, $path, $platform, $project, $queueForEvents, $response, &$site, $type): void { $deployment = $dbForProject->getDocument('deployments', $deploymentId); $uploaded = 0; @@ -349,47 +337,26 @@ class Create extends Action if ($chunksUploaded === $chunks && $uploaded < $chunks) { $deviceForSites->finalizeUpload($path, $chunks, $metadata); - if ($activate) { - // Remove deploy for all other deployments. - $activeDeployments = $dbForProject->find('deployments', [ - Query::equal('activate', [true]), - Query::equal('resourceId', [$siteId]), - Query::equal('resourceType', ['sites']) - ]); - - foreach ($activeDeployments as $activeDeployment) { - $dbForProject->updateDocument('deployments', $activeDeployment->getId(), new Document(['activate' => false])); - } - } - $fileSize = $deviceForSites->getFileSize($path); + $isNewDeployment = $deployment->isEmpty(); + $deployment = $deployments->createFromUpload($site, $deployment->setAttributes([ + '$id' => $deploymentId, + 'buildCommands' => \implode(' && ', $commands), + 'startCommand' => $site->getAttribute('startCommand', ''), + 'buildOutput' => $outputDirectory, + 'adapter' => $site->getAttribute('adapter', ''), + 'fallbackFile' => $site->getAttribute('fallbackFile', ''), + 'sourcePath' => $path, + 'sourceSize' => $fileSize, + 'totalSize' => $fileSize, + 'sourceChunksTotal' => $chunks, + 'sourceChunksUploaded' => $chunksUploaded, + 'activate' => $activate, + 'sourceMetadata' => $metadata, + 'type' => $type, + ])); - if ($deployment->isEmpty()) { - $deployment = $dbForProject->createDocument('deployments', new Document([ - '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceInternalId' => $site->getSequence(), - 'resourceId' => $site->getId(), - 'resourceType' => 'sites', - 'buildCommands' => \implode(' && ', $commands), - 'startCommand' => $site->getAttribute('startCommand', ''), - 'buildOutput' => $outputDirectory, - 'adapter' => $site->getAttribute('adapter', ''), - 'fallbackFile' => $site->getAttribute('fallbackFile', ''), - 'sourcePath' => $path, - 'sourceSize' => $fileSize, - 'totalSize' => $fileSize, - 'sourceChunksTotal' => $chunks, - 'sourceChunksUploaded' => $chunksUploaded, - 'activate' => $activate, - 'sourceMetadata' => $metadata, - 'type' => $type, - ])); - + if ($isNewDeployment) { $sitesDomain = $platform['sitesDomain']; $domain = ID::unique() . "." . $sitesDomain; @@ -405,8 +372,8 @@ class Create extends Action 'domain' => $domain, 'type' => 'deployment', 'trigger' => 'deployment', - 'deploymentId' => $deployment->isEmpty() ? '' : $deployment->getId(), - 'deploymentInternalId' => $deployment->isEmpty() ? '' : $deployment->getSequence(), + 'deploymentId' => $deployment->getId(), + 'deploymentInternalId' => $deployment->getSequence(), 'deploymentResourceType' => 'site', 'deploymentResourceId' => $site->getId(), 'deploymentResourceInternalId' => $site->getSequence(), @@ -417,22 +384,7 @@ class Create extends Action 'region' => $project->getAttribute('region') ])) ); - } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ - 'sourceSize' => $fileSize, - 'sourceChunksUploaded' => $chunksUploaded, - 'sourceMetadata' => $metadata, - ])); } - - // Start the build - $publisherForBuilds->enqueue(new BuildMessage( - project: $project, - resource: $site, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - platform: $platform, - )); } else { $deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([ 'sourceChunksUploaded' => $chunksUploaded, diff --git a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php index 31cf2fa7d9..8010e9c4e1 100644 --- a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php +++ b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php @@ -35,6 +35,7 @@ use Utopia\Emails\Validator\Email as EmailValidator; use Utopia\Locale\Locale; use Utopia\Messaging\Adapter\SMS\GEOSMS\CallingCode; use Utopia\Platform\Scope\HTTP; +use Utopia\Storage\Validator\FileName; use Utopia\System\System; use Utopia\Validator\ArrayList; use Utopia\Validator\Text; @@ -326,6 +327,19 @@ class Create extends Action $url = Template::unParseURL($url); if (! empty($email)) { $projectName = $project->isEmpty() ? 'Console' : $project->getAttribute('name', '[APP-NAME]'); + if ($project->getId() === 'console') { + $projectName = $platform['platformName']; + } + + $smtpBaseTemplate = $project->getAttribute('smtpBaseTemplate', 'email-base'); + if (! (new FileName())->isValid($smtpBaseTemplate)) { + throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Invalid template path'); + } + $bodyTemplate = APP_CE_CONFIG_DIR . '/locale/templates/' . $smtpBaseTemplate . '.tpl'; + if (! \is_readable($bodyTemplate)) { + $smtpBaseTemplate = 'email-base'; + $bodyTemplate = APP_CE_CONFIG_DIR . '/locale/templates/email-base.tpl'; + } $body = $locale->getText('emails.invitation.body'); $preview = $locale->getText('emails.invitation.preview'); @@ -412,16 +426,32 @@ class Create extends Action 'project' => $projectName, ]; + if ($smtpBaseTemplate === APP_BRANDED_EMAIL_BASE_TEMPLATE) { + $emailVariables = [ + ...$emailVariables, + 'accentColor' => $platform['accentColor'], + 'logoUrl' => $platform['logoUrl'], + 'twitter' => $platform['twitterUrl'], + 'discord' => $platform['discordUrl'], + 'github' => $platform['githubUrl'], + 'terms' => $platform['termsUrl'], + 'privacy' => $platform['privacyUrl'], + 'platform' => $platform['platformName'], + ]; + } + $publisherForMails->enqueue(new MailMessage( project: $project, recipient: $invitee->getAttribute('email'), name: $invitee->getAttribute('name', ''), subject: $subject, template: MAIL_TEMPLATE_INVITATION, + bodyTemplate: $bodyTemplate, body: $body, preview: $preview, smtp: $smtpConfig, variables: $emailVariables, + customMailOptions: $smtpBaseTemplate === APP_BRANDED_EMAIL_BASE_TEMPLATE ? ['senderName' => $platform['emailSenderName']] : [], platform: $platform, )); } elseif (! empty($phone)) { diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Authorize/External/Update.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Authorize/External/Update.php index e06912452a..df629138e4 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Authorize/External/Update.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Authorize/External/Update.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Modules\VCS\Http\GitHub\Authorize\External; +use Appwrite\Deployment\Backend; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; @@ -11,7 +12,6 @@ use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Utopia\Response; use Appwrite\Vcs\Factory as VcsFactory; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; @@ -61,7 +61,7 @@ class Update extends Action ->inject('authorization') ->inject('getProjectDB') ->inject('publisherForBuilds') - ->inject('jobs') + ->inject('deployments') ->inject('platform') ->callback($this->action(...)); } @@ -77,7 +77,7 @@ class Update extends Action Authorization $authorization, callable $getProjectDB, BuildPublisher $publisherForBuilds, - Jobs $jobs, + Backend $deployments, array $platform ) { $installation = $dbForPlatform->getDocument('installations', $installationId); @@ -115,7 +115,7 @@ class Update extends Action throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); } - $owner = $vcs->getOwnerName($providerInstallationId); + $owner = $vcs->getOwnerName($providerInstallationId, (int) $providerRepositoryId); $pullRequestResponse = $vcs->getPullRequest($owner, $providerRepositoryName, $providerPullRequestId); $providerRepositoryUrl = $pullRequestResponse['head']['repo']['html_url'] ?? ''; @@ -137,7 +137,7 @@ class Update extends Action ...array_filter(array_column($prFiles, 'previous_filename')) ]; - $this->createGitDeployments($vcs, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, true, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs); + $this->createGitDeployments($vcs, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, true, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments); $response->noContent(); } diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php index fb11603285..5c20f10eb4 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php @@ -2,14 +2,12 @@ namespace Appwrite\Platform\Modules\VCS\Http\GitHub; -use Appwrite\Compute\Job; -use Appwrite\Event\Event; +use Appwrite\Deployment\Backend; use Appwrite\Event\Message\Build as BuildMessage; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\Filter\BranchDomain as BranchDomainFilter; use Appwrite\Vcs\Comment; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Config\Config; use Utopia\Console; use Utopia\Database\Database; @@ -52,7 +50,7 @@ trait Deployment BuildPublisher $publisherForBuilds, callable $getProjectDB, array $platform, - ?Jobs $jobs = null, + ?Backend $deployments = null, ) { $errors = []; foreach ($repositories as $repository) { @@ -363,25 +361,8 @@ trait Deployment $commands[] = $resource->getAttribute('commands', ''); } - // Build function deployments on the jobs-service when the caller - // opts in ($jobs) and _APP_BUILDS_BACKEND=orchestrator. Sites stay - // on the executor. On jobs the deployment is pre-declared 'waiting' - // with its buildPath so build.sh writes onto the mounted volume. - $useJobs = $jobs !== null - && $resourceCollection === 'functions' - && System::getEnv('_APP_BUILDS_BACKEND', 'executor') === 'orchestrator'; - $buildFields = $useJobs - ? ['status' => 'waiting', 'buildPath' => Job::buildPath($projectId, $deploymentId)] - : []; - - $deployment = $authorization->skip(fn () => $dbForProject->createDocument('deployments', new Document([ + $deployment = new Document([ '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - ...$buildFields, 'resourceId' => $resourceId, 'resourceInternalId' => $resourceInternalId, 'resourceType' => $resourceCollection, @@ -409,7 +390,37 @@ trait Deployment 'providerCommentId' => \strval($latestCommentId), 'providerBranch' => $providerBranch, 'activate' => $activate, - ]))); + ]); + + // Build function deployments through $deployments (executor + // or jobs-service, decided by _APP_BUILDS_BACKEND) when the + // caller opts in. Sites always stay on the executor. + if ($deployments !== null && $resourceCollection === 'functions') { + $deployment = $authorization->skip(fn () => $deployments->createFromUrl( + $resource, + $deployment, + $vcs->getRepositoryPresignedUrl($providerRepositoryOwner, $providerRepositoryName, $providerCommitHash), + $resource->getAttribute('providerRootDirectory', ''), + )); + } else { + $deployment = $authorization->skip(fn () => $dbForProject->createDocument('deployments', new Document([ + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ...$deployment->getArrayCopy(), + 'status' => 'waiting', + ]))); + + $publisherForBuilds->enqueue(new BuildMessage( + project: $project, + resource: $resource, + deployment: $deployment, + type: BUILD_TYPE_DEPLOYMENT, + platform: $platform, + )); + } if ($resource->getCollection() === 'sites') { $projectId = $project->getId(); @@ -572,27 +583,6 @@ trait Deployment $vcs->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'pending', $message, $providerTargetUrl, $name); } - if ($useJobs) { - $source = [ - 'url' => $vcs->getRepositoryPresignedUrl($providerRepositoryOwner, $providerRepositoryName, $providerCommitHash), - 'subdir' => $resource->getAttribute('providerRootDirectory', ''), - ]; - $jobs->create(...Job::build($project, $resource, $deployment, $platform, $source)); - } else { - $queueName = $this->getBuildQueueName($project, $dbForPlatform, $authorization); - - $publisherForBuilds->enqueue( - new BuildMessage( - project: $project, - resource: $resource, - deployment: $deployment, - type: BUILD_TYPE_DEPLOYMENT, - platform: $platform, - ), - new \Utopia\Queue\Queue($queueName) - ); - } - Span::add("{$logBase}.build.triggered", 'true'); //TODO: Add event? } catch (Exception $e) { @@ -618,9 +608,4 @@ trait Deployment { } - protected function getBuildQueueName(Document $project, Database $dbForPlatform, Authorization $authorization): string - { - return System::getEnv('_APP_BUILDS_QUEUE_NAME', Event::BUILDS_QUEUE_NAME); - } - } diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Events/Create.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Events/Create.php index dfbc1656f7..9e48159505 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Events/Create.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Events/Create.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Modules\VCS\Http\GitHub\Events; +use Appwrite\Deployment\Backend; use Appwrite\Event\Publisher\Build as BuildPublisher; use Appwrite\Extend\Exception; use Appwrite\Platform\Action; @@ -9,7 +10,6 @@ use Appwrite\Platform\Modules\VCS\Http\GitHub\Deployment; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Appwrite\Vcs\Factory as VcsFactory; -use OpenRuntimes\Orchestrator\Jobs; use Utopia\Console; use Utopia\Database\Database; use Utopia\Database\Document; @@ -45,7 +45,7 @@ class Create extends Action ->inject('authorization') ->inject('getProjectDB') ->inject('publisherForBuilds') - ->inject('jobs') + ->inject('deployments') ->inject('platform') ->callback($this->action(...)); } @@ -59,7 +59,7 @@ class Create extends Action Authorization $authorization, callable $getProjectDB, BuildPublisher $publisherForBuilds, - Jobs $jobs, + Backend $deployments, array $platform ) { $this->preprocessEvent($request); @@ -84,8 +84,8 @@ class Create extends Action match ($event) { GitHub::EVENT_INSTALLATION => $this->handleInstallationEvent($parsedPayload, $dbForPlatform, $authorization, $getProjectDB), - GitHub::EVENT_PUSH => $this->handlePushEvent($parsedPayload, $vcsFactory, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs), - GitHub::EVENT_PULL_REQUEST => $this->handlePullRequestEvent($parsedPayload, $vcsFactory, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs), + GitHub::EVENT_PUSH => $this->handlePushEvent($parsedPayload, $vcsFactory, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments), + GitHub::EVENT_PULL_REQUEST => $this->handlePullRequestEvent($parsedPayload, $vcsFactory, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments), default => null, }; @@ -190,7 +190,7 @@ class Create extends Action BuildPublisher $publisherForBuilds, callable $getProjectDB, array $platform, - ?Jobs $jobs = null, + ?Backend $deployments = null, ) { $providerBranchDeleted = $parsedPayload["branchDeleted"] ?? false; $providerBranch = $parsedPayload["branch"] ?? ''; @@ -226,7 +226,7 @@ class Create extends Action // Create new deployment only on push (not committed by us) and not when branch is deleted if ($providerCommitAuthorEmail !== APP_VCS_GITHUB_EMAIL && !$providerBranchDeleted) { $providerAffectedFiles = $parsedPayload['affectedFiles'] ?? []; - $this->createGitDeployments($vcs, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthorName, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', $providerAffectedFiles, false, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs); + $this->createGitDeployments($vcs, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthorName, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, '', $providerAffectedFiles, false, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments); } } @@ -238,7 +238,7 @@ class Create extends Action BuildPublisher $publisherForBuilds, callable $getProjectDB, array $platform, - ?Jobs $jobs = null, + ?Backend $deployments = null, ) { $action = $parsedPayload["action"] ?? ''; @@ -292,7 +292,7 @@ class Create extends Action Query::orderDesc('$createdAt') ])); - $this->createGitDeployments($vcs, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, $external, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $jobs); + $this->createGitDeployments($vcs, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $providerAffectedFiles, $external, $dbForPlatform, $authorization, $publisherForBuilds, $getProjectDB, $platform, $deployments); } elseif ($action == "closed") { // Allowed external contributions cleanup diff --git a/src/Appwrite/Platform/Workers/Mails.php b/src/Appwrite/Platform/Workers/Mails.php index 8c1b03f726..3595ecc2ef 100644 --- a/src/Appwrite/Platform/Workers/Mails.php +++ b/src/Appwrite/Platform/Workers/Mails.php @@ -121,7 +121,7 @@ class Mails extends Action if (!empty($preview)) { $previewTemplate = Template::fromString($preview); foreach ($variables as $key => $value) { - $previewTemplate->setParam('{{' . $key . '}}', $value); + $previewTemplate->setParam('{{' . $key . '}}', $value, escapeHtml: false); } // render() will return the subject in

tags, so use strip_tags() to remove them $preview = \strip_tags($previewTemplate->render()); @@ -140,7 +140,7 @@ class Mails extends Action $subjectTemplate = Template::fromString($subject); foreach ($variables as $key => $value) { - $subjectTemplate->setParam('{{' . $key . '}}', $value); + $subjectTemplate->setParam('{{' . $key . '}}', $value, escapeHtml: false); } // render() will return the subject in

tags, so use strip_tags() to remove them $subject = \strip_tags($subjectTemplate->render()); diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php index bce33f0efb..5466ade630 100644 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -259,6 +259,8 @@ class Webhooks extends Action $projectId = $project->getId(); $projectInternalId = $project->getSequence(); + $region = $project->getAttribute('region', 'default'); + $webhookId = $webhook->getId(); $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS', 'disabled') === 'disabled' ? 'http' : 'https'; $consoleHostname = System::getEnv('_APP_CONSOLE_DOMAIN', System::getEnv('_APP_DOMAIN', 'localhost')); @@ -302,7 +304,7 @@ class Webhooks extends Action $template->setParam('{{url}}', $webhook->getAttribute('url')); $template->setParam('{{error}}', 'The server returned ' . $statusCode . ' status code'); $template->setParam('{{host}}', $protocol . '://' . $consoleHostname); - $template->setParam('{{path}}', "/projects/$projectId/settings/webhooks"); + $template->setParam('{{path}}', "/console/project-$region-$projectId/settings/webhooks/$webhookId"); $template->setParam('{{attempts}}', $attempts); $publisherForNotifications->enqueue(new NotificationMessage( diff --git a/src/Appwrite/Vcs/Comment.php b/src/Appwrite/Vcs/Comment.php index 77df933a04..8741ecff6c 100644 --- a/src/Appwrite/Vcs/Comment.php +++ b/src/Appwrite/Vcs/Comment.php @@ -161,7 +161,7 @@ class Comment }; if ($site['action']['type'] === 'logs') { - $action = '[View Logs](' . $protocol . '://' . $hostname . '/projects/' . $projectId . '/sites/' . $siteId . '/deployments/' . $site['deploymentId'] . ')'; + $action = '[View Logs](' . $protocol . '://' . $hostname . '/console/project-' . $site['region'] . '-' . $projectId . '/sites/site-' . $siteId . '/deployments/deployment-' . $site['deploymentId'] . ')'; } else { $action = '[Authorize](' . $site['action']['url'] . ')'; } @@ -209,7 +209,7 @@ class Comment }; if ($function['action']['type'] === 'logs') { - $action = '[View Logs](' . $protocol . '://' . $hostname . '/projects/' . $projectId . '/functions/' . $functionId . '/deployments/' . $function['deploymentId'] . ')'; + $action = '[View Logs](' . $protocol . '://' . $hostname . '/console/project-' . $function['region'] . '-' . $projectId . '/functions/function-' . $functionId . '/deployment-' . $function['deploymentId'] . ')'; } else { $action = '[Authorize](' . $function['action']['url'] . ')'; } diff --git a/tests/e2e/General/HTTPTest.php b/tests/e2e/General/HTTPTest.php index 68331ae47f..16f0c8199f 100644 --- a/tests/e2e/General/HTTPTest.php +++ b/tests/e2e/General/HTTPTest.php @@ -26,7 +26,6 @@ final class HTTPTest extends Scope /** * Test for SUCCESS */ - $this->client->setEndpoint('http://localhost'); $response = $this->client->call(Client::METHOD_OPTIONS, '/', \array_merge([ 'origin' => 'http://localhost', 'content-type' => 'application/json', @@ -52,7 +51,6 @@ final class HTTPTest extends Scope /** * Test for SUCCESS */ - $this->client->setEndpoint('http://localhost'); $response = $this->client->call(Client::METHOD_GET, '/humans.txt', \array_merge([ 'origin' => 'http://localhost', ])); @@ -66,7 +64,6 @@ final class HTTPTest extends Scope /** * Test for SUCCESS */ - $this->client->setEndpoint('http://localhost'); $response = $this->client->call(Client::METHOD_GET, '/robots.txt', \array_merge([ 'origin' => 'http://localhost', ])); @@ -80,7 +77,6 @@ final class HTTPTest extends Scope /** * Test for SUCCESS */ - $this->client->setEndpoint('http://localhost'); $response = $this->client->call(Client::METHOD_GET, '/.well-known/acme-challenge/8DdIKX257k6Dih5s_saeVMpTnjPJdKO5Ase0OCiJrIg'); // 'Unknown path', but validation passed @@ -91,8 +87,8 @@ final class HTTPTest extends Scope */ $response = $this->client->call(Client::METHOD_GET, '/.well-known/acme-challenge/../../../../../../../etc/passwd'); - // 'Invalid challenge token', traversal rejected by validation - $this->assertEquals(400, $response['headers']['status-code']); + // 'Unknown path', but validation passed + $this->assertEquals(404, $response['headers']['status-code']); } public function testVersions() @@ -100,7 +96,6 @@ final class HTTPTest extends Scope /** * Test without header */ - $this->client->setEndpoint('http://localhost'); $response = $this->client->call(Client::METHOD_GET, '/versions', \array_merge([ 'content-type' => 'application/json', ], $this->getHeaders())); @@ -177,23 +172,11 @@ final class HTTPTest extends Scope /** * Test for SUCCESS */ - $this->client->setEndpoint('http://localhost'); $endpoint = '/invite?membershipId=123&userId=asdf'; - $response = $this->client->call(Client::METHOD_GET, $endpoint, [], [], true, false); + $response = $this->client->call(Client::METHOD_GET, $endpoint); $this->assertEquals('/console' . $endpoint, $response['headers']['location']); } - - public function testConsoleServed() - { - /** - * Test for SUCCESS - */ - $response = $this->client->call(Client::METHOD_GET, '/'); - - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertStringContainsString('text/html', (string) $response['headers']['content-type']); - } } diff --git a/tests/e2e/Services/Functions/FunctionsBase.php b/tests/e2e/Services/Functions/FunctionsBase.php index 55c511d515..00d6a14c3f 100644 --- a/tests/e2e/Services/Functions/FunctionsBase.php +++ b/tests/e2e/Services/Functions/FunctionsBase.php @@ -492,4 +492,13 @@ trait FunctionsBase return $specifications; } + + protected function getEnabledSpecification(array $specifications): string + { + $specification = array_find($specifications, fn (array $specification) => $specification['enabled']); + + $this->assertNotNull($specification, 'Expected at least one enabled specification.'); + + return $specification['slug']; + } } diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 608ba28eb0..c2dbe7c614 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -190,7 +190,7 @@ final class FunctionsCustomServerTest extends Scope $buildSpecifications = $this->listSpecifications(['type' => 'builds']); $this->assertEquals(200, $buildSpecifications['headers']['status-code']); $this->assertEquals($specifications['body']['total'], $buildSpecifications['body']['total']); - $buildSpecification = $buildSpecifications['body']['specifications'][0]['slug']; + $buildSpecification = $this->getEnabledSpecification($buildSpecifications['body']['specifications']); $function = $this->createFunction([ 'functionId' => ID::unique(), diff --git a/tests/e2e/Services/Sites/SitesBase.php b/tests/e2e/Services/Sites/SitesBase.php index c90298d8f8..00cc0a6cc6 100644 --- a/tests/e2e/Services/Sites/SitesBase.php +++ b/tests/e2e/Services/Sites/SitesBase.php @@ -487,4 +487,13 @@ trait SitesBase return $specifications; } + + protected function getEnabledSpecification(array $specifications): string + { + $specification = array_find($specifications, fn (array $specification) => $specification['enabled']); + + $this->assertNotNull($specification, 'Expected at least one enabled specification.'); + + return $specification['slug']; + } } diff --git a/tests/e2e/Services/Sites/SitesCustomServerTest.php b/tests/e2e/Services/Sites/SitesCustomServerTest.php index 88e50ec7b4..c5c424bd52 100644 --- a/tests/e2e/Services/Sites/SitesCustomServerTest.php +++ b/tests/e2e/Services/Sites/SitesCustomServerTest.php @@ -43,7 +43,7 @@ final class SitesCustomServerTest extends Scope $buildSpecifications = $this->listSpecifications(['type' => 'builds']); $this->assertEquals(200, $buildSpecifications['headers']['status-code']); $this->assertEquals($specifications['body']['total'], $buildSpecifications['body']['total']); - $buildSpecification = $buildSpecifications['body']['specifications'][0]['slug']; + $buildSpecification = $this->getEnabledSpecification($buildSpecifications['body']['specifications']); $site = $this->createSite([ 'buildRuntime' => 'node-22', @@ -86,7 +86,7 @@ final class SitesCustomServerTest extends Scope public function testCreateSite(): void { $buildSpecifications = $this->listSpecifications(['type' => 'builds']); - $buildSpecification = $buildSpecifications['body']['specifications'][0]['slug']; + $buildSpecification = $this->getEnabledSpecification($buildSpecifications['body']['specifications']); /** * Test for SUCCESS @@ -748,7 +748,7 @@ final class SitesCustomServerTest extends Scope */ $siteId = $this->setupSite([ 'buildRuntime' => 'node-22', - 'buildSpecification' => $buildSpecifications['body']['specifications'][0]['slug'], + 'buildSpecification' => $this->getEnabledSpecification($buildSpecifications['body']['specifications']), 'fallbackFile' => '', 'framework' => 'analog', 'name' => 'Test List Sites', @@ -894,7 +894,7 @@ final class SitesCustomServerTest extends Scope $siteId = $this->setupSite([ 'buildRuntime' => 'node-22', - 'buildSpecification' => $buildSpecifications['body']['specifications'][0]['slug'], + 'buildSpecification' => $this->getEnabledSpecification($buildSpecifications['body']['specifications']), 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', @@ -1682,7 +1682,7 @@ final class SitesCustomServerTest extends Scope public function testUpdateSpecs(): void { $buildSpecifications = $this->listSpecifications(['type' => 'builds']); - $buildSpecification = $buildSpecifications['body']['specifications'][0]['slug']; + $buildSpecification = $this->getEnabledSpecification($buildSpecifications['body']['specifications']); $siteId = $this->setupSite([ 'buildRuntime' => 'node-22', @@ -2229,7 +2229,7 @@ final class SitesCustomServerTest extends Scope $proxyClient->setEndpoint('http://' . $deploymentDomain); $response = $proxyClient->call(Client::METHOD_GET, '/', followRedirects: false); $this->assertEquals(301, $response['headers']['status-code']); - $this->assertStringContainsString('/auth/preview', (string) $response['headers']['location']); + $this->assertStringContainsString('/console/auth/preview', (string) $response['headers']['location']); $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 900, 0); $apiKey = $jwtObj->encode([ @@ -2561,7 +2561,7 @@ final class SitesCustomServerTest extends Scope $response = $proxyClient->call(Client::METHOD_GET, '/contact', followRedirects: false); $this->assertEquals(301, $response['headers']['status-code']); - $this->assertStringContainsString('/auth/preview', (string) $response['headers']['location']); + $this->assertStringContainsString('/console/auth/preview', (string) $response['headers']['location']); $this->assertStringContainsString('projectId=' . $this->getProject()['$id'], (string) $response['headers']['location']); $this->assertStringContainsString('origin=', (string) $response['headers']['location']); $this->assertStringContainsString('path=%2Fcontact', (string) $response['headers']['location']); @@ -2624,7 +2624,7 @@ final class SitesCustomServerTest extends Scope 'cookie' => 'a_jwt_console=' . $jwt['body']['jwt'] ], followRedirects: false); $this->assertEquals(301, $response['headers']['status-code']); - $this->assertStringContainsString('/auth/preview', (string) $response['headers']['location']); + $this->assertStringContainsString('/console/auth/preview', (string) $response['headers']['location']); // Failure: User missing $cookie = 'a_session_console=' .$this->getRoot()['session']; @@ -2659,7 +2659,7 @@ final class SitesCustomServerTest extends Scope 'cookie' => 'a_jwt_console=' . $jwt['body']['jwt'] ], followRedirects: false); $this->assertEquals(301, $response['headers']['status-code']); - $this->assertStringContainsString('/auth/preview', (string) $response['headers']['location']); + $this->assertStringContainsString('/console/auth/preview', (string) $response['headers']['location']); // Failure: Membership missing $email = \uniqid() . 'newuser@appwrite.io'; @@ -2699,7 +2699,7 @@ final class SitesCustomServerTest extends Scope 'cookie' => 'a_jwt_console=' . $jwt['body']['jwt'] ], followRedirects: false); $this->assertEquals(301, $response['headers']['status-code']); - $this->assertStringContainsString('/auth/preview', (string) $response['headers']['location']); + $this->assertStringContainsString('/console/auth/preview', (string) $response['headers']['location']); $this->cleanupSite($siteId); } diff --git a/tests/unit/Auth/OAuth2/GithubTest.php b/tests/unit/Auth/OAuth2/GithubTest.php new file mode 100644 index 0000000000..1bf85befde --- /dev/null +++ b/tests/unit/Auth/OAuth2/GithubTest.php @@ -0,0 +1,166 @@ +createGithub(\json_encode([ + 'access_token' => 'access-token', + 'scope' => 'user:email', + 'token_type' => 'bearer', + ], JSON_THROW_ON_ERROR)); + + $this->assertSame('access-token', $github->getAccessToken('authorization-code')); + } + + public function testProviderError(): void + { + $github = $this->createGithub(\json_encode([ + 'error' => 'bad_verification_code', + 'error_description' => 'The code passed is incorrect or expired.', + ], JSON_THROW_ON_ERROR), 'expired-code'); + + try { + $github->getAccessToken('expired-code'); + $this->fail('Expected the GitHub OAuth2 provider error to be thrown.'); + } catch (Exception $exception) { + $this->assertSame(AppwriteException::USER_OAUTH2_BAD_REQUEST, $exception->getType()); + $this->assertSame('bad_verification_code', $exception->getError()); + $this->assertSame('The code passed is incorrect or expired.', $exception->getErrorDescription()); + } + } + + public function testFormEncodedProviderError(): void + { + $github = $this->createGithub( + 'error=bad_verification_code&error_description=The+code+passed+is+incorrect+or+expired.', + 'expired-code', + ); + + try { + $github->getAccessToken('expired-code'); + $this->fail('Expected the form-encoded GitHub OAuth2 provider error to be thrown.'); + } catch (Exception $exception) { + $this->assertSame(AppwriteException::USER_OAUTH2_BAD_REQUEST, $exception->getType()); + $this->assertSame('bad_verification_code', $exception->getError()); + $this->assertSame('The code passed is incorrect or expired.', $exception->getErrorDescription()); + } + } + + public function testProviderErrorWithInvalidUtf8(): void + { + $github = $this->createGithub( + 'error=bad_verification_code&error_description=Invalid+byte%3A+%FF', + 'expired-code', + ); + + try { + $github->getAccessToken('expired-code'); + $this->fail('Expected the GitHub OAuth2 provider error with invalid UTF-8 to be thrown.'); + } catch (Exception $exception) { + $this->assertSame(AppwriteException::USER_OAUTH2_BAD_REQUEST, $exception->getType()); + $this->assertSame('bad_verification_code', $exception->getError()); + $this->assertSame('Invalid byte: �', $exception->getErrorDescription()); + } + } + + public function testMissingAccessToken(): void + { + $github = $this->createGithub('{}'); + + try { + $github->getAccessToken('authorization-code'); + $this->fail('Expected a missing access token error to be thrown.'); + } catch (Exception $exception) { + $this->assertSame(AppwriteException::USER_OAUTH2_BAD_REQUEST, $exception->getType()); + $this->assertSame('access_token_missing', $exception->getError()); + $this->assertSame('GitHub did not return an access token.', $exception->getErrorDescription()); + } + } + + public function testProviderFailure(): void + { + $previous = new Exception(\json_encode([ + 'error' => 'bad_verification_code', + 'error_description' => 'The code passed is incorrect or expired.', + ], JSON_THROW_ON_ERROR), 400); + + $exception = new AppwriteException( + AppwriteException::USER_OAUTH2_PROVIDER_FAILURE, + previous: $previous, + params: ['GitHub', $previous->getError()], + ); + + $this->assertSame(AppwriteException::USER_OAUTH2_PROVIDER_FAILURE, $exception->getType()); + $this->assertSame(424, $exception->getCode()); + $this->assertSame( + 'GitHub couldn\'t complete sign-in (bad_verification_code). Please try again.', + $exception->getMessage(), + ); + $this->assertSame($previous, $exception->getPrevious()); + } + + public function testPlainTextProviderFailure(): void + { + $previous = new Exception('Invalid secret'); + $providerError = $previous->getError() ?: $previous->getMessage(); + + $exception = new AppwriteException( + AppwriteException::USER_OAUTH2_PROVIDER_FAILURE, + previous: $previous, + params: ['Apple', $providerError], + ); + + $this->assertSame( + 'Apple couldn\'t complete sign-in (Invalid secret). Please try again.', + $exception->getMessage(), + ); + $this->assertSame($previous, $exception->getPrevious()); + } + + private function createGithub(string $response, string $code = 'authorization-code'): Github&MockObject + { + $github = $this->getMockBuilder(Github::class) + ->setConstructorArgs(['client-id', 'client-secret', 'https://example.com/callback']) + ->onlyMethods(['request']) + ->getMock(); + + $github + ->expects($this->once()) + ->method('request') + ->with( + 'POST', + 'https://github.com/login/oauth/access_token', + ['Accept: application/json'], + $this->callback(function (mixed $payload) use ($code): bool { + if (!\is_string($payload)) { + return false; + } + + \parse_str($payload, $params); + + $this->assertSame([ + 'client_id' => 'client-id', + 'redirect_uri' => 'https://example.com/callback', + 'client_secret' => 'client-secret', + 'code' => $code, + ], $params); + + return true; + }), + ) + ->willReturn($response); + + return $github; + } +} diff --git a/tests/unit/Docker/Compose/GeneratorTest.php b/tests/unit/Docker/Compose/GeneratorTest.php index 76b2a13378..29187a4939 100644 --- a/tests/unit/Docker/Compose/GeneratorTest.php +++ b/tests/unit/Docker/Compose/GeneratorTest.php @@ -111,7 +111,7 @@ final class GeneratorTest extends TestCase $this->assertSame(['appwrite'], $compose['services']['traefik']['depends_on']); $this->assertArrayHasKey('postgresql', $compose['services']['appwrite']['depends_on']); - $this->assertArrayNotHasKey('${_APP_DB_HOST:-mongodb}', $compose['services']['appwrite']['depends_on']); + $this->assertArrayNotHasKey('${_APP_DB_HOST:-postgresql}', $compose['services']['appwrite']['depends_on']); } public function testKeepsLongCommandsReadable(): void diff --git a/tests/unit/Platform/Workers/WebhooksTest.php b/tests/unit/Platform/Workers/WebhooksTest.php index cf611e285b..356db9f146 100644 --- a/tests/unit/Platform/Workers/WebhooksTest.php +++ b/tests/unit/Platform/Workers/WebhooksTest.php @@ -69,7 +69,7 @@ final class WebhooksTest extends TestCase ); $this->assertStringContainsString('Payments', (string) $payload['body']); $this->assertStringContainsString('Ada Lovelace', (string) $payload['body']); - $this->assertStringContainsString('/projects/project-1/settings/webhooks', (string) $payload['body']); + $this->assertStringContainsString('/console/project-fra-project-1/settings/webhooks/webhook-1', (string) $payload['body']); $this->assertStringNotContainsString('{{', (string) $payload['body']); $this->assertSame(APP_NAME, $payload['variables']['platform']); $this->assertSame(APP_EMAIL_LOGO_URL, $payload['variables']['logoUrl']);