From 8dd99b7d2a4eaecabb5d52c00d04b2bdde610948 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Thu, 6 Feb 2025 23:59:43 +0530 Subject: [PATCH 01/31] Add framework detection to Appwrite --- app/controllers/api/vcs.php | 217 +++++++++++++----- composer.json | 9 +- composer.lock | 106 +++++++-- src/Appwrite/Utopia/Response.php | 9 +- .../Response/Model/FrameworkDetection.php | 58 +++++ .../{Detection.php => RuntimeDetection.php} | 18 +- 6 files changed, 334 insertions(+), 83 deletions(-) create mode 100644 src/Appwrite/Utopia/Response/Model/FrameworkDetection.php rename src/Appwrite/Utopia/Response/Model/{Detection.php => RuntimeDetection.php} (50%) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 921a840c0a..dc64f34a7e 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -26,22 +26,35 @@ use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Query\Cursor; -use Utopia\Detector\Adapter\Bun; -use Utopia\Detector\Adapter\CPP; -use Utopia\Detector\Adapter\Dart; -use Utopia\Detector\Adapter\Deno; -use Utopia\Detector\Adapter\Dotnet; -use Utopia\Detector\Adapter\Java; -use Utopia\Detector\Adapter\JavaScript; -use Utopia\Detector\Adapter\PHP; -use Utopia\Detector\Adapter\Python; -use Utopia\Detector\Adapter\Ruby; -use Utopia\Detector\Adapter\Swift; -use Utopia\Detector\Detector; +use Utopia\Detector\Detection\Framework\Astro; +use Utopia\Detector\Detection\Framework\Flutter; +use Utopia\Detector\Detection\Framework\NextJs; +use Utopia\Detector\Detection\Framework\Nuxt; +use Utopia\Detector\Detection\Framework\Remix; +use Utopia\Detector\Detection\Framework\SvelteKit; +use Utopia\Detector\Detection\Packager\NPM; +use Utopia\Detector\Detection\Packager\PNPM; +use Utopia\Detector\Detection\Packager\Yarn; +use Utopia\Detector\Detection\Runtime\Bun; +use Utopia\Detector\Detection\Runtime\CPP; +use Utopia\Detector\Detection\Runtime\Dart; +use Utopia\Detector\Detection\Runtime\Deno; +use Utopia\Detector\Detection\Runtime\Dotnet; +use Utopia\Detector\Detection\Runtime\Java; +use Utopia\Detector\Detection\Runtime\Node; +use Utopia\Detector\Detection\Runtime\PHP; +use Utopia\Detector\Detection\Runtime\Python; +use Utopia\Detector\Detection\Runtime\Ruby; +use Utopia\Detector\Detection\Runtime\Swift; +use Utopia\Detector\Detector\Framework; +use Utopia\Detector\Detector\Packager; +use Utopia\Detector\Detector\Runtime; +use Utopia\Detector\Detector\Strategy; use Utopia\System\System; use Utopia\Validator\Boolean; use Utopia\Validator\Host; use Utopia\Validator\Text; +use Utopia\Validator\WhiteList; use Utopia\VCS\Adapter\Git\GitHub; use Utopia\VCS\Exception\RepositoryNotFound; @@ -544,8 +557,9 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories/:pro ]), Response::MODEL_VCS_CONTENT_LIST); }); -App::post('/v1/vcs/github/installations/:installationId/providerRepositories/:providerRepositoryId/detection') - ->desc('Detect runtime settings from source code') +App::post('/v1/vcs/github/installations/:installationId/providerRepositories/:providerRepositoryId/detections') + ->alias('/v1/vcs/github/installations/:installationId/providerRepositories/:providerRepositoryId/detection') + ->desc('Detect runtime and framework settings from source code') ->groups(['api', 'vcs']) ->label('scope', 'vcs.write') ->label('sdk', new Method( @@ -556,18 +570,22 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories/:pr responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, - model: Response::MODEL_DETECTION, + model: Response::MODEL_RUNTIME_DETECTION, + ), + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_FRAMEWORK_DETECTION, ) ] )) ->param('installationId', '', new Text(256), 'Installation Id') ->param('providerRepositoryId', '', new Text(256), 'Repository Id') ->param('providerRootDirectory', '', new Text(256, 0), 'Path to Root Directory', true) + ->param('type', '', new WhiteList(['runtime', 'framework']), 'Detector type. Must be one of the following: runtime, framework', true) ->inject('gitHub') ->inject('response') - ->inject('project') ->inject('dbForPlatform') - ->action(function (string $installationId, string $providerRepositoryId, string $providerRootDirectory, GitHub $github, Response $response, Document $project, Database $dbForPlatform) { + ->action(function (string $installationId, string $providerRepositoryId, string $providerRootDirectory, string $type, GitHub $github, Response $response, Database $dbForPlatform) { $installation = $dbForPlatform->getDocument('installations', $installationId); if ($installation->isEmpty()) { @@ -593,32 +611,100 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories/:pr $files = \array_column($files, 'name'); $languages = $github->listRepositoryLanguages($owner, $repositoryName); - $detectorFactory = new Detector($files, $languages); + $detector = new Packager($files); + $detector + ->addOption(new Yarn()) + ->addOption(new PNPM()) + ->addOption(new NPM()); + $detectedPackager = $detector->detect(); - $detectorFactory - ->addDetector(new JavaScript()) - ->addDetector(new Bun()) - ->addDetector(new PHP()) - ->addDetector(new Python()) - ->addDetector(new Dart()) - ->addDetector(new Swift()) - ->addDetector(new Ruby()) - ->addDetector(new Java()) - ->addDetector(new CPP()) - ->addDetector(new Deno()) - ->addDetector(new Dotnet()); - - $runtime = $detectorFactory->detect(); - - $runtimes = Config::getParam('runtimes'); - $runtimeDetail = \array_reverse(\array_filter(\array_keys($runtimes), function ($key) use ($runtime, $runtimes) { - return $runtimes[$key]['key'] === $runtime; - }))[0] ?? ''; + $packagerName = $detectedPackager ? $detectedPackager->getName() : 'npm'; $detection = []; - $detection['runtime'] = $runtimeDetail; - $response->dynamic(new Document($detection), Response::MODEL_DETECTION); + if ($type === 'framework') { + $detection = [ + 'framework' => '', + 'installCommand' => '', + 'buildCommand' => '', + 'outputDirectory' => '', + ]; + + $frameworkDetector = new Framework($files, $packagerName); + $frameworkDetector + ->addOption(new Flutter()) + ->addOption(new Nuxt()) + ->addOption(new Astro()) + ->addOption(new Remix()) + ->addOption(new SvelteKit()) + ->addOption(new NextJs()); + + $detectedFramework = $frameworkDetector->detect(); + + if ($detectedFramework) { + $framework = $detectedFramework->getName(); + $detection['installCommand'] = $detectedFramework->getInstallCommand(); + $detection['buildCommand'] = $detectedFramework->getBuildCommand(); + $detection['outputDirectory'] = $detectedFramework->getOutputDirectory(); + } + + if (!empty($framework)) { + $frameworks = Config::getParam('frameworks'); + $frameworkDetail = \array_reverse(\array_filter(\array_keys($frameworks), function ($key) use ($framework, $frameworks) { + return $frameworks[$key]['key'] === $framework; + }))[0] ?? ''; + $detection['framework'] = $frameworkDetail; + } + + $response->dynamic(new Document($detection), Response::MODEL_FRAMEWORK_DETECTION); + } else { + $detection = [ + 'runtime' => '', + 'commands' => '', + 'entrypoint' => '', + ]; + + $strategies = [ + new Strategy(Strategy::FILEMATCH), + new Strategy(Strategy::LANGUAGES), + new Strategy(Strategy::EXTENSION), + ]; + + foreach ($strategies as $strategy) { + $runtimeDetector = new Runtime($strategy === Strategy::LANGUAGES ? $languages : $files, $strategy, $packagerName); + $runtimeDetector + ->addOption(new Node()) + ->addOption(new Bun()) + ->addOption(new Deno()) + ->addOption(new PHP()) + ->addOption(new Python()) + ->addOption(new Dart()) + ->addOption(new Swift()) + ->addOption(new Ruby()) + ->addOption(new Java()) + ->addOption(new CPP()) + ->addOption(new Dotnet()); + + $detectedRuntime = $runtimeDetector->detect(); + + if ($detectedRuntime) { + $detection['commands'] = $detectedRuntime->getCommands(); + $detection['entrypoint'] = $detectedRuntime->getEntrypoint(); + $runtime = $detectedRuntime->getName(); + break; + } + } + + if (!empty($runtime)) { + $runtimes = Config::getParam('runtimes'); + $runtimeDetail = \array_reverse(\array_filter(\array_keys($runtimes), function ($key) use ($runtime, $runtimes) { + return $runtimes[$key]['key'] === $runtime; + }))[0] ?? ''; + $detection['runtime'] = $runtimeDetail; + } + + $response->dynamic(new Document($detection), Response::MODEL_RUNTIME_DETECTION); + } }); App::get('/v1/vcs/github/installations/:installationId/providerRepositories') @@ -680,29 +766,44 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') $files = \array_column($files, 'name'); $languages = $github->listRepositoryLanguages($repo['organization'], $repo['name']); - $detectorFactory = new Detector($files, $languages); + $strategies = [ + new Strategy(Strategy::FILEMATCH), + new Strategy(Strategy::LANGUAGES), + new Strategy(Strategy::EXTENSION), + ]; - $detectorFactory - ->addDetector(new JavaScript()) - ->addDetector(new Bun()) - ->addDetector(new PHP()) - ->addDetector(new Python()) - ->addDetector(new Dart()) - ->addDetector(new Swift()) - ->addDetector(new Ruby()) - ->addDetector(new Java()) - ->addDetector(new CPP()) - ->addDetector(new Deno()) - ->addDetector(new Dotnet()); + foreach ($strategies as $strategy) { + $runtimeDetector = new Runtime($strategy === Strategy::LANGUAGES ? $languages : $files, $strategy, 'npm'); + $runtimeDetector + ->addOption(new Node()) + ->addOption(new Bun()) + ->addOption(new Deno()) + ->addOption(new PHP()) + ->addOption(new Python()) + ->addOption(new Dart()) + ->addOption(new Swift()) + ->addOption(new Ruby()) + ->addOption(new Java()) + ->addOption(new CPP()) + ->addOption(new Dotnet()); - $runtime = $detectorFactory->detect(); + $detectedRuntime = $runtimeDetector->detect(); - $runtimes = Config::getParam('runtimes'); - $runtimeDetail = \array_reverse(\array_filter(\array_keys($runtimes), function ($key) use ($runtime, $runtimes) { - return $runtimes[$key]['key'] === $runtime; - }))[0] ?? ''; + if ($detectedRuntime) { + $runtime = $detectedRuntime->getName(); + break; + } + } - $repo['runtime'] = $runtimeDetail; + if (!empty($runtime)) { + $runtimes = Config::getParam('runtimes'); + $runtimeDetail = \array_reverse(\array_filter(\array_keys($runtimes), function ($key) use ($runtime, $runtimes) { + return $runtimes[$key]['key'] === $runtime; + }))[0] ?? ''; + $repo['runtime'] = $runtimeDetail; + } else { + throw new Exception("Runtime not detected"); + } } catch (Throwable $error) { $repo['runtime'] = ""; Console::warning("Runtime not detected for " . $repo['organization'] . "/" . $repo['name']); diff --git a/composer.json b/composer.json index b497451efd..2d5eb3edb1 100644 --- a/composer.json +++ b/composer.json @@ -52,6 +52,7 @@ "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", "utopia-php/database": "0.56.4", + "utopia-php/detector": "dev-feat-pseudocode-draft2 as 0.1.99", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "dev-fix-prevent-duplicate-compression as 0.33.99", @@ -102,5 +103,11 @@ "php-http/discovery": true, "tbachert/spi": true } - } + }, + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/utopia-php/detector" + } + ] } diff --git a/composer.lock b/composer.lock index a0bcb622b2..00855c40c6 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2caa51e1b7d11e3e67ade41347530f92", + "content-hash": "895d12203ebc543ae26dced08f811b04", "packages": [ { "name": "adhocore/jwt", @@ -1237,16 +1237,16 @@ }, { "name": "open-telemetry/api", - "version": "1.2.1", + "version": "1.2.2", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "74b1a03263be8c5acb578f41da054b4bac3af4a0" + "reference": "8b925df3047628968bc5be722468db1b98b82d51" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/74b1a03263be8c5acb578f41da054b4bac3af4a0", - "reference": "74b1a03263be8c5acb578f41da054b4bac3af4a0", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/8b925df3047628968bc5be722468db1b98b82d51", + "reference": "8b925df3047628968bc5be722468db1b98b82d51", "shasum": "" }, "require": { @@ -1303,7 +1303,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-20T23:35:16+00:00" + "time": "2025-02-03T21:49:11+00:00" }, { "name": "open-telemetry/context", @@ -1493,16 +1493,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.2.1", + "version": "1.2.2", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "96aeaee5b7cb8c0bc4af7ff4717b429f2d9f67e1" + "reference": "37eec0fe47ddd627911f318f29b6cd48196be0c0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/96aeaee5b7cb8c0bc4af7ff4717b429f2d9f67e1", - "reference": "96aeaee5b7cb8c0bc4af7ff4717b429f2d9f67e1", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/37eec0fe47ddd627911f318f29b6cd48196be0c0", + "reference": "37eec0fe47ddd627911f318f29b6cd48196be0c0", "shasum": "" }, "require": { @@ -1579,24 +1579,24 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-09T23:17:14+00:00" + "time": "2025-01-29T21:40:28+00:00" }, { "name": "open-telemetry/sem-conv", - "version": "1.27.1", + "version": "1.30.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sem-conv.git", - "reference": "1dba705fea74bc0718d04be26090e3697e56f4e6" + "reference": "4178c9f390da8e4dbca9b181a9d1efd50cf7ee0a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sem-conv/zipball/1dba705fea74bc0718d04be26090e3697e56f4e6", - "reference": "1dba705fea74bc0718d04be26090e3697e56f4e6", + "url": "https://api.github.com/repos/opentelemetry-php/sem-conv/zipball/4178c9f390da8e4dbca9b181a9d1efd50cf7ee0a", + "reference": "4178c9f390da8e4dbca9b181a9d1efd50cf7ee0a", "shasum": "" }, "require": { - "php": "^8.1" + "php": "^8.0" }, "type": "library", "extra": { @@ -1636,7 +1636,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2024-08-28T09:20:31+00:00" + "time": "2025-02-06T00:21:48+00:00" }, { "name": "paragonie/constant_time_encoding", @@ -3530,6 +3530,69 @@ }, "time": "2025-01-20T09:22:08+00:00" }, + { + "name": "utopia-php/detector", + "version": "dev-feat-pseudocode-draft2", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/detector.git", + "reference": "09512d06b4b3a1a4acca2403ea9c22f03975d79e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/detector/zipball/09512d06b4b3a1a4acca2403ea9c22f03975d79e", + "reference": "09512d06b4b3a1a4acca2403ea9c22f03975d79e", + "shasum": "" + }, + "require": { + "php": ">=8.0" + }, + "require-dev": { + "laravel/pint": "1.2.*", + "phpstan/phpstan": "1.8.*", + "phpunit/phpunit": "^9.4" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Detector\\": "src/" + } + }, + "autoload-dev": { + "psr-4": { + "Utopia\\Tests\\": "tests/Detector/" + } + }, + "scripts": { + "lint": [ + "./vendor/bin/pint --test --config pint.json" + ], + "format": [ + "./vendor/bin/pint --config pint.json" + ], + "check": [ + "./vendor/bin/phpstan analyse --level 8 -c phpstan.neon src tests" + ], + "test": [ + "./vendor/bin/phpunit --configuration phpunit.xml --debug" + ] + }, + "license": [ + "MIT" + ], + "description": "A simple library for fast and reliable environment identification.", + "keywords": [ + "detector", + "framework", + "php", + "utopia" + ], + "support": { + "source": "https://github.com/utopia-php/detector/tree/feat-pseudocode-draft2", + "issues": "https://github.com/utopia-php/detector/issues" + }, + "time": "2025-02-06T13:36:29+00:00" + }, { "name": "utopia-php/domains", "version": "0.5.0", @@ -8502,6 +8565,12 @@ } ], "aliases": [ + { + "package": "utopia-php/detector", + "version": "dev-feat-pseudocode-draft2", + "alias": "0.1.99", + "alias_normalized": "0.1.99.0" + }, { "package": "utopia-php/framework", "version": "dev-fix-prevent-duplicate-compression", @@ -8511,6 +8580,7 @@ ], "minimum-stability": "stable", "stability-flags": { + "utopia-php/detector": 20, "utopia-php/framework": 20 }, "prefer-stable": false, @@ -8536,5 +8606,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.6.0" } diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index da222822e0..e9c3540672 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -38,7 +38,6 @@ use Appwrite\Utopia\Response\Model\Country; use Appwrite\Utopia\Response\Model\Currency; use Appwrite\Utopia\Response\Model\Database; use Appwrite\Utopia\Response\Model\Deployment; -use Appwrite\Utopia\Response\Model\Detection; use Appwrite\Utopia\Response\Model\Document as ModelDocument; use Appwrite\Utopia\Response\Model\Error; use Appwrite\Utopia\Response\Model\ErrorDev; @@ -46,6 +45,7 @@ use Appwrite\Utopia\Response\Model\Execution; use Appwrite\Utopia\Response\Model\File; use Appwrite\Utopia\Response\Model\Framework; use Appwrite\Utopia\Response\Model\FrameworkAdapter; +use Appwrite\Utopia\Response\Model\FrameworkDetection; use Appwrite\Utopia\Response\Model\Func; use Appwrite\Utopia\Response\Model\Headers; use Appwrite\Utopia\Response\Model\HealthAntivirus; @@ -85,6 +85,7 @@ use Appwrite\Utopia\Response\Model\Provider; use Appwrite\Utopia\Response\Model\ProviderRepository; use Appwrite\Utopia\Response\Model\Rule; use Appwrite\Utopia\Response\Model\Runtime; +use Appwrite\Utopia\Response\Model\RuntimeDetection; use Appwrite\Utopia\Response\Model\Session; use Appwrite\Utopia\Response\Model\Site; use Appwrite\Utopia\Response\Model\Specification; @@ -249,7 +250,8 @@ class Response extends SwooleResponse public const MODEL_PROVIDER_REPOSITORY_LIST = 'providerRepositoryList'; public const MODEL_BRANCH = 'branch'; public const MODEL_BRANCH_LIST = 'branchList'; - public const MODEL_DETECTION = 'detection'; + public const MODEL_FRAMEWORK_DETECTION = 'frameworkDetection'; + public const MODEL_RUNTIME_DETECTION = 'runtimeDetection'; public const MODEL_VCS_CONTENT = 'vcsContent'; public const MODEL_VCS_CONTENT_LIST = 'vcsContentList'; @@ -453,7 +455,8 @@ class Response extends SwooleResponse ->setModel(new TemplateVariable()) ->setModel(new Installation()) ->setModel(new ProviderRepository()) - ->setModel(new Detection()) + ->setModel(new FrameworkDetection()) + ->setModel(new RuntimeDetection()) ->setModel(new VcsContent()) ->setModel(new Branch()) ->setModel(new Runtime()) diff --git a/src/Appwrite/Utopia/Response/Model/FrameworkDetection.php b/src/Appwrite/Utopia/Response/Model/FrameworkDetection.php new file mode 100644 index 0000000000..d0c666ae91 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/FrameworkDetection.php @@ -0,0 +1,58 @@ +addRule('framework', [ + 'type' => self::TYPE_STRING, + 'description' => 'Framework', + 'default' => '', + 'example' => 'nuxt', + ]) + ->addRule('installCommand', [ + 'type' => self::TYPE_STRING, + 'description' => 'Site Install Command', + 'default' => '', + 'example' => 'npm install', + ]) + ->addRule('buildCommand', [ + 'type' => self::TYPE_STRING, + 'description' => 'Site Build Command', + 'default' => '', + 'example' => 'npm run build', + ]) + ->addRule('outputDirectory', [ + 'type' => self::TYPE_STRING, + 'description' => 'Site Output Directory', + 'default' => '', + 'example' => 'dist', + ]); + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'FrameworkDetection'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_FRAMEWORK_DETECTION; + } +} diff --git a/src/Appwrite/Utopia/Response/Model/Detection.php b/src/Appwrite/Utopia/Response/Model/RuntimeDetection.php similarity index 50% rename from src/Appwrite/Utopia/Response/Model/Detection.php rename to src/Appwrite/Utopia/Response/Model/RuntimeDetection.php index c71baa0b0c..b79dccff9d 100644 --- a/src/Appwrite/Utopia/Response/Model/Detection.php +++ b/src/Appwrite/Utopia/Response/Model/RuntimeDetection.php @@ -5,7 +5,7 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; -class Detection extends Model +class RuntimeDetection extends Model { public function __construct() { @@ -15,6 +15,18 @@ class Detection extends Model 'description' => 'Runtime', 'default' => '', 'example' => 'node', + ]) + ->addRule('entrypoint', [ + 'type' => self::TYPE_STRING, + 'description' => 'Function Entrypoint', + 'default' => '', + 'example' => 'index.js', + ]) + ->addRule('commands', [ + 'type' => self::TYPE_STRING, + 'description' => 'Function install and build commands', + 'default' => '', + 'example' => 'npm install && npm run build', ]); } @@ -25,7 +37,7 @@ class Detection extends Model */ public function getName(): string { - return 'Detection'; + return 'RuntimeDetection'; } /** @@ -35,6 +47,6 @@ class Detection extends Model */ public function getType(): string { - return Response::MODEL_DETECTION; + return Response::MODEL_RUNTIME_DETECTION; } } From 0a3f284e14f48cd4288289525698bbc4818f5fa5 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Fri, 7 Feb 2025 11:18:07 +0530 Subject: [PATCH 02/31] Update composer.json --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index a60b643445..000aa2170c 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4da9bee4423753c2e592c081b19b8711", + "content-hash": "c238139c9b44c646364734186c481036", "packages": [ { "name": "adhocore/jwt", From 7be0fbf61600afe16da771e6223ba8a135aa27af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 10 Feb 2025 15:07:01 +0100 Subject: [PATCH 03/31] Implementation of authorized previews --- app/controllers/general.php | 75 ++++++++++++++++++++++++++++++++++++- src/Appwrite/Auth/Auth.php | 5 +++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 6764cbae40..68584edd10 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -3,6 +3,7 @@ require_once __DIR__ . '/../init.php'; use Ahc\Jwt\JWT; +use Ahc\Jwt\JWTException; use Appwrite\Auth\Auth; use Appwrite\Event\Certificate; use Appwrite\Event\Event; @@ -130,7 +131,7 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw }; } - if ($type === 'function' || $type === 'site') { + if ($type === 'function' || $type === 'site' || $type === 'deployment') { $method = $utopia->getRoute()?->getLabel('sdk', null); if (empty($method)) { @@ -173,6 +174,78 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw $path .= '?' . $query; } + $protocol = $request->getProtocol(); + + // Preview authorization (configure) + if (\str_starts_with($path, '/_appwrite/authorize')) { + $jwt = $request->getParam('jwt', ''); + $path = $request->getParam('path', ''); + + $duration = 60 * 60 * 24; // 1 day in seconds + $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)); + + $response + ->addCookie(Auth::$cookieNamePreview, $jwt, (new \DateTime($expire))->getTimestamp(), '/', $host, ('https' === $protocol), true, null) + ->addHeader('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') + ->addHeader('Pragma', 'no-cache') + ->redirect($protocol . '://' . $host . $path); + return true; + } + + // Preview authorization (validate) + if ($type === 'deployment') { + $cookie = $request->getCookie(Auth::$cookieNamePreview, ''); + $ok = true; + + if (empty($cookie)) { + $ok = false; + } else { + $jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 3600, 0); + + $payload = []; + try { + $payload = $jwt->decode($cookie); + } catch (JWTException $error) { + $ok = false; + } + + $jwtUserId = $payload['userId'] ?? ''; + if (empty($jwtUserId)) { + $ok = false; + } else { + $user = $dbForPlatform->getDocument('users', $jwtUserId); + if ($user->isEmpty()) { + $ok = false; + } + } + + $jwtSessionId = $payload['sessionId'] ?? ''; + if (empty($jwtSessionId)) { + $ok = false; + } else { + if (empty($user->find('$id', $jwtSessionId, 'sessions'))) { + $ok = false; + } + } + + // TODO: Ensure user has access to projectId + } + + if ($ok === false) { + $url = (System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https') . "://" . System::getEnv('_APP_DOMAIN'); + $response + ->addHeader('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') + ->addHeader('Pragma', 'no-cache') + ->redirect($url . '/console/auth/preview?' + . \http_build_query([ + 'projectId' => $projectId, + 'origin' => $protocol . '://' . $host, + 'path' => $path + ])); + return true; + } + } + $body = $swooleRequest->getContent() ?? ''; $method = $swooleRequest->server['request_method']; diff --git a/src/Appwrite/Auth/Auth.php b/src/Appwrite/Auth/Auth.php index 8555d5cb00..9af5045fa4 100644 --- a/src/Appwrite/Auth/Auth.php +++ b/src/Appwrite/Auth/Auth.php @@ -103,6 +103,11 @@ class Auth */ public static $cookieName = 'a_session'; + /** + * @var string + */ + public static $cookieNamePreview = 'a_jwt_console'; + /** * User Unique ID. * From feffab137f52adc380abc58dd77d92c9af0e02b4 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Mon, 17 Feb 2025 14:00:30 +0530 Subject: [PATCH 04/31] Resolve merge conflicts --- composer.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/composer.lock b/composer.lock index c0bb98367f..cc51254ca3 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "5473f6b65d54314228e69b6bed489d78", + "content-hash": "44bd3c613083a66e4f446a27a94d6977", "packages": [ { "name": "adhocore/jwt", @@ -3777,12 +3777,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/detector.git", - "reference": "09512d06b4b3a1a4acca2403ea9c22f03975d79e" + "reference": "0f1d84c2a1f5c26e1a8decee24dabdb8e1d6ce1a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/detector/zipball/09512d06b4b3a1a4acca2403ea9c22f03975d79e", - "reference": "09512d06b4b3a1a4acca2403ea9c22f03975d79e", + "url": "https://api.github.com/repos/utopia-php/detector/zipball/0f1d84c2a1f5c26e1a8decee24dabdb8e1d6ce1a", + "reference": "0f1d84c2a1f5c26e1a8decee24dabdb8e1d6ce1a", "shasum": "" }, "require": { @@ -3832,7 +3832,7 @@ "source": "https://github.com/utopia-php/detector/tree/feat-pseudocode-draft2", "issues": "https://github.com/utopia-php/detector/issues" }, - "time": "2025-02-06T13:36:29+00:00" + "time": "2025-02-17T08:26:03+00:00" }, { "name": "utopia-php/domains", From 76b154bfce8cb30f6a677626ac5b0a2d75951265 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Mon, 17 Feb 2025 17:54:13 +0530 Subject: [PATCH 05/31] Add framework detection to list repos --- app/controllers/api/vcs.php | 121 ++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 45 deletions(-) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index dc64f34a7e..dcfa412170 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -725,11 +725,11 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') )) ->param('installationId', '', new Text(256), 'Installation Id') ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->param('type', '', new WhiteList(['runtime', 'framework']), 'Detector type. Must be one of the following: runtime, framework', true) ->inject('gitHub') ->inject('response') - ->inject('project') ->inject('dbForPlatform') - ->action(function (string $installationId, string $search, GitHub $github, Response $response, Document $project, Database $dbForPlatform) { + ->action(function (string $installationId, string $search, string $type, GitHub $github, Response $response, Database $dbForPlatform) { if (empty($search)) { $search = ""; } @@ -759,54 +759,85 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') return $repo; }, $repos); - $repos = batch(\array_map(function ($repo) use ($github) { - return function () use ($repo, $github) { - try { - $files = $github->listRepositoryContents($repo['organization'], $repo['name'], ''); - $files = \array_column($files, 'name'); - $languages = $github->listRepositoryLanguages($repo['organization'], $repo['name']); + $repos = batch(\array_map(function ($repo, $type) use ($github) { + return function () use ($repo, $type, $github) { + $files = $github->listRepositoryContents($repo['organization'], $repo['name'], ''); + $files = \array_column($files, 'name'); + $languages = $github->listRepositoryLanguages($repo['organization'], $repo['name']); + if ($type === 'framework') { + try { + $frameworkDetector = new Framework($files, 'npm'); + $frameworkDetector + ->addOption(new Flutter()) + ->addOption(new Nuxt()) + ->addOption(new Astro()) + ->addOption(new Remix()) + ->addOption(new SvelteKit()) + ->addOption(new NextJs()); - $strategies = [ - new Strategy(Strategy::FILEMATCH), - new Strategy(Strategy::LANGUAGES), - new Strategy(Strategy::EXTENSION), - ]; + $detectedFramework = $frameworkDetector->detect(); - foreach ($strategies as $strategy) { - $runtimeDetector = new Runtime($strategy === Strategy::LANGUAGES ? $languages : $files, $strategy, 'npm'); - $runtimeDetector - ->addOption(new Node()) - ->addOption(new Bun()) - ->addOption(new Deno()) - ->addOption(new PHP()) - ->addOption(new Python()) - ->addOption(new Dart()) - ->addOption(new Swift()) - ->addOption(new Ruby()) - ->addOption(new Java()) - ->addOption(new CPP()) - ->addOption(new Dotnet()); - - $detectedRuntime = $runtimeDetector->detect(); - - if ($detectedRuntime) { - $runtime = $detectedRuntime->getName(); - break; + if ($detectedFramework) { + $framework = $detectedFramework->getName(); } - } - if (!empty($runtime)) { - $runtimes = Config::getParam('runtimes'); - $runtimeDetail = \array_reverse(\array_filter(\array_keys($runtimes), function ($key) use ($runtime, $runtimes) { - return $runtimes[$key]['key'] === $runtime; - }))[0] ?? ''; - $repo['runtime'] = $runtimeDetail; - } else { - throw new Exception("Runtime not detected"); + if (!empty($framework)) { + $frameworks = Config::getParam('frameworks'); + $frameworkDetail = \array_reverse(\array_filter(\array_keys($frameworks), function ($key) use ($framework, $frameworks) { + return $frameworks[$key]['key'] === $framework; + }))[0] ?? ''; + $detection['framework'] = $frameworkDetail; + } else { + throw new Exception("Framework not detected"); + } + } catch (Throwable $error) { + $repo['framework'] = ""; + Console::warning("Framework not detected for " . $repo['organization'] . "/" . $repo['name']); + } + } else { + try { + $strategies = [ + new Strategy(Strategy::FILEMATCH), + new Strategy(Strategy::LANGUAGES), + new Strategy(Strategy::EXTENSION), + ]; + + foreach ($strategies as $strategy) { + $runtimeDetector = new Runtime($strategy === Strategy::LANGUAGES ? $languages : $files, $strategy, 'npm'); + $runtimeDetector + ->addOption(new Node()) + ->addOption(new Bun()) + ->addOption(new Deno()) + ->addOption(new PHP()) + ->addOption(new Python()) + ->addOption(new Dart()) + ->addOption(new Swift()) + ->addOption(new Ruby()) + ->addOption(new Java()) + ->addOption(new CPP()) + ->addOption(new Dotnet()); + + $detectedRuntime = $runtimeDetector->detect(); + + if ($detectedRuntime) { + $runtime = $detectedRuntime->getName(); + break; + } + } + + if (!empty($runtime)) { + $runtimes = Config::getParam('runtimes'); + $runtimeDetail = \array_reverse(\array_filter(\array_keys($runtimes), function ($key) use ($runtime, $runtimes) { + return $runtimes[$key]['key'] === $runtime; + }))[0] ?? ''; + $repo['runtime'] = $runtimeDetail; + } else { + throw new Exception("Runtime not detected"); + } + } catch (Throwable $error) { + $repo['runtime'] = ""; + Console::warning("Runtime not detected for " . $repo['organization'] . "/" . $repo['name']); } - } catch (Throwable $error) { - $repo['runtime'] = ""; - Console::warning("Runtime not detected for " . $repo['organization'] . "/" . $repo['name']); } return $repo; }; From c0b574fe9beaa0a9cced0b9421cd463970381305 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Wed, 19 Feb 2025 12:19:41 +0530 Subject: [PATCH 06/31] Update detector library version --- composer.lock | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/composer.lock b/composer.lock index cc51254ca3..00f755f400 100644 --- a/composer.lock +++ b/composer.lock @@ -3777,12 +3777,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/detector.git", - "reference": "0f1d84c2a1f5c26e1a8decee24dabdb8e1d6ce1a" + "reference": "69958f4d765c5338a703f3e3638a3249ec227be8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/detector/zipball/0f1d84c2a1f5c26e1a8decee24dabdb8e1d6ce1a", - "reference": "0f1d84c2a1f5c26e1a8decee24dabdb8e1d6ce1a", + "url": "https://api.github.com/repos/utopia-php/detector/zipball/69958f4d765c5338a703f3e3638a3249ec227be8", + "reference": "69958f4d765c5338a703f3e3638a3249ec227be8", "shasum": "" }, "require": { @@ -3832,7 +3832,7 @@ "source": "https://github.com/utopia-php/detector/tree/feat-pseudocode-draft2", "issues": "https://github.com/utopia-php/detector/issues" }, - "time": "2025-02-17T08:26:03+00:00" + "time": "2025-02-18T15:22:56+00:00" }, { "name": "utopia-php/domains", @@ -5433,16 +5433,16 @@ }, { "name": "laravel/pint", - "version": "v1.20.0", + "version": "v1.21.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "53072e8ea22213a7ed168a8a15b96fbb8b82d44b" + "reference": "531fa0871fbde719c51b12afa3a443b8f4e4b425" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/53072e8ea22213a7ed168a8a15b96fbb8b82d44b", - "reference": "53072e8ea22213a7ed168a8a15b96fbb8b82d44b", + "url": "https://api.github.com/repos/laravel/pint/zipball/531fa0871fbde719c51b12afa3a443b8f4e4b425", + "reference": "531fa0871fbde719c51b12afa3a443b8f4e4b425", "shasum": "" }, "require": { @@ -5450,15 +5450,15 @@ "ext-mbstring": "*", "ext-tokenizer": "*", "ext-xml": "*", - "php": "^8.1.0" + "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.66.0", - "illuminate/view": "^10.48.25", - "larastan/larastan": "^2.9.12", - "laravel-zero/framework": "^10.48.25", + "friendsofphp/php-cs-fixer": "^3.68.5", + "illuminate/view": "^11.42.0", + "larastan/larastan": "^3.0.4", + "laravel-zero/framework": "^11.36.1", "mockery/mockery": "^1.6.12", - "nunomaduro/termwind": "^1.17.0", + "nunomaduro/termwind": "^2.3", "pestphp/pest": "^2.36.0" }, "bin": [ @@ -5495,7 +5495,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-01-14T16:20:53+00:00" + "time": "2025-02-18T03:18:57+00:00" }, { "name": "matthiasmullie/minify", @@ -6253,16 +6253,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "2.0.1", + "version": "2.0.2", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "72e51f7c32c5aef7c8b462195b8c599b11199893" + "reference": "51087f87dcce2663e1fed4dfd4e56eccd580297e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/72e51f7c32c5aef7c8b462195b8c599b11199893", - "reference": "72e51f7c32c5aef7c8b462195b8c599b11199893", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/51087f87dcce2663e1fed4dfd4e56eccd580297e", + "reference": "51087f87dcce2663e1fed4dfd4e56eccd580297e", "shasum": "" }, "require": { @@ -6294,9 +6294,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/2.0.1" + "source": "https://github.com/phpstan/phpdoc-parser/tree/2.0.2" }, - "time": "2025-02-13T12:25:43+00:00" + "time": "2025-02-17T20:25:51+00:00" }, { "name": "phpstan/phpstan", From a7c1bd94ed19a9f3b0b874a11a8032a21c4c4ce8 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Thu, 20 Feb 2025 12:54:26 +0530 Subject: [PATCH 07/31] Add framework detection to list repos --- app/controllers/api/vcs.php | 79 +++++++++---------- .../Response/Model/ProviderRepository.php | 6 ++ 2 files changed, 45 insertions(+), 40 deletions(-) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index dcfa412170..67833e5699 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -557,7 +557,7 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories/:pro ]), Response::MODEL_VCS_CONTENT_LIST); }); -App::post('/v1/vcs/github/installations/:installationId/providerRepositories/:providerRepositoryId/detections') +App::post('/v1/vcs/github/installations/:installationId/detections') ->alias('/v1/vcs/github/installations/:installationId/providerRepositories/:providerRepositoryId/detection') ->desc('Detect runtime and framework settings from source code') ->groups(['api', 'vcs']) @@ -580,12 +580,12 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories/:pr )) ->param('installationId', '', new Text(256), 'Installation Id') ->param('providerRepositoryId', '', new Text(256), 'Repository Id') + ->param('type', '', new WhiteList(['runtime', 'framework']), 'Detector type. Must be one of the following: runtime, framework') ->param('providerRootDirectory', '', new Text(256, 0), 'Path to Root Directory', true) - ->param('type', '', new WhiteList(['runtime', 'framework']), 'Detector type. Must be one of the following: runtime, framework', true) ->inject('gitHub') ->inject('response') ->inject('dbForPlatform') - ->action(function (string $installationId, string $providerRepositoryId, string $providerRootDirectory, string $type, GitHub $github, Response $response, Database $dbForPlatform) { + ->action(function (string $installationId, string $providerRepositoryId, string $type, string $providerRootDirectory, GitHub $github, Response $response, Database $dbForPlatform) { $installation = $dbForPlatform->getDocument('installations', $installationId); if ($installation->isEmpty()) { @@ -618,17 +618,15 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories/:pr ->addOption(new NPM()); $detectedPackager = $detector->detect(); - $packagerName = $detectedPackager ? $detectedPackager->getName() : 'npm'; - - $detection = []; + $packagerName = !\is_null($detectedPackager) ? $detectedPackager->getName() : 'npm'; if ($type === 'framework') { - $detection = [ + $detection = new Document([ 'framework' => '', 'installCommand' => '', 'buildCommand' => '', 'outputDirectory' => '', - ]; + ]); $frameworkDetector = new Framework($files, $packagerName); $frameworkDetector @@ -643,26 +641,29 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories/:pr if ($detectedFramework) { $framework = $detectedFramework->getName(); - $detection['installCommand'] = $detectedFramework->getInstallCommand(); - $detection['buildCommand'] = $detectedFramework->getBuildCommand(); - $detection['outputDirectory'] = $detectedFramework->getOutputDirectory(); + $detection->setAttribute('installCommand', $detectedFramework->getInstallCommand()); + $detection->setAttribute('buildCommand', $detectedFramework->getBuildCommand()); + $detection->setAttribute('outputDirectory', $detectedFramework->getOutputDirectory()); + } else { + $framework = 'other'; + $detection->setAttribute('installCommand', ''); + $detection->setAttribute('buildCommand', ''); + $detection->setAttribute('outputDirectory', './'); } - if (!empty($framework)) { - $frameworks = Config::getParam('frameworks'); - $frameworkDetail = \array_reverse(\array_filter(\array_keys($frameworks), function ($key) use ($framework, $frameworks) { - return $frameworks[$key]['key'] === $framework; - }))[0] ?? ''; - $detection['framework'] = $frameworkDetail; - } + $frameworks = Config::getParam('frameworks'); + $frameworkDetail = \array_reverse(\array_filter(\array_keys($frameworks), function ($key) use ($framework, $frameworks) { + return $frameworks[$key]['key'] === $framework; + }))[0] ?? ''; + $detection->setAttribute('framework', $frameworkDetail); - $response->dynamic(new Document($detection), Response::MODEL_FRAMEWORK_DETECTION); + $response->dynamic($detection, Response::MODEL_FRAMEWORK_DETECTION); } else { - $detection = [ + $detection = new Document([ 'runtime' => '', 'commands' => '', 'entrypoint' => '', - ]; + ]); $strategies = [ new Strategy(Strategy::FILEMATCH), @@ -688,8 +689,8 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories/:pr $detectedRuntime = $runtimeDetector->detect(); if ($detectedRuntime) { - $detection['commands'] = $detectedRuntime->getCommands(); - $detection['entrypoint'] = $detectedRuntime->getEntrypoint(); + $detection->setAttribute('commands', $detectedRuntime->getCommands()); + $detection->setAttribute('entrypoint', $detectedRuntime->getEntrypoint()); $runtime = $detectedRuntime->getName(); break; } @@ -700,10 +701,12 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories/:pr $runtimeDetail = \array_reverse(\array_filter(\array_keys($runtimes), function ($key) use ($runtime, $runtimes) { return $runtimes[$key]['key'] === $runtime; }))[0] ?? ''; - $detection['runtime'] = $runtimeDetail; + $detection->setAttribute('runtime', $runtimeDetail); + } else { + throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Runtime not detected.'); } - $response->dynamic(new Document($detection), Response::MODEL_RUNTIME_DETECTION); + $response->dynamic($detection, Response::MODEL_RUNTIME_DETECTION); } }); @@ -724,12 +727,12 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') ] )) ->param('installationId', '', new Text(256), 'Installation Id') + ->param('type', '', new WhiteList(['runtime', 'framework']), 'Detector type. Must be one of the following: runtime, framework') ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) - ->param('type', '', new WhiteList(['runtime', 'framework']), 'Detector type. Must be one of the following: runtime, framework', true) ->inject('gitHub') ->inject('response') ->inject('dbForPlatform') - ->action(function (string $installationId, string $search, string $type, GitHub $github, Response $response, Database $dbForPlatform) { + ->action(function (string $installationId, string $type, string $search, GitHub $github, Response $response, Database $dbForPlatform) { if (empty($search)) { $search = ""; } @@ -759,14 +762,14 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') return $repo; }, $repos); - $repos = batch(\array_map(function ($repo, $type) use ($github) { + $repos = batch(\array_map(function ($repo) use ($type, $github) { return function () use ($repo, $type, $github) { $files = $github->listRepositoryContents($repo['organization'], $repo['name'], ''); $files = \array_column($files, 'name'); $languages = $github->listRepositoryLanguages($repo['organization'], $repo['name']); if ($type === 'framework') { try { - $frameworkDetector = new Framework($files, 'npm'); + $frameworkDetector = new Framework($files); $frameworkDetector ->addOption(new Flutter()) ->addOption(new Nuxt()) @@ -779,17 +782,15 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') if ($detectedFramework) { $framework = $detectedFramework->getName(); + } else { + $framework = 'other'; } - if (!empty($framework)) { - $frameworks = Config::getParam('frameworks'); - $frameworkDetail = \array_reverse(\array_filter(\array_keys($frameworks), function ($key) use ($framework, $frameworks) { - return $frameworks[$key]['key'] === $framework; - }))[0] ?? ''; - $detection['framework'] = $frameworkDetail; - } else { - throw new Exception("Framework not detected"); - } + $frameworks = Config::getParam('frameworks'); + $frameworkDetail = \array_reverse(\array_filter(\array_keys($frameworks), function ($key) use ($framework, $frameworks) { + return $frameworks[$key]['key'] === $framework; + }))[0] ?? ''; + $repo['framework'] = $frameworkDetail; } catch (Throwable $error) { $repo['framework'] = ""; Console::warning("Framework not detected for " . $repo['organization'] . "/" . $repo['name']); @@ -831,8 +832,6 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') return $runtimes[$key]['key'] === $runtime; }))[0] ?? ''; $repo['runtime'] = $runtimeDetail; - } else { - throw new Exception("Runtime not detected"); } } catch (Throwable $error) { $repo['runtime'] = ""; diff --git a/src/Appwrite/Utopia/Response/Model/ProviderRepository.php b/src/Appwrite/Utopia/Response/Model/ProviderRepository.php index c133ae3164..ec3d879f3d 100644 --- a/src/Appwrite/Utopia/Response/Model/ProviderRepository.php +++ b/src/Appwrite/Utopia/Response/Model/ProviderRepository.php @@ -47,6 +47,12 @@ class ProviderRepository extends Model 'default' => '', 'example' => 'node', ]) + ->addRule('framework', [ + 'type' => self::TYPE_STRING, + 'description' => 'Auto-detected framework suggestion. Empty if getting response of getFramework().', + 'default' => '', + 'example' => 'nextjs', + ]) ->addRule('pushedAt', [ 'type' => self::TYPE_DATETIME, 'description' => 'Last commit date in ISO 8601 format.', From 1394b4349c840bcb92fbaafaaf2fc2934e1f2ddf Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:22:15 +0530 Subject: [PATCH 08/31] Separate response models for list repos --- app/controllers/api/vcs.php | 18 +++-- src/Appwrite/Utopia/Response.php | 15 ++-- ...ry.php => FrameworkProviderRepository.php} | 12 +-- .../Model/RuntimeProviderRepository.php | 78 +++++++++++++++++++ 4 files changed, 102 insertions(+), 21 deletions(-) rename src/Appwrite/Utopia/Response/Model/{ProviderRepository.php => FrameworkProviderRepository.php} (85%) create mode 100644 src/Appwrite/Utopia/Response/Model/RuntimeProviderRepository.php diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 67833e5699..c6d1b25c5a 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -722,7 +722,11 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, - model: Response::MODEL_PROVIDER_REPOSITORY_LIST, + model: Response::MODEL_RUNTIME_PROVIDER_REPOSITORY_LIST, + ), + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_FRAMEWORK_PROVIDER_REPOSITORY_LIST, ) ] )) @@ -847,9 +851,9 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') }, $repos); $response->dynamic(new Document([ - 'providerRepositories' => $repos, + $type === 'framework' ? 'frameworkProviderRepositories' : 'runtimeProviderRepositories' => $repos, 'total' => \count($repos), - ]), Response::MODEL_PROVIDER_REPOSITORY_LIST); + ]), ($type === 'framework') ? Response::MODEL_FRAMEWORK_PROVIDER_REPOSITORY_LIST : Response::MODEL_RUNTIME_PROVIDER_REPOSITORY_LIST); }); App::post('/v1/vcs/github/installations/:installationId/providerRepositories') @@ -864,7 +868,7 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories') responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, - model: Response::MODEL_PROVIDER_REPOSITORY, + model: Response::MODEL_RUNTIME_PROVIDER_REPOSITORY, ) ] )) @@ -961,7 +965,7 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories') $repository['organization'] = $installation->getAttribute('organization', ''); $repository['provider'] = $installation->getAttribute('provider', ''); - $response->dynamic(new Document($repository), Response::MODEL_PROVIDER_REPOSITORY); + $response->dynamic(new Document($repository), Response::MODEL_RUNTIME_PROVIDER_REPOSITORY); }); App::get('/v1/vcs/github/installations/:installationId/providerRepositories/:providerRepositoryId') @@ -976,7 +980,7 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories/:pro responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, - model: Response::MODEL_PROVIDER_REPOSITORY, + model: Response::MODEL_RUNTIME_PROVIDER_REPOSITORY, ) ] )) @@ -1015,7 +1019,7 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories/:pro $repository['organization'] = $installation->getAttribute('organization', ''); $repository['provider'] = $installation->getAttribute('provider', ''); - $response->dynamic(new Document($repository), Response::MODEL_PROVIDER_REPOSITORY); + $response->dynamic(new Document($repository), Response::MODEL_RUNTIME_PROVIDER_REPOSITORY); }); App::get('/v1/vcs/github/installations/:installationId/providerRepositories/:providerRepositoryId/branches') diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index e9c3540672..81b6989df2 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -46,6 +46,7 @@ use Appwrite\Utopia\Response\Model\File; use Appwrite\Utopia\Response\Model\Framework; use Appwrite\Utopia\Response\Model\FrameworkAdapter; use Appwrite\Utopia\Response\Model\FrameworkDetection; +use Appwrite\Utopia\Response\Model\FrameworkProviderRepository; use Appwrite\Utopia\Response\Model\Func; use Appwrite\Utopia\Response\Model\Headers; use Appwrite\Utopia\Response\Model\HealthAntivirus; @@ -82,10 +83,10 @@ use Appwrite\Utopia\Response\Model\Platform; use Appwrite\Utopia\Response\Model\Preferences; use Appwrite\Utopia\Response\Model\Project; use Appwrite\Utopia\Response\Model\Provider; -use Appwrite\Utopia\Response\Model\ProviderRepository; use Appwrite\Utopia\Response\Model\Rule; use Appwrite\Utopia\Response\Model\Runtime; use Appwrite\Utopia\Response\Model\RuntimeDetection; +use Appwrite\Utopia\Response\Model\RuntimeProviderRepository; use Appwrite\Utopia\Response\Model\Session; use Appwrite\Utopia\Response\Model\Site; use Appwrite\Utopia\Response\Model\Specification; @@ -246,8 +247,10 @@ class Response extends SwooleResponse // VCS public const MODEL_INSTALLATION = 'installation'; public const MODEL_INSTALLATION_LIST = 'installationList'; - public const MODEL_PROVIDER_REPOSITORY = 'providerRepository'; - public const MODEL_PROVIDER_REPOSITORY_LIST = 'providerRepositoryList'; + public const MODEL_FRAMEWORK_PROVIDER_REPOSITORY = 'frameworkProviderRepository'; + public const MODEL_FRAMEWORK_PROVIDER_REPOSITORY_LIST = 'frameworkProviderRepositoryList'; + public const MODEL_RUNTIME_PROVIDER_REPOSITORY = 'runtimeProviderRepository'; + public const MODEL_RUNTIME_PROVIDER_REPOSITORY_LIST = 'runtimeProviderRepositoryList'; public const MODEL_BRANCH = 'branch'; public const MODEL_BRANCH_LIST = 'branchList'; public const MODEL_FRAMEWORK_DETECTION = 'frameworkDetection'; @@ -377,7 +380,8 @@ class Response extends SwooleResponse ->setModel(new BaseList('Functions List', self::MODEL_FUNCTION_LIST, 'functions', self::MODEL_FUNCTION)) ->setModel(new BaseList('Function Templates List', self::MODEL_TEMPLATE_FUNCTION_LIST, 'templates', self::MODEL_TEMPLATE_FUNCTION)) ->setModel(new BaseList('Installations List', self::MODEL_INSTALLATION_LIST, 'installations', self::MODEL_INSTALLATION)) - ->setModel(new BaseList('Provider Repositories List', self::MODEL_PROVIDER_REPOSITORY_LIST, 'providerRepositories', self::MODEL_PROVIDER_REPOSITORY)) + ->setModel(new BaseList('Framework Provider Repositories List', self::MODEL_FRAMEWORK_PROVIDER_REPOSITORY_LIST, 'frameworkProviderRepositories', self::MODEL_FRAMEWORK_PROVIDER_REPOSITORY)) + ->setModel(new BaseList('Runtime Provider Repositories List', self::MODEL_RUNTIME_PROVIDER_REPOSITORY_LIST, 'runtimeProviderRepositories', self::MODEL_RUNTIME_PROVIDER_REPOSITORY)) ->setModel(new BaseList('Branches List', self::MODEL_BRANCH_LIST, 'branches', self::MODEL_BRANCH)) ->setModel(new BaseList('Frameworks List', self::MODEL_FRAMEWORK_LIST, 'frameworks', self::MODEL_FRAMEWORK)) ->setModel(new BaseList('Runtimes List', self::MODEL_RUNTIME_LIST, 'runtimes', self::MODEL_RUNTIME)) @@ -454,7 +458,8 @@ class Response extends SwooleResponse ->setModel(new TemplateRuntime()) ->setModel(new TemplateVariable()) ->setModel(new Installation()) - ->setModel(new ProviderRepository()) + ->setModel(new FrameworkProviderRepository()) + ->setModel(new RuntimeProviderRepository()) ->setModel(new FrameworkDetection()) ->setModel(new RuntimeDetection()) ->setModel(new VcsContent()) diff --git a/src/Appwrite/Utopia/Response/Model/ProviderRepository.php b/src/Appwrite/Utopia/Response/Model/FrameworkProviderRepository.php similarity index 85% rename from src/Appwrite/Utopia/Response/Model/ProviderRepository.php rename to src/Appwrite/Utopia/Response/Model/FrameworkProviderRepository.php index ec3d879f3d..305fb28852 100644 --- a/src/Appwrite/Utopia/Response/Model/ProviderRepository.php +++ b/src/Appwrite/Utopia/Response/Model/FrameworkProviderRepository.php @@ -5,7 +5,7 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; -class ProviderRepository extends Model +class FrameworkProviderRepository extends Model { public function __construct() { @@ -41,12 +41,6 @@ class ProviderRepository extends Model 'default' => false, 'example' => true, ]) - ->addRule('runtime', [ - 'type' => self::TYPE_STRING, - 'description' => 'Auto-detected runtime suggestion. Empty if getting response of getRuntime().', - 'default' => '', - 'example' => 'node', - ]) ->addRule('framework', [ 'type' => self::TYPE_STRING, 'description' => 'Auto-detected framework suggestion. Empty if getting response of getFramework().', @@ -69,7 +63,7 @@ class ProviderRepository extends Model */ public function getName(): string { - return 'ProviderRepository'; + return 'FrameworkProviderRepository'; } /** @@ -79,6 +73,6 @@ class ProviderRepository extends Model */ public function getType(): string { - return Response::MODEL_PROVIDER_REPOSITORY; + return Response::MODEL_FRAMEWORK_PROVIDER_REPOSITORY; } } diff --git a/src/Appwrite/Utopia/Response/Model/RuntimeProviderRepository.php b/src/Appwrite/Utopia/Response/Model/RuntimeProviderRepository.php new file mode 100644 index 0000000000..cb8d163b9a --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/RuntimeProviderRepository.php @@ -0,0 +1,78 @@ +addRule('id', [ + 'type' => self::TYPE_STRING, + 'description' => 'VCS (Version Control System) repository ID.', + 'default' => '', + 'example' => '5e5ea5c16897e', + ]) + ->addRule('name', [ + 'type' => self::TYPE_STRING, + 'description' => 'VCS (Version Control System) repository name.', + 'default' => '', + 'example' => 'appwrite', + ]) + ->addRule('organization', [ + 'type' => self::TYPE_STRING, + 'description' => 'VCS (Version Control System) organization name', + 'default' => [], + 'example' => 'appwrite', + 'array' => false, + ]) + ->addRule('provider', [ + 'type' => self::TYPE_STRING, + 'description' => 'VCS (Version Control System) provider name.', + 'default' => '', + 'example' => 'github', + ]) + ->addRule('private', [ + 'type' => self::TYPE_BOOLEAN, + 'description' => 'Is VCS (Version Control System) repository private?', + 'default' => false, + 'example' => true, + ]) + ->addRule('runtime', [ + 'type' => self::TYPE_STRING, + 'description' => 'Auto-detected runtime suggestion. Empty if getting response of getRuntime().', + 'default' => '', + 'example' => 'node', + ]) + ->addRule('pushedAt', [ + 'type' => self::TYPE_DATETIME, + 'description' => 'Last commit date in ISO 8601 format.', + 'default' => APP_DATABASE_ATTRIBUTE_DATETIME, + 'example' => APP_DATABASE_ATTRIBUTE_DATETIME, + 'array' => false, + ]); + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'RuntimeProviderRepository'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_RUNTIME_PROVIDER_REPOSITORY; + } +} From f830918c7cce2be697f8e8b1f0c9b04a555583a3 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Tue, 25 Feb 2025 16:19:30 +0530 Subject: [PATCH 09/31] Add logic to detect rendering strategy and fallback file --- docker-compose.yml | 2 +- .../Modules/Functions/Workers/Builds.php | 27 ++++++++++++++++++- src/Executor/Executor.php | 25 +++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6dec6e3918..f03e4097b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -952,7 +952,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.7.4 + image: openruntimes-executor-2 restart: unless-stopped networks: - appwrite diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 0a4907fdb7..e55375e12f 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -24,6 +24,9 @@ use Utopia\Database\Exception\Structure; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; +use Utopia\Detector\Detection\Rendering\SSG; +use Utopia\Detector\Detection\Rendering\SSR; +use Utopia\Detector\Detector\Rendering; use Utopia\Logger\Log; use Utopia\Platform\Action; use Utopia\Queue\Message; @@ -590,7 +593,7 @@ class Builds extends Action cpus: $cpus, memory: $memory, timeout: $timeout, - remove: true, + remove: false, entrypoint: $deployment->getAttribute('entrypoint', 'package.json'), // TODO: change this later so that sites don't need to have an entrypoint destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", variables: $vars, @@ -691,6 +694,28 @@ class Builds extends Action throw new \Exception('Build size should be less than ' . number_format($buildSizeLimit / 1048576, 2) . ' MBs.'); } + $listFilesCommand = "cd /usr/local/build/" . $resource->getAttribute('outputDirectory') . " && find . -name 'node_modules' -prune -o -type f -print"; + $response = $executor->executeCommand( + deploymentId: $deployment->getId(), + projectId: $project->getId(), + command: $listFilesCommand, + timeout: 60 + ); + + $files = array_map('trim', array_filter(explode("\n", $response['output']))); + + $detector = new Rendering($files, $resource->getAttribute('framework')); + $detector + ->addOption(new SSR()) + ->addOption(new SSG()); + $detectedRenderingStrategy = $detector->detect(); + + $renderingStrategy = $detectedRenderingStrategy->getName(); + $fallbackFile = $detectedRenderingStrategy->getFallbackFile(); + + $resource->setAttribute('adapter', $renderingStrategy); + $resource->setAttribute('fallbackFile', $fallbackFile); + /** Update the build document */ $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); $build->setAttribute('endTime', $endTime); diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index 0c6b5a14ff..a105503088 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -249,6 +249,31 @@ class Executor return $response['body']; } + public function executeCommand( + string $deploymentId, + string $projectId, + string $command, + int $timeout + ) { + $runtimeId = "$projectId-$deploymentId-build"; + $route = "/runtimes/$runtimeId/commands"; + + $params = [ + 'command' => $command, + 'timeout' => $timeout + ]; + + $response = $this->call(self::METHOD_POST, $route, [ 'x-opr-runtime-id' => $runtimeId ], $params, true, $timeout); + + $status = $response['headers']['status-code']; + if ($status >= 400) { + $message = \is_string($response['body']) ? $response['body'] : $response['body']['message']; + throw new \Exception($message, $status); + } + + return $response['body']; + } + /** * Call * From 8b8f632abe029c47d58f2dc530f169ccc496f72e Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Mon, 3 Mar 2025 13:54:07 +0530 Subject: [PATCH 10/31] Add providerRepository model and extend it for framework and runtime --- app/config/errors.php | 7 +- app/controllers/api/vcs.php | 112 ++++++++++-------- docker-compose.yml | 2 +- src/Appwrite/Extend/Exception.php | 1 + src/Appwrite/Utopia/Response.php | 4 + .../Model/FrameworkProviderRepository.php | 56 ++------- .../Response/Model/ProviderRepository.php | 72 +++++++++++ .../Model/RuntimeProviderRepository.php | 56 ++------- 8 files changed, 167 insertions(+), 143 deletions(-) create mode 100644 src/Appwrite/Utopia/Response/Model/ProviderRepository.php diff --git a/app/config/errors.php b/app/config/errors.php index e41f663168..73cb9d38cf 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -527,7 +527,7 @@ return [ 'code' => 404, ], Exception::FUNCTION_ENTRYPOINT_MISSING => [ - 'name' => Exception::FUNCTION_RUNTIME_UNSUPPORTED, + 'name' => Exception::FUNCTION_ENTRYPOINT_MISSING, 'description' => 'Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 'code' => 404, ], @@ -541,6 +541,11 @@ return [ 'description' => 'Function Template with the requested ID could not be found.', 'code' => 404, ], + Exception::FUNCTION_RUNTIME_NOT_DETECTED => [ + 'name' => Exception::FUNCTION_RUNTIME_NOT_DETECTED, + 'description' => 'Function runtime could not be detected.', + 'code' => 400, + ], /** Sites */ Exception::SITE_NOT_FOUND => [ diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 481171be18..62d6a3363d 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -14,7 +14,6 @@ use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Appwrite\Vcs\Comment; use Utopia\App; -use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; @@ -696,16 +695,18 @@ App::post('/v1/vcs/github/installations/:installationId/detections') $framework = 'other'; $detection->setAttribute('installCommand', ''); $detection->setAttribute('buildCommand', ''); - $detection->setAttribute('outputDirectory', './'); + $detection->setAttribute('outputDirectory', ''); } $frameworks = Config::getParam('frameworks'); - $frameworkDetail = \array_reverse(\array_filter(\array_keys($frameworks), function ($key) use ($framework, $frameworks) { - return $frameworks[$key]['key'] === $framework; - }))[0] ?? ''; + $frameworkDetail = ''; + foreach ($frameworks as $key => $config) { + if ($config['key'] === $framework) { + $frameworkDetail = $key; + break; + } + } $detection->setAttribute('framework', $frameworkDetail); - - $response->dynamic($detection, Response::MODEL_FRAMEWORK_DETECTION); } else { $detection = new Document([ 'runtime' => '', @@ -746,16 +747,19 @@ App::post('/v1/vcs/github/installations/:installationId/detections') if (!empty($runtime)) { $runtimes = Config::getParam('runtimes'); - $runtimeDetail = \array_reverse(\array_filter(\array_keys($runtimes), function ($key) use ($runtime, $runtimes) { - return $runtimes[$key]['key'] === $runtime; - }))[0] ?? ''; + $runtimeDetail = ''; + foreach ($runtimes as $key => $config) { + if ($config['key'] === $runtime) { + $runtimeDetail = $key; + break; + } + } $detection->setAttribute('runtime', $runtimeDetail); } else { - throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Runtime not detected.'); + throw new Exception(Exception::FUNCTION_RUNTIME_NOT_DETECTED); } - - $response->dynamic($detection, Response::MODEL_RUNTIME_DETECTION); } + $response->dynamic($detection, $type === 'framework' ? Response::MODEL_FRAMEWORK_DETECTION : Response::MODEL_RUNTIME_DETECTION); }); App::get('/v1/vcs/github/installations/:installationId/providerRepositories') @@ -818,37 +822,47 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') return function () use ($repo, $type, $github) { $files = $github->listRepositoryContents($repo['organization'], $repo['name'], ''); $files = \array_column($files, 'name'); - $languages = $github->listRepositoryLanguages($repo['organization'], $repo['name']); + + $detector = new Packager($files); + $detector + ->addOption(new Yarn()) + ->addOption(new PNPM()) + ->addOption(new NPM()); + $detectedPackager = $detector->detect(); + + $packagerName = !\is_null($detectedPackager) ? $detectedPackager->getName() : 'npm'; + if ($type === 'framework') { - try { - $frameworkDetector = new Framework($files); - $frameworkDetector - ->addOption(new Flutter()) - ->addOption(new Nuxt()) - ->addOption(new Astro()) - ->addOption(new Remix()) - ->addOption(new SvelteKit()) - ->addOption(new NextJs()); + $frameworkDetector = new Framework($files, $packagerName); + $frameworkDetector + ->addOption(new Flutter()) + ->addOption(new Nuxt()) + ->addOption(new Astro()) + ->addOption(new Remix()) + ->addOption(new SvelteKit()) + ->addOption(new NextJs()); - $detectedFramework = $frameworkDetector->detect(); + $detectedFramework = $frameworkDetector->detect(); - if ($detectedFramework) { - $framework = $detectedFramework->getName(); - } else { - $framework = 'other'; - } - - $frameworks = Config::getParam('frameworks'); - $frameworkDetail = \array_reverse(\array_filter(\array_keys($frameworks), function ($key) use ($framework, $frameworks) { - return $frameworks[$key]['key'] === $framework; - }))[0] ?? ''; - $repo['framework'] = $frameworkDetail; - } catch (Throwable $error) { - $repo['framework'] = ""; - Console::warning("Framework not detected for " . $repo['organization'] . "/" . $repo['name']); + if ($detectedFramework) { + $framework = $detectedFramework->getName(); + } else { + $framework = 'other'; } + + $frameworks = Config::getParam('frameworks'); + $frameworkDetail = ''; + foreach ($frameworks as $key => $config) { + if ($config['key'] === $framework) { + $frameworkDetail = $key; + break; + } + } + $repo['framework'] = !empty($frameworkDetail) ? $frameworkDetail : 'other'; } else { try { + $languages = $github->listRepositoryLanguages($repo['organization'], $repo['name']); + $strategies = [ new Strategy(Strategy::FILEMATCH), new Strategy(Strategy::LANGUAGES), @@ -856,7 +870,7 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') ]; foreach ($strategies as $strategy) { - $runtimeDetector = new Runtime($strategy === Strategy::LANGUAGES ? $languages : $files, $strategy, 'npm'); + $runtimeDetector = new Runtime($strategy === Strategy::LANGUAGES ? $languages : $files, $strategy, $packagerName); $runtimeDetector ->addOption(new Node()) ->addOption(new Bun()) @@ -880,14 +894,18 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') if (!empty($runtime)) { $runtimes = Config::getParam('runtimes'); - $runtimeDetail = \array_reverse(\array_filter(\array_keys($runtimes), function ($key) use ($runtime, $runtimes) { - return $runtimes[$key]['key'] === $runtime; - }))[0] ?? ''; + $runtimeDetail = ''; + foreach ($runtimes as $key => $config) { + if ($config['key'] === $runtime) { + $runtimeDetail = $key; + break; + } + } $repo['runtime'] = $runtimeDetail; } } catch (Throwable $error) { $repo['runtime'] = ""; - Console::warning("Runtime not detected for " . $repo['organization'] . "/" . $repo['name']); + throw new Exception(Exception::FUNCTION_RUNTIME_NOT_DETECTED, "Runtime not detected for " . $repo['organization'] . "/" . $repo['name']); } } return $repo; @@ -916,7 +934,7 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories') responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, - model: Response::MODEL_RUNTIME_PROVIDER_REPOSITORY, + model: Response::MODEL_PROVIDER_REPOSITORY, ) ] )) @@ -1013,7 +1031,7 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories') $repository['organization'] = $installation->getAttribute('organization', ''); $repository['provider'] = $installation->getAttribute('provider', ''); - $response->dynamic(new Document($repository), Response::MODEL_RUNTIME_PROVIDER_REPOSITORY); + $response->dynamic(new Document($repository), Response::MODEL_PROVIDER_REPOSITORY); }); App::get('/v1/vcs/github/installations/:installationId/providerRepositories/:providerRepositoryId') @@ -1028,7 +1046,7 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories/:pro responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, - model: Response::MODEL_RUNTIME_PROVIDER_REPOSITORY, + model: Response::MODEL_PROVIDER_REPOSITORY, ) ] )) @@ -1067,7 +1085,7 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories/:pro $repository['organization'] = $installation->getAttribute('organization', ''); $repository['provider'] = $installation->getAttribute('provider', ''); - $response->dynamic(new Document($repository), Response::MODEL_RUNTIME_PROVIDER_REPOSITORY); + $response->dynamic(new Document($repository), Response::MODEL_PROVIDER_REPOSITORY); }); App::get('/v1/vcs/github/installations/:installationId/providerRepositories/:providerRepositoryId/branches') diff --git a/docker-compose.yml b/docker-compose.yml index 410862ca72..99d04f95fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -962,7 +962,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes-executor-2 + image: openruntimes/executor:0.7.4 restart: unless-stopped networks: - appwrite diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index f1494b5d67..5974d12f00 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -165,6 +165,7 @@ class Exception extends \Exception public const FUNCTION_ENTRYPOINT_MISSING = 'function_entrypoint_missing'; public const FUNCTION_SYNCHRONOUS_TIMEOUT = 'function_synchronous_timeout'; public const FUNCTION_TEMPLATE_NOT_FOUND = 'function_template_not_found'; + public const FUNCTION_RUNTIME_NOT_DETECTED = 'function_runtime_not_detected'; /** Deployments */ public const DEPLOYMENT_NOT_FOUND = 'deployment_not_found'; diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 81b6989df2..2863606106 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -83,6 +83,7 @@ use Appwrite\Utopia\Response\Model\Platform; use Appwrite\Utopia\Response\Model\Preferences; use Appwrite\Utopia\Response\Model\Project; use Appwrite\Utopia\Response\Model\Provider; +use Appwrite\Utopia\Response\Model\ProviderRepository; use Appwrite\Utopia\Response\Model\Rule; use Appwrite\Utopia\Response\Model\Runtime; use Appwrite\Utopia\Response\Model\RuntimeDetection; @@ -247,6 +248,8 @@ class Response extends SwooleResponse // VCS public const MODEL_INSTALLATION = 'installation'; public const MODEL_INSTALLATION_LIST = 'installationList'; + public const MODEL_PROVIDER_REPOSITORY = 'providerRepository'; + public const MODEL_PROVIDER_REPOSITORY_LIST = 'providerRepositoryList'; public const MODEL_FRAMEWORK_PROVIDER_REPOSITORY = 'frameworkProviderRepository'; public const MODEL_FRAMEWORK_PROVIDER_REPOSITORY_LIST = 'frameworkProviderRepositoryList'; public const MODEL_RUNTIME_PROVIDER_REPOSITORY = 'runtimeProviderRepository'; @@ -458,6 +461,7 @@ class Response extends SwooleResponse ->setModel(new TemplateRuntime()) ->setModel(new TemplateVariable()) ->setModel(new Installation()) + ->setModel(new ProviderRepository()) ->setModel(new FrameworkProviderRepository()) ->setModel(new RuntimeProviderRepository()) ->setModel(new FrameworkDetection()) diff --git a/src/Appwrite/Utopia/Response/Model/FrameworkProviderRepository.php b/src/Appwrite/Utopia/Response/Model/FrameworkProviderRepository.php index 305fb28852..2ec75badb6 100644 --- a/src/Appwrite/Utopia/Response/Model/FrameworkProviderRepository.php +++ b/src/Appwrite/Utopia/Response/Model/FrameworkProviderRepository.php @@ -3,57 +3,19 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; -use Appwrite\Utopia\Response\Model; -class FrameworkProviderRepository extends Model +class FrameworkProviderRepository extends ProviderRepository { public function __construct() { - $this - ->addRule('id', [ - 'type' => self::TYPE_STRING, - 'description' => 'VCS (Version Control System) repository ID.', - 'default' => '', - 'example' => '5e5ea5c16897e', - ]) - ->addRule('name', [ - 'type' => self::TYPE_STRING, - 'description' => 'VCS (Version Control System) repository name.', - 'default' => '', - 'example' => 'appwrite', - ]) - ->addRule('organization', [ - 'type' => self::TYPE_STRING, - 'description' => 'VCS (Version Control System) organization name', - 'default' => [], - 'example' => 'appwrite', - 'array' => false, - ]) - ->addRule('provider', [ - 'type' => self::TYPE_STRING, - 'description' => 'VCS (Version Control System) provider name.', - 'default' => '', - 'example' => 'github', - ]) - ->addRule('private', [ - 'type' => self::TYPE_BOOLEAN, - 'description' => 'Is VCS (Version Control System) repository private?', - 'default' => false, - 'example' => true, - ]) - ->addRule('framework', [ - 'type' => self::TYPE_STRING, - 'description' => 'Auto-detected framework suggestion. Empty if getting response of getFramework().', - 'default' => '', - 'example' => 'nextjs', - ]) - ->addRule('pushedAt', [ - 'type' => self::TYPE_DATETIME, - 'description' => 'Last commit date in ISO 8601 format.', - 'default' => APP_DATABASE_ATTRIBUTE_DATETIME, - 'example' => APP_DATABASE_ATTRIBUTE_DATETIME, - 'array' => false, - ]); + parent::__construct(); + + $this->addRule('framework', [ + 'type' => self::TYPE_STRING, + 'description' => 'Auto-detected framework suggestion. Empty if getting response of getFramework().', + 'default' => '', + 'example' => 'nextjs', + ]); } /** diff --git a/src/Appwrite/Utopia/Response/Model/ProviderRepository.php b/src/Appwrite/Utopia/Response/Model/ProviderRepository.php new file mode 100644 index 0000000000..518b5de9ee --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/ProviderRepository.php @@ -0,0 +1,72 @@ +addRule('id', [ + 'type' => self::TYPE_STRING, + 'description' => 'VCS (Version Control System) repository ID.', + 'default' => '', + 'example' => '5e5ea5c16897e', + ]) + ->addRule('name', [ + 'type' => self::TYPE_STRING, + 'description' => 'VCS (Version Control System) repository name.', + 'default' => '', + 'example' => 'appwrite', + ]) + ->addRule('organization', [ + 'type' => self::TYPE_STRING, + 'description' => 'VCS (Version Control System) organization name', + 'default' => [], + 'example' => 'appwrite', + 'array' => false, + ]) + ->addRule('provider', [ + 'type' => self::TYPE_STRING, + 'description' => 'VCS (Version Control System) provider name.', + 'default' => '', + 'example' => 'github', + ]) + ->addRule('private', [ + 'type' => self::TYPE_BOOLEAN, + 'description' => 'Is VCS (Version Control System) repository private?', + 'default' => false, + 'example' => true, + ]) + ->addRule('pushedAt', [ + 'type' => self::TYPE_DATETIME, + 'description' => 'Last commit date in ISO 8601 format.', + 'default' => APP_DATABASE_ATTRIBUTE_DATETIME, + 'example' => APP_DATABASE_ATTRIBUTE_DATETIME, + 'array' => false, + ]); + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'ProviderRepository'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_PROVIDER_REPOSITORY; + } +} diff --git a/src/Appwrite/Utopia/Response/Model/RuntimeProviderRepository.php b/src/Appwrite/Utopia/Response/Model/RuntimeProviderRepository.php index cb8d163b9a..5e0067d23f 100644 --- a/src/Appwrite/Utopia/Response/Model/RuntimeProviderRepository.php +++ b/src/Appwrite/Utopia/Response/Model/RuntimeProviderRepository.php @@ -3,57 +3,19 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; -use Appwrite\Utopia\Response\Model; -class RuntimeProviderRepository extends Model +class RuntimeProviderRepository extends ProviderRepository { public function __construct() { - $this - ->addRule('id', [ - 'type' => self::TYPE_STRING, - 'description' => 'VCS (Version Control System) repository ID.', - 'default' => '', - 'example' => '5e5ea5c16897e', - ]) - ->addRule('name', [ - 'type' => self::TYPE_STRING, - 'description' => 'VCS (Version Control System) repository name.', - 'default' => '', - 'example' => 'appwrite', - ]) - ->addRule('organization', [ - 'type' => self::TYPE_STRING, - 'description' => 'VCS (Version Control System) organization name', - 'default' => [], - 'example' => 'appwrite', - 'array' => false, - ]) - ->addRule('provider', [ - 'type' => self::TYPE_STRING, - 'description' => 'VCS (Version Control System) provider name.', - 'default' => '', - 'example' => 'github', - ]) - ->addRule('private', [ - 'type' => self::TYPE_BOOLEAN, - 'description' => 'Is VCS (Version Control System) repository private?', - 'default' => false, - 'example' => true, - ]) - ->addRule('runtime', [ - 'type' => self::TYPE_STRING, - 'description' => 'Auto-detected runtime suggestion. Empty if getting response of getRuntime().', - 'default' => '', - 'example' => 'node', - ]) - ->addRule('pushedAt', [ - 'type' => self::TYPE_DATETIME, - 'description' => 'Last commit date in ISO 8601 format.', - 'default' => APP_DATABASE_ATTRIBUTE_DATETIME, - 'example' => APP_DATABASE_ATTRIBUTE_DATETIME, - 'array' => false, - ]); + parent::__construct(); + + $this->addRule('runtime', [ + 'type' => self::TYPE_STRING, + 'description' => 'Auto-detected runtime suggestion. Empty if getting response of getRuntime().', + 'default' => '', + 'example' => 'node', + ]); } /** From 20ad91a9065ffe52e1e4ab66b5d190d7a21547a6 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Tue, 4 Mar 2025 13:45:56 +0530 Subject: [PATCH 11/31] Simplify logic to fetch versionedRuntime --- app/controllers/api/vcs.php | 50 ++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 62d6a3363d..e2e53671db 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -699,14 +699,11 @@ App::post('/v1/vcs/github/installations/:installationId/detections') } $frameworks = Config::getParam('frameworks'); - $frameworkDetail = ''; - foreach ($frameworks as $key => $config) { - if ($config['key'] === $framework) { - $frameworkDetail = $key; - break; - } + $frameworkKey = 'other'; + if (\in_array($framework, array_keys($frameworks), true)) { + $frameworkKey = $framework; } - $detection->setAttribute('framework', $frameworkDetail); + $detection->setAttribute('framework', $frameworkKey); } else { $detection = new Document([ 'runtime' => '', @@ -747,14 +744,18 @@ App::post('/v1/vcs/github/installations/:installationId/detections') if (!empty($runtime)) { $runtimes = Config::getParam('runtimes'); - $runtimeDetail = ''; - foreach ($runtimes as $key => $config) { - if ($config['key'] === $runtime) { - $runtimeDetail = $key; - break; + $versionedRuntime = ''; + foreach ($runtimes as $runtimeKey => $runtimeConfig) { + if ($runtimeConfig['key'] === $runtime) { + $versionedRuntime = $runtimeKey; } } - $detection->setAttribute('runtime', $runtimeDetail); + + if (empty($versionedRuntime)) { + throw new Exception(Exception::FUNCTION_RUNTIME_NOT_DETECTED); + } + + $detection->setAttribute('runtime', $versionedRuntime); } else { throw new Exception(Exception::FUNCTION_RUNTIME_NOT_DETECTED); } @@ -851,14 +852,11 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') } $frameworks = Config::getParam('frameworks'); - $frameworkDetail = ''; - foreach ($frameworks as $key => $config) { - if ($config['key'] === $framework) { - $frameworkDetail = $key; - break; - } + $frameworkKey = ''; + if (\in_array($framework, array_keys($frameworks), true)) { + $frameworkKey = $framework; } - $repo['framework'] = !empty($frameworkDetail) ? $frameworkDetail : 'other'; + $repo['framework'] = !empty($frameworkKey) ? $frameworkKey : 'other'; } else { try { $languages = $github->listRepositoryLanguages($repo['organization'], $repo['name']); @@ -894,14 +892,14 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') if (!empty($runtime)) { $runtimes = Config::getParam('runtimes'); - $runtimeDetail = ''; - foreach ($runtimes as $key => $config) { - if ($config['key'] === $runtime) { - $runtimeDetail = $key; - break; + $versionedRuntime = ''; + foreach ($runtimes as $runtimeKey => $runtimeConfig) { + if ($runtimeConfig['key'] === $runtime) { + $versionedRuntime = $runtimeKey; } } - $repo['runtime'] = $runtimeDetail; + + $repo['runtime'] = $versionedRuntime; } } catch (Throwable $error) { $repo['runtime'] = ""; From 9b8dd02a886e6c7c22dbe415327ef1e66b4bff8d Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Wed, 5 Mar 2025 13:48:44 +0530 Subject: [PATCH 12/31] Don't throw exception from list repos detection --- app/controllers/api/vcs.php | 73 +++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index e2e53671db..014a1233d5 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -858,52 +858,47 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') } $repo['framework'] = !empty($frameworkKey) ? $frameworkKey : 'other'; } else { - try { - $languages = $github->listRepositoryLanguages($repo['organization'], $repo['name']); + $languages = $github->listRepositoryLanguages($repo['organization'], $repo['name']); - $strategies = [ - new Strategy(Strategy::FILEMATCH), - new Strategy(Strategy::LANGUAGES), - new Strategy(Strategy::EXTENSION), - ]; + $strategies = [ + new Strategy(Strategy::FILEMATCH), + new Strategy(Strategy::LANGUAGES), + new Strategy(Strategy::EXTENSION), + ]; - foreach ($strategies as $strategy) { - $runtimeDetector = new Runtime($strategy === Strategy::LANGUAGES ? $languages : $files, $strategy, $packagerName); - $runtimeDetector - ->addOption(new Node()) - ->addOption(new Bun()) - ->addOption(new Deno()) - ->addOption(new PHP()) - ->addOption(new Python()) - ->addOption(new Dart()) - ->addOption(new Swift()) - ->addOption(new Ruby()) - ->addOption(new Java()) - ->addOption(new CPP()) - ->addOption(new Dotnet()); + foreach ($strategies as $strategy) { + $runtimeDetector = new Runtime($strategy === Strategy::LANGUAGES ? $languages : $files, $strategy, $packagerName); + $runtimeDetector + ->addOption(new Node()) + ->addOption(new Bun()) + ->addOption(new Deno()) + ->addOption(new PHP()) + ->addOption(new Python()) + ->addOption(new Dart()) + ->addOption(new Swift()) + ->addOption(new Ruby()) + ->addOption(new Java()) + ->addOption(new CPP()) + ->addOption(new Dotnet()); - $detectedRuntime = $runtimeDetector->detect(); + $detectedRuntime = $runtimeDetector->detect(); - if ($detectedRuntime) { - $runtime = $detectedRuntime->getName(); - break; + if ($detectedRuntime) { + $runtime = $detectedRuntime->getName(); + break; + } + } + + if (!empty($runtime)) { + $runtimes = Config::getParam('runtimes'); + $versionedRuntime = ''; + foreach ($runtimes as $runtimeKey => $runtimeConfig) { + if ($runtimeConfig['key'] === $runtime) { + $versionedRuntime = $runtimeKey; } } - if (!empty($runtime)) { - $runtimes = Config::getParam('runtimes'); - $versionedRuntime = ''; - foreach ($runtimes as $runtimeKey => $runtimeConfig) { - if ($runtimeConfig['key'] === $runtime) { - $versionedRuntime = $runtimeKey; - } - } - - $repo['runtime'] = $versionedRuntime; - } - } catch (Throwable $error) { - $repo['runtime'] = ""; - throw new Exception(Exception::FUNCTION_RUNTIME_NOT_DETECTED, "Runtime not detected for " . $repo['organization'] . "/" . $repo['name']); + $repo['runtime'] = $versionedRuntime ?? ''; } } return $repo; From 9a44e8130f3b9d698e94e03186a292e3a8bcbefc Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Wed, 5 Mar 2025 22:35:49 +0530 Subject: [PATCH 13/31] Add tests for framework, runtime and rendering detection --- composer.lock | 85 ++++----- docker-compose.yml | 2 +- .../Modules/Functions/Workers/Builds.php | 41 +++-- .../Services/Sites/SitesCustomServerTest.php | 34 ++++ .../e2e/Services/VCS/VCSConsoleClientTest.php | 174 ++++++++++++++++-- 5 files changed, 250 insertions(+), 86 deletions(-) diff --git a/composer.lock b/composer.lock index 7c53b343a8..80a76d5b8d 100644 --- a/composer.lock +++ b/composer.lock @@ -279,16 +279,16 @@ }, { "name": "brick/math", - "version": "0.12.2", + "version": "0.12.3", "source": { "type": "git", "url": "https://github.com/brick/math.git", - "reference": "901eddb1e45a8e0f689302e40af871c181ecbe40" + "reference": "866551da34e9a618e64a819ee1e01c20d8a588ba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/brick/math/zipball/901eddb1e45a8e0f689302e40af871c181ecbe40", - "reference": "901eddb1e45a8e0f689302e40af871c181ecbe40", + "url": "https://api.github.com/repos/brick/math/zipball/866551da34e9a618e64a819ee1e01c20d8a588ba", + "reference": "866551da34e9a618e64a819ee1e01c20d8a588ba", "shasum": "" }, "require": { @@ -327,7 +327,7 @@ ], "support": { "issues": "https://github.com/brick/math/issues", - "source": "https://github.com/brick/math/tree/0.12.2" + "source": "https://github.com/brick/math/tree/0.12.3" }, "funding": [ { @@ -335,7 +335,7 @@ "type": "github" } ], - "time": "2025-02-26T10:21:45+00:00" + "time": "2025-02-28T13:11:00+00:00" }, { "name": "chillerlan/php-qrcode", @@ -709,16 +709,16 @@ }, { "name": "google/protobuf", - "version": "v4.29.3", + "version": "v4.30.0", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "ab5077c2cfdd1f415f42d11fdbdf903ba8e3d9b7" + "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/ab5077c2cfdd1f415f42d11fdbdf903ba8e3d9b7", - "reference": "ab5077c2cfdd1f415f42d11fdbdf903ba8e3d9b7", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/e1d66682f6836aa87820400f0aa07d9eb566feb6", + "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6", "shasum": "" }, "require": { @@ -747,9 +747,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.3" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.0" }, - "time": "2025-01-08T21:00:13+00:00" + "time": "2025-03-04T22:54:49+00:00" }, { "name": "jean85/pretty-package-versions", @@ -2371,16 +2371,16 @@ }, { "name": "ramsey/collection", - "version": "2.0.0", + "version": "2.1.0", "source": { "type": "git", "url": "https://github.com/ramsey/collection.git", - "reference": "a4b48764bfbb8f3a6a4d1aeb1a35bb5e9ecac4a5" + "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ramsey/collection/zipball/a4b48764bfbb8f3a6a4d1aeb1a35bb5e9ecac4a5", - "reference": "a4b48764bfbb8f3a6a4d1aeb1a35bb5e9ecac4a5", + "url": "https://api.github.com/repos/ramsey/collection/zipball/3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", + "reference": "3c5990b8a5e0b79cd1cf11c2dc1229e58e93f109", "shasum": "" }, "require": { @@ -2388,25 +2388,22 @@ }, "require-dev": { "captainhook/plugin-composer": "^5.3", - "ergebnis/composer-normalize": "^2.28.3", - "fakerphp/faker": "^1.21", + "ergebnis/composer-normalize": "^2.45", + "fakerphp/faker": "^1.24", "hamcrest/hamcrest-php": "^2.0", - "jangregor/phpstan-prophecy": "^1.0", - "mockery/mockery": "^1.5", + "jangregor/phpstan-prophecy": "^2.1", + "mockery/mockery": "^1.6", "php-parallel-lint/php-console-highlighter": "^1.0", - "php-parallel-lint/php-parallel-lint": "^1.3", - "phpcsstandards/phpcsutils": "^1.0.0-rc1", - "phpspec/prophecy-phpunit": "^2.0", - "phpstan/extension-installer": "^1.2", - "phpstan/phpstan": "^1.9", - "phpstan/phpstan-mockery": "^1.1", - "phpstan/phpstan-phpunit": "^1.3", - "phpunit/phpunit": "^9.5", - "psalm/plugin-mockery": "^1.1", - "psalm/plugin-phpunit": "^0.18.4", - "ramsey/coding-standard": "^2.0.3", - "ramsey/conventional-commits": "^1.3", - "vimeo/psalm": "^5.4" + "php-parallel-lint/php-parallel-lint": "^1.4", + "phpspec/prophecy-phpunit": "^2.3", + "phpstan/extension-installer": "^1.4", + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-mockery": "^2.0", + "phpstan/phpstan-phpunit": "^2.0", + "phpunit/phpunit": "^10.5", + "ramsey/coding-standard": "^2.3", + "ramsey/conventional-commits": "^1.6", + "roave/security-advisories": "dev-latest" }, "type": "library", "extra": { @@ -2444,19 +2441,9 @@ ], "support": { "issues": "https://github.com/ramsey/collection/issues", - "source": "https://github.com/ramsey/collection/tree/2.0.0" + "source": "https://github.com/ramsey/collection/tree/2.1.0" }, - "funding": [ - { - "url": "https://github.com/ramsey", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/ramsey/collection", - "type": "tidelift" - } - ], - "time": "2022-12-31T21:50:55+00:00" + "time": "2025-03-02T04:48:29+00:00" }, { "name": "ramsey/uuid", @@ -3777,12 +3764,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/detector.git", - "reference": "69958f4d765c5338a703f3e3638a3249ec227be8" + "reference": "f839ad4b75725381bca1b0badfda6fc7e5cbc44f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/detector/zipball/69958f4d765c5338a703f3e3638a3249ec227be8", - "reference": "69958f4d765c5338a703f3e3638a3249ec227be8", + "url": "https://api.github.com/repos/utopia-php/detector/zipball/f839ad4b75725381bca1b0badfda6fc7e5cbc44f", + "reference": "f839ad4b75725381bca1b0badfda6fc7e5cbc44f", "shasum": "" }, "require": { @@ -3832,7 +3819,7 @@ "source": "https://github.com/utopia-php/detector/tree/feat-pseudocode-draft2", "issues": "https://github.com/utopia-php/detector/issues" }, - "time": "2025-02-18T15:22:56+00:00" + "time": "2025-03-04T15:07:14+00:00" }, { "name": "utopia-php/domains", diff --git a/docker-compose.yml b/docker-compose.yml index 394505aeb4..d43c84ccf1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -962,7 +962,7 @@ services: hostname: exc1 <<: *x-logging stop_signal: SIGINT - image: openruntimes/executor:0.7.5 + image: openruntimes/executor:0.7.7 restart: unless-stopped networks: - appwrite diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 298f1d4496..8d0c36a8e1 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -26,8 +26,8 @@ use Utopia\Database\Exception\Structure; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; -use Utopia\Detector\Detection\Rendering\SSG; use Utopia\Detector\Detection\Rendering\SSR; +use Utopia\Detector\Detection\Rendering\XStatic; use Utopia\Detector\Detector\Rendering; use Utopia\Fetch\Client as FetchClient; use Utopia\Logger\Log; @@ -39,6 +39,8 @@ use Utopia\Storage\Device\Local; use Utopia\System\System; use Utopia\VCS\Adapter\Git\GitHub; +use function PHPUnit\Framework\isEmpty; + class Builds extends Action { public static function getName(): string @@ -708,27 +710,30 @@ class Builds extends Action throw new \Exception('Build size should be less than ' . number_format($buildSizeLimit / 1048576, 2) . ' MBs.'); } - $listFilesCommand = "cd /usr/local/build/" . $resource->getAttribute('outputDirectory') . " && find . -name 'node_modules' -prune -o -type f -print"; - $response = $executor->executeCommand( - deploymentId: $deployment->getId(), - projectId: $project->getId(), - command: $listFilesCommand, - timeout: 60 - ); + if ($resource->getCollection() === 'sites' && isEmpty($resource->getAttribute('adapter'))) { + $listFilesCommand = "cd /usr/local/build/" . $resource->getAttribute('outputDirectory') . " && find . -name 'node_modules' -prune -o -type f -print"; + $commandExecutionResponse = $executor->executeCommand( + deploymentId: $deployment->getId(), + projectId: $project->getId(), + command: $listFilesCommand, + timeout: 60 + ); - $files = array_map('trim', array_filter(explode("\n", $response['output']))); + $files = array_map('trim', array_filter(explode("\n", $commandExecutionResponse['output']))); - $detector = new Rendering($files, $resource->getAttribute('framework')); - $detector - ->addOption(new SSR()) - ->addOption(new SSG()); - $detectedRenderingStrategy = $detector->detect(); + $detector = new Rendering($files, $resource->getAttribute('framework')); + $detector + ->addOption(new SSR()) + ->addOption(new XStatic()); + $detectedRenderingStrategy = $detector->detect(); - $renderingStrategy = $detectedRenderingStrategy->getName(); - $fallbackFile = $detectedRenderingStrategy->getFallbackFile(); + $renderingStrategy = $detectedRenderingStrategy->getName(); + $fallbackFile = $detectedRenderingStrategy->getFallbackFile() ?: ''; - $resource->setAttribute('adapter', $renderingStrategy); - $resource->setAttribute('fallbackFile', $fallbackFile); + $resource->setAttribute('adapter', $renderingStrategy); + $resource->setAttribute('fallbackFile', $fallbackFile); + $resource = $dbForProject->updateDocument('sites', $resource->getId(), $resource); + } /** Update the build document */ $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); diff --git a/tests/e2e/Services/Sites/SitesCustomServerTest.php b/tests/e2e/Services/Sites/SitesCustomServerTest.php index 43d9fc173b..bb4ead022d 100644 --- a/tests/e2e/Services/Sites/SitesCustomServerTest.php +++ b/tests/e2e/Services/Sites/SitesCustomServerTest.php @@ -324,6 +324,40 @@ class SitesCustomServerTest extends Scope $this->cleanupSite($siteId); } + public function testRenderingDetectionSSR(): void + { + $siteId = $this->setupSite([ + 'siteId' => ID::unique(), + 'name' => 'Astro site', + 'framework' => 'astro', + 'buildRuntime' => 'ssr-22', + 'outputDirectory' => './dist', + 'buildCommand' => 'npm run build', + 'installCommand' => 'npm install', + ]); + + $this->assertNotEmpty($siteId); + + $domain = $this->setupSiteDomain($siteId); + + $deploymentId = $this->setupDeployment($siteId, [ + 'code' => $this->packageSite('astro'), + 'activate' => 'true' + ]); + + $this->assertNotEmpty($deploymentId); + + $domain = $this->getSiteDomain($siteId); + $proxyClient = new Client(); + $proxyClient->setEndpoint('http://' . $domain); + + $response = $proxyClient->call(Client::METHOD_GET, '/'); + + $this->assertEquals(200, $response['headers']['status-code']); + + $this->cleanupSite($siteId); + } + public function testListSites(): void { /** diff --git a/tests/e2e/Services/VCS/VCSConsoleClientTest.php b/tests/e2e/Services/VCS/VCSConsoleClientTest.php index f04667a0f5..f60b91c27f 100644 --- a/tests/e2e/Services/VCS/VCSConsoleClientTest.php +++ b/tests/e2e/Services/VCS/VCSConsoleClientTest.php @@ -22,6 +22,8 @@ class VCSConsoleClientTest extends Scope public string $providerInstallationId = '42954928'; // appwrite-test public string $providerRepositoryId = '705764267'; // ruby-starter (public) public string $providerRepositoryId2 = '708688544'; // function1.4 (private) + public string $providerRepositoryId3 = '943139433'; // svelte-starter (public) + public string $providerRepositoryId4 = '943245292'; // templates-for-sites (public) public function testGitHubAuthorize(): string { @@ -67,24 +69,104 @@ class VCSConsoleClientTest extends Scope * Test for SUCCESS */ - $runtime = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/providerRepositories/' . $this->providerRepositoryId . '/detection', array_merge([ + $runtime = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/detections', array_merge([ 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); + 'content-type' => 'application/json', + ], $this->getHeaders()), [ + 'providerRepositoryId' => $this->providerRepositoryId, + 'type' => 'runtime', + ]); $this->assertEquals(200, $runtime['headers']['status-code']); - $this->assertEquals($runtime['body']['runtime'], 'ruby-3.1'); + $this->assertEquals($runtime['body']['runtime'], 'ruby-3.3'); + $this->assertEquals($runtime['body']['commands'], 'bundle install && bundle exec rake build'); + $this->assertEquals($runtime['body']['entrypoint'], 'main.rb'); /** * Test for FAILURE */ - $runtime = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/providerRepositories/randomRepositoryId/detection', array_merge([ + $runtime = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/detections', array_merge([ 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); + 'content-type' => 'application/json', + ], $this->getHeaders()), [ + 'providerRepositoryId' => 'randomRepositoryId', // Invalid repository ID + 'type' => 'runtime', + ]); $this->assertEquals(404, $runtime['headers']['status-code']); } + /** + * @depends testGitHubAuthorize + */ + public function testDetectFramework(string $installationId) + { + /** + * Test for SUCCESS + */ + + $framework = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/detections', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + 'content-type' => 'application/json', + ], $this->getHeaders()), [ + 'providerRepositoryId' => $this->providerRepositoryId3, + 'type' => 'framework', + ]); + + $this->assertEquals(200, $framework['headers']['status-code']); + $this->assertEquals($framework['body']['framework'], 'sveltekit'); + $this->assertEquals($framework['body']['installCommand'], 'npm install'); + $this->assertEquals($framework['body']['buildCommand'], 'npm run build'); + $this->assertEquals($framework['body']['outputDirectory'], './build'); + + $framework = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/detections', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + 'content-type' => 'application/json', + ], $this->getHeaders()), [ + 'providerRepositoryId' => $this->providerRepositoryId4, + 'type' => 'framework', + 'providerRootDirectory' => 'astro/starter' + ]); + + $this->assertEquals(200, $framework['headers']['status-code']); + $this->assertEquals($framework['body']['framework'], 'astro'); + $this->assertEquals($framework['body']['installCommand'], 'npm install'); + $this->assertEquals($framework['body']['buildCommand'], 'npm run build'); + $this->assertEquals($framework['body']['outputDirectory'], './dist'); + + $framework = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/detections', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + 'content-type' => 'application/json', + ], $this->getHeaders()), [ + 'providerRepositoryId' => $this->providerRepositoryId4, + 'type' => 'framework', + 'providerRootDirectory' => 'remix/starter' + ]); + + $this->assertEquals(200, $framework['headers']['status-code']); + $this->assertEquals($framework['body']['framework'], 'remix'); + $this->assertEquals($framework['body']['installCommand'], 'npm install'); + $this->assertEquals($framework['body']['buildCommand'], 'npm run build'); + $this->assertEquals($framework['body']['outputDirectory'], './build'); + + /** + * Test for FAILURE + */ + + $framework = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/providerRepositories/randomRepositoryId/frameworks', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + 'content-type' => 'application/json', + ], $this->getHeaders()), [ + 'providerRepositoryId' => 'randomRepositoryId', // Invalid repository ID + 'type' => 'framework', + ]); + + $this->assertEquals(404, $framework['headers']['status-code']); + } + + // TODO: Change search to provideScenarios + /** * @depends testGitHubAuthorize */ @@ -94,7 +176,7 @@ class VCSConsoleClientTest extends Scope * Test for SUCCESS */ - $runtime = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/providerRepositories/' . $this->providerRepositoryId . '/contents', array_merge([ + $runtime = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories/' . $this->providerRepositoryId . '/contents', array_merge([ 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders())); @@ -126,7 +208,7 @@ class VCSConsoleClientTest extends Scope $this->assertTrue($libContent['isDirectory']); $this->assertEquals(0, $gemfileContent['size']); - $runtime = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/providerRepositories/' . $this->providerRepositoryId . '/contents?providerRootDirectory=lib', array_merge([ + $runtime = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories/' . $this->providerRepositoryId . '/contents?providerRootDirectory=lib', array_merge([ 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders())); @@ -150,7 +232,7 @@ class VCSConsoleClientTest extends Scope * Test for FAILURE */ - $runtime = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/providerRepositories/randomRepositoryId/contents', array_merge([ + $runtime = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories/randomRepositoryId/contents', array_merge([ 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders())); @@ -166,28 +248,64 @@ class VCSConsoleClientTest extends Scope * Test for SUCCESS */ + //runtimes $repositories = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories', array_merge([ 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); + ], $this->getHeaders()), [ + 'type' => 'runtime' + ]); $this->assertEquals(200, $repositories['headers']['status-code']); - $this->assertEquals($repositories['body']['total'], 3); - $this->assertEquals($repositories['body']['providerRepositories'][0]['name'], 'function1.4'); - $this->assertEquals($repositories['body']['providerRepositories'][0]['organization'], 'appwrite-test'); - $this->assertEquals($repositories['body']['providerRepositories'][0]['provider'], 'github'); - $this->assertEquals($repositories['body']['providerRepositories'][1]['name'], 'appwrite'); - $this->assertEquals($repositories['body']['providerRepositories'][2]['name'], 'ruby-starter'); + $this->assertEquals($repositories['body']['total'], 4); + $this->assertEquals($repositories['body']['runtimeProviderRepositories'][0]['name'], 'starter-for-svelte'); + $this->assertEquals($repositories['body']['runtimeProviderRepositories'][0]['organization'], 'appwrite-test'); + $this->assertEquals($repositories['body']['runtimeProviderRepositories'][0]['provider'], 'github'); + $this->assertEquals($repositories['body']['runtimeProviderRepositories'][0]['runtime'], 'node-22'); + $this->assertEquals($repositories['body']['runtimeProviderRepositories'][1]['name'], 'templates-for-functions'); + $this->assertEquals($repositories['body']['runtimeProviderRepositories'][1]['runtime'], ''); + + $this->assertEquals($repositories['body']['runtimeProviderRepositories'][2]['name'], 'function1.4'); + $this->assertEquals($repositories['body']['runtimeProviderRepositories'][2]['runtime'], 'node-22'); + + $this->assertEquals($repositories['body']['runtimeProviderRepositories'][3]['name'], 'appwrite'); + $this->assertEquals($repositories['body']['runtimeProviderRepositories'][3]['runtime'], 'php-8.3'); $searchedRepositories = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories', array_merge([ 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ - 'search' => 'func' + 'search' => 'rub', + 'type' => 'runtime' ]); $this->assertEquals(200, $searchedRepositories['headers']['status-code']); $this->assertEquals($searchedRepositories['body']['total'], 1); - $this->assertEquals($searchedRepositories['body']['providerRepositories'][0]['name'], 'function1.4'); + $this->assertEquals($searchedRepositories['body']['runtimeProviderRepositories'][0]['name'], 'ruby-starter'); + $this->assertEquals($searchedRepositories['body']['runtimeProviderRepositories'][0]['runtime'], 'ruby-3.3'); + + // frameworks + $repositories = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'type' => 'framework' + ]); + + $this->assertEquals(200, $repositories['headers']['status-code']); + $this->assertEquals($repositories['body']['total'], 4); + + $this->assertEquals($repositories['body']['frameworkProviderRepositories'][0]['name'], 'starter-for-svelte'); + $this->assertEquals($repositories['body']['frameworkProviderRepositories'][0]['organization'], 'appwrite-test'); + $this->assertEquals($repositories['body']['frameworkProviderRepositories'][0]['provider'], 'github'); + $this->assertEquals($repositories['body']['frameworkProviderRepositories'][0]['framework'], 'sveltekit'); + + $this->assertEquals($repositories['body']['frameworkProviderRepositories'][1]['name'], 'templates-for-functions'); + $this->assertEquals($repositories['body']['frameworkProviderRepositories'][1]['framework'], 'other'); + + $this->assertEquals($repositories['body']['frameworkProviderRepositories'][2]['name'], 'function1.4'); + $this->assertEquals($repositories['body']['frameworkProviderRepositories'][2]['framework'], 'other'); + + $this->assertEquals($repositories['body']['frameworkProviderRepositories'][3]['name'], 'appwrite'); + $this->assertEquals($repositories['body']['frameworkProviderRepositories'][3]['framework'], 'other'); /** * Test for FAILURE @@ -195,9 +313,29 @@ class VCSConsoleClientTest extends Scope $repositories = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/randomInstallationId/providerRepositories', array_merge([ 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); + ], $this->getHeaders()), [ + 'type' => 'runtime' + ]); $this->assertEquals(404, $repositories['headers']['status-code']); + + $repositories = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'type' => 'randomType' + ]); + + $this->assertEquals(400, $repositories['headers']['status-code']); + + $repositories = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'randomSearch', + 'type' => 'framework' + ]); + + $this->assertEquals(200, $repositories['headers']['status-code']); + $this->assertEquals($repositories['body']['total'], 0); } /** From 2e30c1ba1f7e13705f2e417df57065075d8198a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 6 Mar 2025 15:34:06 +0100 Subject: [PATCH 14/31] Reapply "Remove builds collection" This reverts commit 654d178cf24343d772c4b4e72617a3e7d4dedbf8. --- app/config/collections/projects.php | 256 ++++++------------ app/controllers/general.php | 10 +- .../Http/Deployments/Download/Get.php | 7 +- .../Functions/Http/Deployments/Get.php | 7 - .../Http/Deployments/Status/Update.php | 55 +--- .../Functions/Http/Deployments/XList.php | 9 - .../Functions/Http/Executions/Create.php | 12 +- .../Http/Functions/Deployment/Update.php | 7 +- .../Modules/Functions/Workers/Builds.php | 169 +++++------- .../Sites/Http/Deployments/Download/Get.php | 7 +- .../Modules/Sites/Http/Deployments/Get.php | 7 - .../Sites/Http/Deployments/Status/Update.php | 55 +--- .../Modules/Sites/Http/Deployments/XList.php | 9 - .../Sites/Http/Sites/Deployment/Update.php | 7 +- src/Appwrite/Platform/Workers/Deletes.php | 49 +--- src/Appwrite/Platform/Workers/Functions.php | 16 +- .../Platform/Workers/StatsResources.php | 11 +- src/Appwrite/Utopia/Response.php | 5 - src/Appwrite/Utopia/Response/Model/Build.php | 94 ------- 19 files changed, 213 insertions(+), 579 deletions(-) delete mode 100644 src/Appwrite/Utopia/Response/Model/Build.php diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 597f02f7d6..5fd997bb47 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1154,7 +1154,6 @@ return [ ] ], ], - 'deployments' => [ '$collection' => ID::custom(Database::METADATA), '$id' => ID::custom('deployments'), @@ -1204,17 +1203,6 @@ return [ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('buildId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], [ 'array' => false, '$id' => ID::custom('entrypoint'), @@ -1551,6 +1539,94 @@ return [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('startTime'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => ID::custom('endTime'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => ID::custom('buildTime'), + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('buildSize'), + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('runtime'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => true, + 'default' => '', + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('status'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, + 'signed' => true, + 'required' => true, + 'default' => 'processing', + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('buildPath'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 2048, + 'signed' => true, + 'required' => false, + 'default' => '', + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('buildLogs'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 1000000, + 'signed' => true, + 'required' => false, + 'default' => '', + 'array' => false, + 'filters' => [], + ] ], 'indexes' => [ [ @@ -1581,13 +1657,6 @@ return [ 'lengths' => [], 'orders' => [Database::ORDER_ASC], ], - [ - '$id' => ID::custom('_key_buildId'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['buildId'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - ], [ '$id' => ID::custom('_key_activate'), 'type' => Database::INDEX_KEY, @@ -1598,155 +1667,6 @@ return [ ], ], - 'builds' => [ - '$collection' => ID::custom(Database::METADATA), - '$id' => ID::custom('builds'), - 'name' => 'Builds', - 'attributes' => [ - [ - '$id' => ID::custom('startTime'), - 'type' => Database::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ], - [ - '$id' => ID::custom('endTime'), - 'type' => Database::VAR_DATETIME, - 'format' => '', - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ], - [ - '$id' => ID::custom('duration'), - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('size'), - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('deploymentInternalId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('deploymentId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('runtime'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => true, - 'default' => '', - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('status'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 256, - 'signed' => true, - 'required' => true, - 'default' => 'processing', - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('path'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => '', - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('logs'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 1000000, - 'signed' => true, - 'required' => false, - 'default' => '', - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('sourceType'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => true, - 'default' => 'local', - 'array' => false, - 'filters' => [], - ], - [ - '$id' => ID::custom('source'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => true, - 'default' => '', - 'array' => false, - 'filters' => [], - ] - ], - 'indexes' => [ - [ - '$id' => ID::custom('_key_deployment'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['deploymentId'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ] - ], - ], - 'executions' => [ '$collection' => ID::custom(Database::METADATA), '$id' => ID::custom('executions'), diff --git a/app/controllers/general.php b/app/controllers/general.php index d4728fb5af..5e3edca418 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -191,13 +191,7 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw throw new AppwriteException(AppwriteException::FUNCTION_RUNTIME_UNSUPPORTED, 'Runtime "' . $resource->getAttribute('runtime', '') . '" is not supported'); } - /** Check if build has completed */ - $build = Authorization::skip(fn () => $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', ''))); - if ($build->isEmpty()) { - throw new AppwriteException(AppwriteException::BUILD_NOT_FOUND); - } - - if ($build->getAttribute('status') !== 'ready') { + if ($deployment->getAttribute('status') !== 'ready') { throw new AppwriteException(AppwriteException::BUILD_NOT_READY); } @@ -388,7 +382,7 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw variables: $vars, timeout: $resource->getAttribute('timeout', 30), image: $runtime['image'], - source: $build->getAttribute('path', ''), + source: $deployment->getAttribute('buildPath', ''), entrypoint: $entrypoint, version: $version, path: $path, diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php index a63ac18354..d528bc9995 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php @@ -81,12 +81,7 @@ class Get extends Action switch ($type) { case 'output': - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); - if ($build->isEmpty()) { - throw new Exception(Exception::BUILD_NOT_FOUND); - } - - $path = $build->getAttribute('path', ''); + $path = $deployment->getAttribute('buildPath', ''); $device = $deviceForBuilds; break; case 'source': diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Get.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Get.php index 1d074e9e2a..8e58433e7c 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Get.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Get.php @@ -69,13 +69,6 @@ class Get extends Action throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); - $deployment->setAttribute('status', $build->getAttribute('status', 'waiting')); - $deployment->setAttribute('buildLogs', $build->getAttribute('logs', '')); - $deployment->setAttribute('buildTime', $build->getAttribute('duration', 0)); - $deployment->setAttribute('buildSize', $build->getAttribute('size', 0)); - $deployment->setAttribute('size', $deployment->getAttribute('size', 0)); - $response->dynamic($deployment, Response::MODEL_DEPLOYMENT); } } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php index e349292e4b..5b58a972a3 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php @@ -13,8 +13,6 @@ use Utopia\App; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Helpers\ID; -use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; @@ -50,7 +48,7 @@ class Update extends Action responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, - model: Response::MODEL_BUILD, + model: Response::MODEL_DEPLOYMENT, ) ] )) @@ -77,46 +75,19 @@ class Update extends Action throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } - $build = Authorization::skip(fn () => $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', ''))); - - if ($build->isEmpty()) { - $buildId = ID::unique(); - $build = $dbForProject->createDocument('builds', new Document([ - '$id' => $buildId, - '$permissions' => [], - 'startTime' => DateTime::now(), - 'deploymentInternalId' => $deployment->getInternalId(), - 'deploymentId' => $deployment->getId(), - 'status' => 'canceled', - 'path' => '', - 'runtime' => $function->getAttribute('runtime'), - 'source' => $deployment->getAttribute('path', ''), - 'sourceType' => '', - 'logs' => '', - 'duration' => 0, - 'size' => 0 - ])); - - $deployment->setAttribute('buildId', $build->getId()); - $deployment->setAttribute('buildInternalId', $build->getInternalId()); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); - } else { - if (\in_array($build->getAttribute('status'), ['ready', 'failed'])) { - throw new Exception(Exception::BUILD_ALREADY_COMPLETED); - } - - $startTime = new \DateTime($build->getAttribute('startTime')); - $endTime = new \DateTime('now'); - $duration = $endTime->getTimestamp() - $startTime->getTimestamp(); - - $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttributes([ - 'endTime' => DateTime::now(), - 'duration' => $duration, - 'status' => 'canceled' - ])); + if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'])) { + throw new Exception(Exception::BUILD_ALREADY_COMPLETED); } - $dbForProject->purgeCachedDocument('deployments', $deployment->getId()); + $startTime = new \DateTime($deployment->getAttribute('startTime')); + $endTime = new \DateTime('now'); + $duration = $endTime->getTimestamp() - $startTime->getTimestamp(); + + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttributes([ + 'endTime' => DateTime::now(), + 'duration' => $duration, + 'status' => 'canceled' + ])); try { $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); @@ -132,6 +103,6 @@ class Update extends Action ->setParam('functionId', $function->getId()) ->setParam('deploymentId', $deployment->getId()); - $response->dynamic($build, Response::MODEL_BUILD); + $response->dynamic($deployment, Response::MODEL_DEPLOYMENT); } } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/XList.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/XList.php index 9e03a86bc7..a416001e2f 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/XList.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/XList.php @@ -110,15 +110,6 @@ class XList extends Action $results = $dbForProject->find('deployments', $queries); $total = $dbForProject->count('deployments', $filterQueries, APP_LIMIT_COUNT); - foreach ($results as $result) { - $build = $dbForProject->getDocument('builds', $result->getAttribute('buildId', '')); - $result->setAttribute('status', $build->getAttribute('status', 'processing')); - $result->setAttribute('buildLogs', $build->getAttribute('logs', '')); - $result->setAttribute('buildTime', $build->getAttribute('duration', 0)); - $result->setAttribute('buildSize', $build->getAttribute('size', 0)); - $result->setAttribute('size', $result->getAttribute('size', 0)); - } - $response->dynamic(new Document([ 'deployments' => $results, 'total' => $total, diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php index f522e2dc62..9678b1ef64 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php @@ -4,8 +4,6 @@ namespace Appwrite\Platform\Modules\Functions\Http\Executions; use Ahc\Jwt\JWT; use Appwrite\Auth\Auth; -use Appwrite\Event\Build; -use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\StatsUsage; @@ -158,13 +156,7 @@ class Create extends Base throw new Exception(Exception::DEPLOYMENT_NOT_FOUND, 'Deployment not found. Create a deployment before trying to execute a function'); } - /** Check if build has completed */ - $build = Authorization::skip(fn () => $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', ''))); - if ($build->isEmpty()) { - throw new Exception(Exception::BUILD_NOT_FOUND); - } - - if ($build->getAttribute('status') !== 'ready') { + if ($deployment->getAttribute('status') !== 'ready') { throw new Exception(Exception::BUILD_NOT_READY); } @@ -385,7 +377,7 @@ class Create extends Base variables: $vars, timeout: $function->getAttribute('timeout', 0), image: $runtime['image'], - source: $build->getAttribute('path', ''), + source: $deployment->getAttribute('buildPath', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), version: $version, path: $path, diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Deployment/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Deployment/Update.php index f45aa0a108..cda0b8b6d9 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Deployment/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Deployment/Update.php @@ -68,7 +68,6 @@ class Update extends Base { $function = $dbForProject->getDocument('functions', $functionId); $deployment = $dbForProject->getDocument('deployments', $deploymentId); - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); if ($function->isEmpty()) { throw new Exception(Exception::FUNCTION_NOT_FOUND); @@ -78,11 +77,7 @@ class Update extends Base throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } - if ($build->isEmpty()) { - throw new Exception(Exception::BUILD_NOT_FOUND); - } - - if ($build->getAttribute('status') !== 'ready') { + if ($deployment->getAttribute('status') !== 'ready') { throw new Exception(Exception::BUILD_NOT_READY); } diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 53556f60ba..f66c4b4638 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -186,38 +186,20 @@ class Builds extends Action $startTime = DateTime::now(); $durationStart = \microtime(true); - $buildId = $deployment->getAttribute('buildId', ''); - $build = $dbForProject->getDocument('builds', $buildId); - $isNewBuild = empty($buildId); - if ($build->isEmpty()) { - $buildId = ID::unique(); - $build = $dbForProject->createDocument('builds', new Document([ - '$id' => $buildId, - '$permissions' => [], - 'startTime' => $startTime, - 'deploymentInternalId' => $deployment->getInternalId(), - 'deploymentId' => $deployment->getId(), - 'status' => 'processing', - 'path' => '', - 'runtime' => $resource->getAttribute('runtime'), - 'source' => $deployment->getAttribute('path', ''), - 'sourceType' => strtolower($deviceForFunctions->getType()), - 'logs' => '', - 'endTime' => null, - 'duration' => 0, - 'size' => 0 - ])); - $deployment->setAttribute('buildId', $build->getId()); - $deployment->setAttribute('buildInternalId', $build->getInternalId()); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); - } elseif ($build->getAttribute('status') === 'canceled') { + if ($deployment->getAttribute('status') === 'canceled') { Console::info('Build has been canceled'); return; - } else { - $build = $dbForProject->getDocument('builds', $buildId); } + $deploymentId = $deployment->getId(); + + $deployment->setAttribute('startTime', $startTime); + $deployment->setAttribute('status', 'processing'); + $deployment->setAttribute('runtime', $resource->getAttribute('runtime')); + $deployment->setAttribute('sourceType', \strtolower($deviceForFunctions->getType())); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + $source = $deployment->getAttribute('path', ''); $installationId = $deployment->getAttribute('installationId', ''); $providerRepositoryId = $deployment->getAttribute('providerRepositoryId', ''); @@ -236,7 +218,7 @@ class Builds extends Action } try { - if ($isNewBuild && !$isVcsEnabled) { + if (!$isVcsEnabled) { // Non-VCS + Template $templateRepositoryName = $template->getAttribute('repositoryName', ''); $templateOwnerName = $template->getAttribute('ownerName', ''); @@ -252,7 +234,7 @@ class Builds extends Action $stderr = ''; // Clone template repo - $tmpTemplateDirectory = '/tmp/builds/' . $buildId . '-template'; + $tmpTemplateDirectory = '/tmp/builds/' . $deploymentId . '-template'; $gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateVersion, GitHub::CLONE_TYPE_TAG, $tmpTemplateDirectory, $templateRootDirectory); $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr); @@ -286,12 +268,14 @@ class Builds extends Action Console::execute('rm -rf ' . \escapeshellarg($tmpTemplateDirectory), '', $stdout, $stderr); $directorySize = $device->getFileSize($source); - $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source)); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize)); + $deployment + ->setAttribute('path', $source) + ->setAttribute('size', $directorySize); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); } - } elseif ($isNewBuild && $isVcsEnabled) { + } elseif ($isVcsEnabled) { // VCS and VCS+Temaplte - $tmpDirectory = '/tmp/builds/' . $buildId . '/code'; + $tmpDirectory = '/tmp/builds/' . $deploymentId . '/code'; $rootDirectory = $resource->getAttribute('providerRootDirectory', ''); $rootDirectory = \rtrim($rootDirectory, '/'); $rootDirectory = \ltrim($rootDirectory, '.'); @@ -317,9 +301,9 @@ class Builds extends Action $stdout = ''; $stderr = ''; - Console::execute('mkdir -p ' . \escapeshellarg('/tmp/builds/' . $buildId), '', $stdout, $stderr); + Console::execute('mkdir -p ' . \escapeshellarg('/tmp/builds/' . $deploymentId), '', $stdout, $stderr); - if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { + if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { Console::info('Build has been canceled'); return; } @@ -356,7 +340,7 @@ class Builds extends Action if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateVersion)) { // Clone template repo - $tmpTemplateDirectory = '/tmp/builds/' . $buildId . '/template'; + $tmpTemplateDirectory = '/tmp/builds/' . $deploymentId . '/template'; $gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateVersion, GitHub::CLONE_TYPE_TAG, $tmpTemplateDirectory, $templateRootDirectory); $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr); @@ -401,19 +385,19 @@ class Builds extends Action $target = Realtime::fromPayload( // Pass first, most verbose event pattern event: $allEvents[0], - payload: $build, + payload: $deployment, project: $project ); Realtime::send( projectId: 'console', - payload: $build->getArrayCopy(), + payload: $deployment->getArrayCopy(), events: $allEvents, channels: $target['channels'], roles: $target['roles'] ); } - $tmpPath = '/tmp/builds/' . $buildId; + $tmpPath = '/tmp/builds/' . $deployment; $tmpPathFile = $tmpPath . '/code.tar.gz'; $localDevice = new Local(); @@ -430,7 +414,7 @@ class Builds extends Action Console::execute('find ' . \escapeshellarg($tmpDirectory) . ' -type d -name ".git" -exec rm -rf {} +', '', $stdout, $stderr); - $tarParamDirectory = '/tmp/builds/' . $buildId . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory); + $tarParamDirectory = '/tmp/builds/' . $deploymentId . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory); Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax $source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); @@ -442,17 +426,19 @@ class Builds extends Action Console::execute('rm -rf ' . \escapeshellarg($tmpPath), '', $stdout, $stderr); - $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source)); - $directorySize = $device->getFileSize($source); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize)); + + $deployment + ->setAttribute('path', $source) + ->setAttribute('size', $directorySize); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); $this->runGitAction('processing', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform); } /** Request the executor to build the code... */ - $build->setAttribute('status', 'building'); - $build = $dbForProject->updateDocument('builds', $buildId, $build); + $deployment->setAttribute('status', 'building'); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); if ($isVcsEnabled) { $this->runGitAction('building', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform); @@ -481,13 +467,13 @@ class Builds extends Action $target = Realtime::fromPayload( // Pass first, most verbose event pattern event: $allEvents[0], - payload: $build, + payload: $deployment, project: $project ); Realtime::send( projectId: 'console', - payload: $build->getArrayCopy(), + payload: $deployment->getArrayCopy(), events: $allEvents, channels: $target['channels'], roles: $target['roles'] @@ -582,7 +568,7 @@ class Builds extends Action $response = null; $err = null; - if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { + if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { Console::info('Build has been canceled'); return; } @@ -615,26 +601,22 @@ class Builds extends Action $err = $error; } }), - Co\go(function () use ($executor, $project, $deployment, &$response, &$build, $dbForProject, $allEvents, $timeout, &$err, &$isCanceled) { + Co\go(function () use ($executor, $project, &$deployment, &$response, $dbForProject, $allEvents, $timeout, &$err, &$isCanceled) { try { $executor->getLogs( deploymentId: $deployment->getId(), projectId: $project->getId(), timeout: $timeout, - callback: function ($logs) use (&$response, &$err, &$build, $dbForProject, $allEvents, $project, &$isCanceled) { + callback: function ($logs) use (&$response, &$err, $dbForProject, $allEvents, $project, &$isCanceled, &$deployment) { if ($isCanceled) { return; } // If we have response or error from concurrent coroutine, we already have latest logs if ($response === null && $err === null) { - $build = $dbForProject->getDocument('builds', $build->getId()); + $deployment = $dbForProject->getDocument('deployments', $deployment->getId()); - if ($build->isEmpty()) { - throw new \Exception('Build not found'); - } - - if ($build->getAttribute('status') === 'canceled') { + if ($deployment->getAttribute('status') === 'canceled') { $isCanceled = true; Console::info('Ignoring realtime logs because build has been canceled'); return; @@ -643,7 +625,7 @@ class Builds extends Action // Get only valid UTF8 part - removes leftover half-multibytes causing SQL errors $logs = \mb_substr($logs, 0, null, 'UTF-8'); - $currentLogs = $build->getAttribute('logs', ''); + $currentLogs = $deployment->getAttribute('logs', ''); $streamLogs = \str_replace("\\n", "{APPWRITE_LINEBREAK_PLACEHOLDER}", $logs); foreach (\explode("\n", $streamLogs) as $streamLog) { @@ -658,8 +640,8 @@ class Builds extends Action $currentLogs .= $streamParts[1]; } - $build = $build->setAttribute('logs', $currentLogs); - $build = $dbForProject->updateDocument('builds', $build->getId(), $build); + $deployment = $deployment->setAttribute('logs', $currentLogs); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); /** * Send realtime Event @@ -667,12 +649,12 @@ class Builds extends Action $target = Realtime::fromPayload( // Pass first, most verbose event pattern event: $allEvents[0], - payload: $build, + payload: $deployment, project: $project ); Realtime::send( projectId: 'console', - payload: $build->getArrayCopy(), + payload: $deployment->getArrayCopy(), events: $allEvents, channels: $target['channels'], roles: $target['roles'] @@ -688,7 +670,7 @@ class Builds extends Action }), ]); - if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { + if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { Console::info('Build has been canceled'); return; } @@ -706,26 +688,26 @@ class Builds extends Action } /** Update the build document */ - $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); - $build->setAttribute('endTime', $endTime); - $build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); - $build->setAttribute('status', 'ready'); - $build->setAttribute('path', $response['path']); - $build->setAttribute('size', $response['size']); + $deployment->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); + $deployment->setAttribute('endTime', $endTime); + $deployment->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); + $deployment->setAttribute('status', 'ready'); + $deployment->setAttribute('buildPath', $response['path']); + $deployment->setAttribute('buildSize', $response['size']); $logs = ''; foreach ($response['output'] as $log) { $logs .= $log['content']; } - $build->setAttribute('logs', $logs); + $deployment->setAttribute('logs', $logs); - $build = $dbForProject->updateDocument('builds', $buildId, $build); + $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); if ($isVcsEnabled) { $this->runGitAction('ready', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform); } - Console::success("Build id: $buildId created"); + Console::success("Build id: $deploymentId created"); if ($resource->getCollection() === 'sites') { try { @@ -884,7 +866,7 @@ class Builds extends Action } - if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { + if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { Console::info('Build has been canceled'); return; } @@ -901,20 +883,19 @@ class Builds extends Action Authorization::skip(fn () => $dbForPlatform->updateDocument('schedules', $schedule->getId(), $schedule)); } } catch (\Throwable $th) { - if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { + if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { Console::info('Build has been canceled'); return; } $endTime = DateTime::now(); $durationEnd = \microtime(true); - $build->setAttribute('endTime', $endTime); - $build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); - $build->setAttribute('status', 'failed'); + $deployment->setAttribute('endTime', $endTime); + $deployment->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); + $deployment->setAttribute('status', 'failed'); + $deployment->setAttribute('logs', "" . $th->getMessage()); - $build->setAttribute('logs', "[31m" . $th->getMessage() . "[0m"); - - $build = $dbForProject->updateDocument('builds', $buildId, $build); + $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); if ($isVcsEnabled) { $this->runGitAction('failed', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform); @@ -926,26 +907,26 @@ class Builds extends Action $target = Realtime::fromPayload( // Pass first, most verbose event pattern event: $allEvents[0], - payload: $build, + payload: $deployment, project: $project ); Realtime::send( projectId: 'console', - payload: $build->getArrayCopy(), + payload: $deployment->getArrayCopy(), events: $allEvents, channels: $target['channels'], roles: $target['roles'] ); $this->sendUsage( resource:$resource, - build: $build, + deployment: $deployment, project: $project, queue: $queueForStatsUsage ); } } - protected function sendUsage(Document $resource, Document $build, Document $project, StatsUsage $queue): void + protected function sendUsage(Document $resource, Document $deployment, Document $project, StatsUsage $queue): void { $key = match($resource->getCollection()) { 'functions' => 'functionInternalId', @@ -976,32 +957,32 @@ class Builds extends Action ] }; - switch ($build->getAttribute('status')) { + switch ($deployment->getAttribute('status')) { case 'ready': $queue ->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$deployment->getAttribute('duration', 0) * 1000) ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS), 1) // per function - ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$build->getAttribute('duration', 0) * 1000); + ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$deployment->getAttribute('duration', 0) * 1000); break; case 'failed': $queue ->addMetric(METRIC_BUILDS_FAILED, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$deployment->getAttribute('duration', 0) * 1000) ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsFailed']), 1) // per function - ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsComputeFailed']), (int)$build->getAttribute('duration', 0) * 1000); + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsComputeFailed']), (int)$deployment->getAttribute('duration', 0) * 1000); break; } $queue ->addMetric(METRIC_BUILDS, 1) // per project - ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) - ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(METRIC_BUILDS_STORAGE, $deployment->getAttribute('buildSize', 0)) + ->addMetric(METRIC_BUILDS_COMPUTE, (int)$deployment->getAttribute('duration', 0) * 1000) + ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function - ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0)) - ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0)) + ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$deployment->getAttribute('duration', 0) * 1000) + ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) ->setProject($project) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php index f4b6b73784..1d84288203 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php @@ -80,12 +80,7 @@ class Get extends Action switch ($type) { case 'output': - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); - if ($build->isEmpty()) { - throw new Exception(Exception::BUILD_NOT_FOUND); - } - - $path = $build->getAttribute('path', ''); + $path = $deployment->getAttribute('buildPath', ''); $device = $deviceForBuilds; break; case 'source': diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Get.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Get.php index c7929b2e7a..8c3beaf2c0 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Get.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Get.php @@ -69,13 +69,6 @@ class Get extends Action throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); - $deployment->setAttribute('status', $build->getAttribute('status', 'waiting')); - $deployment->setAttribute('buildLogs', $build->getAttribute('logs', '')); - $deployment->setAttribute('buildTime', $build->getAttribute('duration', 0)); - $deployment->setAttribute('buildSize', $build->getAttribute('size', 0)); - $deployment->setAttribute('size', $deployment->getAttribute('size', 0)); - $response->dynamic($deployment, Response::MODEL_DEPLOYMENT); } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php index 0a15053df1..d4d836af7d 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php @@ -13,8 +13,6 @@ use Utopia\App; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Helpers\ID; -use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; @@ -48,7 +46,7 @@ class Update extends Action responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, - model: Response::MODEL_BUILD, + model: Response::MODEL_DEPLOYMENT, ) ] )) @@ -75,46 +73,19 @@ class Update extends Action throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } - $build = Authorization::skip(fn () => $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', ''))); - - if ($build->isEmpty()) { - $buildId = ID::unique(); - $build = $dbForProject->createDocument('builds', new Document([ - '$id' => $buildId, - '$permissions' => [], - 'startTime' => DateTime::now(), - 'deploymentInternalId' => $deployment->getInternalId(), - 'deploymentId' => $deployment->getId(), - 'status' => 'canceled', - 'path' => '', - 'runtime' => $site->getAttribute('framework'), - 'source' => $deployment->getAttribute('path', ''), - 'sourceType' => '', - 'logs' => '', - 'duration' => 0, - 'size' => 0 - ])); - - $deployment->setAttribute('buildId', $build->getId()); - $deployment->setAttribute('buildInternalId', $build->getInternalId()); - $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); - } else { - if (\in_array($build->getAttribute('status'), ['ready', 'failed'])) { - throw new Exception(Exception::BUILD_ALREADY_COMPLETED); - } - - $startTime = new \DateTime($build->getAttribute('startTime')); - $endTime = new \DateTime('now'); - $duration = $endTime->getTimestamp() - $startTime->getTimestamp(); - - $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttributes([ - 'endTime' => DateTime::now(), - 'duration' => $duration, - 'status' => 'canceled' - ])); + if (\in_array($deployment->getAttribute('status'), ['ready', 'failed'])) { + throw new Exception(Exception::BUILD_ALREADY_COMPLETED); } - $dbForProject->purgeCachedDocument('deployments', $deployment->getId()); + $startTime = new \DateTime($deployment->getAttribute('startTime')); + $endTime = new \DateTime('now'); + $duration = $endTime->getTimestamp() - $startTime->getTimestamp(); + + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttributes([ + 'endTime' => DateTime::now(), + 'duration' => $duration, + 'status' => 'canceled' + ])); try { $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); @@ -130,6 +101,6 @@ class Update extends Action ->setParam('siteId', $site->getId()) ->setParam('deploymentId', $deployment->getId()); - $response->dynamic($build, Response::MODEL_BUILD); + $response->dynamic($deployment, Response::MODEL_DEPLOYMENT); } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/XList.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/XList.php index c29b14b840..e041554132 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/XList.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/XList.php @@ -110,15 +110,6 @@ class XList extends Action $results = $dbForProject->find('deployments', $queries); $total = $dbForProject->count('deployments', $filterQueries, APP_LIMIT_COUNT); - foreach ($results as $result) { - $build = $dbForProject->getDocument('builds', $result->getAttribute('buildId', '')); - $result->setAttribute('status', $build->getAttribute('status', 'processing')); - $result->setAttribute('buildLogs', $build->getAttribute('logs', '')); - $result->setAttribute('buildTime', $build->getAttribute('duration', 0)); - $result->setAttribute('buildSize', $build->getAttribute('size', 0)); - $result->setAttribute('size', $result->getAttribute('size', 0)); - } - $response->dynamic(new Document([ 'deployments' => $results, 'total' => $total, diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Deployment/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Deployment/Update.php index fcb0532929..611cdcb104 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Deployment/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Deployment/Update.php @@ -65,7 +65,6 @@ class Update extends Base { $site = $dbForProject->getDocument('sites', $siteId); $deployment = $dbForProject->getDocument('deployments', $deploymentId); - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); if ($site->isEmpty()) { throw new Exception(Exception::SITE_NOT_FOUND); @@ -75,11 +74,7 @@ class Update extends Base throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } - if ($build->isEmpty()) { - throw new Exception(Exception::BUILD_NOT_FOUND); - } - - if ($build->getAttribute('status') !== 'ready') { + if ($deployment->getAttribute('status') !== 'ready') { throw new Exception(Exception::BUILD_NOT_READY); } diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 09fe98dd7f..fe12c90c4a 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -784,9 +784,10 @@ class Deletes extends Action $deploymentIds = []; $this->deleteByGroup('deployments', [ Query::equal('resourceInternalId', [$siteInternalId]) - ], $dbForProject, function (Document $document) use ($project, $certificates, $deviceForSites, $deviceForFiles, $dbForPlatform, &$deploymentInternalIds) { + ], $dbForProject, function (Document $document) use ($project, $certificates, $deviceForSites, $deviceForBuilds, $deviceForFiles, $dbForPlatform, &$deploymentInternalIds) { $deploymentInternalIds[] = $document->getInternalId(); $deploymentIds[] = $document->getId(); + $this->deleteBuildFiles($deviceForBuilds, $document); $this->deleteDeploymentFiles($deviceForSites, $document); $this->deleteDeploymentScreenshots($deviceForFiles, $dbForPlatform, $document); $this->deleteDeploymentRules($dbForPlatform, $document, $project, $certificates); @@ -806,18 +807,6 @@ class Deletes extends Action }); } - /** - * Delete builds - */ - Console::info("Deleting builds for site " . $siteId); - foreach ($deploymentInternalIds as $deploymentInternalId) { - $this->deleteByGroup('builds', [ - Query::equal('deploymentInternalId', [$deploymentInternalId]) - ], $dbForProject, function (Document $document) use ($deviceForBuilds) { - $this->deleteBuildFiles($deviceForBuilds, $document); - }); - } - /** * Delete VCS Repositories and VCS Comments */ @@ -881,25 +870,13 @@ class Deletes extends Action $deploymentInternalIds = []; $this->deleteByGroup('deployments', [ Query::equal('resourceInternalId', [$functionInternalId]) - ], $dbForProject, function (Document $document) use ($dbForPlatform, $project, $certificates, $deviceForFunctions, &$deploymentInternalIds) { + ], $dbForProject, function (Document $document) use ($dbForPlatform, $project, $certificates, $deviceForFunctions, $deviceForBuilds, &$deploymentInternalIds) { $deploymentInternalIds[] = $document->getInternalId(); $this->deleteDeploymentFiles($deviceForFunctions, $document); + $this->deleteBuildFiles($deviceForBuilds, $document); $this->deleteDeploymentRules($dbForPlatform, $document, $project, $certificates); }); - /** - * Delete builds - */ - Console::info("Deleting builds for function " . $functionId); - - foreach ($deploymentInternalIds as $deploymentInternalId) { - $this->deleteByGroup('builds', [ - Query::equal('deploymentInternalId', [$deploymentInternalId]) - ], $dbForProject, function (Document $document) use ($deviceForBuilds) { - $this->deleteBuildFiles($deviceForBuilds, $document); - }); - } - /** * Delete Executions */ @@ -1028,13 +1005,13 @@ class Deletes extends Action * @param Document $build * @return void */ - private function deleteBuildFiles(Device $device, Document $build): void + private function deleteBuildFiles(Device $device, Document $deployment): void { - $buildId = $build->getId(); - $buildPath = $build->getAttribute('path', ''); + $deploymentId = $deployment->getId(); + $buildPath = $deployment->getAttribute('buildPath', ''); if (empty($buildPath)) { - Console::info("No build files for build " . $buildId); + Console::info("No build files for deployment " . $deploymentId); return; } @@ -1085,15 +1062,9 @@ class Deletes extends Action $this->deleteDeploymentScreenshots($deviceForFiles, $dbForPlatform, $document); /** - * Delete builds + * Delete deployment build */ - Console::info("Deleting builds for deployment " . $deploymentId); - - $this->deleteByGroup('builds', [ - Query::equal('deploymentInternalId', [$deploymentInternalId]) - ], $dbForProject, function (Document $document) use ($deviceForBuilds) { - $this->deleteBuildFiles($deviceForBuilds, $document); - }); + $this->deleteBuildFiles($deviceForBuilds, $document); /** * Delete rules associated with the deployment diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index f6d441ef3c..008bd0c8ec 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -346,19 +346,7 @@ class Functions extends Action return; } - $buildId = $deployment->getAttribute('buildId', ''); - - $log->addTag('buildId', $buildId); - - /** Check if build has exists */ - $build = $dbForProject->getDocument('builds', $buildId); - if ($build->isEmpty()) { - $errorMessage = 'The execution could not be completed because a corresponding deployment was not found. A function deployment needs to be created before it can be executed. Please create a deployment for your function and try again.'; - $this->fail($errorMessage, $dbForProject, $function, $trigger, $path, $method, $user, $jwt, $event); - return; - } - - if ($build->getAttribute('status') !== 'ready') { + if ($deployment->getAttribute('status') !== 'ready') { $errorMessage = 'The execution could not be completed because the build is not ready. Please wait for the build to complete and try again.'; $this->fail($errorMessage, $dbForProject, $function, $trigger, $path, $method, $user, $jwt, $event); return; @@ -513,7 +501,7 @@ class Functions extends Action variables: $vars, timeout: $function->getAttribute('timeout', 0), image: $runtime['image'], - source: $build->getAttribute('path', ''), + source: $deployment->getAttribute('buildPath', ''), entrypoint: $deployment->getAttribute('entrypoint', ''), version: $version, path: $path, diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index a6101522fb..fa8d7490ea 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -268,15 +268,13 @@ class StatsResources extends Action protected function countForFunctions(Database $dbForProject, Database $dbForLogs, string $region) { $deploymentsStorage = $dbForProject->sum('deployments', 'size'); - $buildsStorage = $dbForProject->sum('builds', 'size'); + $buildsStorage = $dbForProject->sum('deployments', 'buildSize'); $this->createStatsDocuments($region, METRIC_DEPLOYMENTS_STORAGE, $deploymentsStorage); $this->createStatsDocuments($region, METRIC_BUILDS_STORAGE, $buildsStorage); $deployments = $dbForProject->count('deployments'); - $builds = $dbForProject->count('builds'); $this->createStatsDocuments($region, METRIC_DEPLOYMENTS, $deployments); - $this->createStatsDocuments($region, METRIC_BUILDS, $builds); - + $this->createStatsDocuments($region, METRIC_BUILDS, $deployments); $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForLogs, $region) { $functionDeploymentsStorage = $dbForProject->sum('deployments', 'size', [ @@ -302,9 +300,8 @@ class StatsResources extends Action $this->foreachDocument($dbForProject, 'deployments', [ Query::equal('resourceInternalId', [$function->getInternalId()]), Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), - ], function (Document $deployment) use ($dbForProject, &$functionBuildsStorage): void { - $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); - $functionBuildsStorage += $build->getAttribute('size', 0); + ], function (Document $deployment) use (&$functionBuildsStorage): void { + $functionBuildsStorage += $deployment->getAttribute('buildSize', 0); }); $this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $functionBuildsStorage); diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index da222822e0..d93c2178d8 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -30,7 +30,6 @@ use Appwrite\Utopia\Response\Model\AuthProvider; use Appwrite\Utopia\Response\Model\BaseList; use Appwrite\Utopia\Response\Model\Branch; use Appwrite\Utopia\Response\Model\Bucket; -use Appwrite\Utopia\Response\Model\Build; use Appwrite\Utopia\Response\Model\Collection; use Appwrite\Utopia\Response\Model\ConsoleVariables; use Appwrite\Utopia\Response\Model\Continent; @@ -272,8 +271,6 @@ class Response extends SwooleResponse public const MODEL_DEPLOYMENT_LIST = 'deploymentList'; public const MODEL_EXECUTION = 'execution'; public const MODEL_EXECUTION_LIST = 'executionList'; - public const MODEL_BUILD = 'build'; - public const MODEL_BUILD_LIST = 'buildList'; // Not used anywhere yet public const MODEL_FUNC_PERMISSIONS = 'funcPermissions'; public const MODEL_HEADERS = 'headers'; public const MODEL_SPECIFICATION = 'specification'; @@ -381,7 +378,6 @@ class Response extends SwooleResponse ->setModel(new BaseList('Runtimes List', self::MODEL_RUNTIME_LIST, 'runtimes', self::MODEL_RUNTIME)) ->setModel(new BaseList('Deployments List', self::MODEL_DEPLOYMENT_LIST, 'deployments', self::MODEL_DEPLOYMENT)) ->setModel(new BaseList('Executions List', self::MODEL_EXECUTION_LIST, 'executions', self::MODEL_EXECUTION)) - ->setModel(new BaseList('Builds List', self::MODEL_BUILD_LIST, 'builds', self::MODEL_BUILD)) // Not used anywhere yet ->setModel(new BaseList('Projects List', self::MODEL_PROJECT_LIST, 'projects', self::MODEL_PROJECT, true, false)) ->setModel(new BaseList('Webhooks List', self::MODEL_WEBHOOK_LIST, 'webhooks', self::MODEL_WEBHOOK, true, false)) ->setModel(new BaseList('API Keys List', self::MODEL_KEY_LIST, 'keys', self::MODEL_KEY, true, false)) @@ -461,7 +457,6 @@ class Response extends SwooleResponse ->setModel(new FrameworkAdapter()) ->setModel(new Deployment()) ->setModel(new Execution()) - ->setModel(new Build()) ->setModel(new Project()) ->setModel(new Webhook()) ->setModel(new Key()) diff --git a/src/Appwrite/Utopia/Response/Model/Build.php b/src/Appwrite/Utopia/Response/Model/Build.php deleted file mode 100644 index d80c17645a..0000000000 --- a/src/Appwrite/Utopia/Response/Model/Build.php +++ /dev/null @@ -1,94 +0,0 @@ -addRule('$id', [ - 'type' => self::TYPE_STRING, - 'description' => 'Build ID.', - 'default' => '', - 'example' => '5e5ea5c16897e', - ]) - ->addRule('deploymentId', [ - 'type' => self::TYPE_STRING, - 'description' => 'The deployment that created this build.', - 'default' => '', - 'example' => '5e5ea5c16897e', - ]) - // Build Status - // Failed - The deployment build has failed. More details can usually be found in buildStderr - // Ready - The deployment build was successful and the deployment is ready to be deployed - // Processing - The deployment is currently waiting to have a build triggered - // Building - The deployment is currently being built - ->addRule('status', [ - 'type' => self::TYPE_STRING, - 'description' => 'The build status. There are a few different types and each one means something different. \nFailed - The deployment build has failed. More details can usually be found in buildStderr\nReady - The deployment build was successful and the deployment is ready to be deployed\nProcessing - The deployment is currently waiting to have a build triggered\nBuilding - The deployment is currently being built', - 'default' => '', - 'example' => 'ready', - ]) - ->addRule('stdout', [ - 'type' => self::TYPE_STRING, - 'description' => 'The stdout of the build.', - 'default' => '', - 'example' => '', - ]) - ->addRule('stderr', [ - 'type' => self::TYPE_STRING, - 'description' => 'The stderr of the build.', - 'default' => '', - 'example' => '', - ]) - ->addRule('startTime', [ - 'type' => self::TYPE_DATETIME, - 'description' => 'The deployment creation date in ISO 8601 format.', - 'default' => '', - 'example' => self::TYPE_DATETIME_EXAMPLE, - ]) - ->addRule('endTime', [ - 'type' => self::TYPE_DATETIME, - 'description' => 'The time the build was finished in ISO 8601 format.', - 'default' => '', - 'example' => self::TYPE_DATETIME_EXAMPLE, - ]) - ->addRule('duration', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'The build duration in seconds.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('size', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'The code size in bytes.', - 'default' => 0, - 'example' => 128, - ]) - ; - } - - /** - * Get Name - * - * @return string - */ - public function getName(): string - { - return 'Build'; - } - - /** - * Get Type - * - * @return string - */ - public function getType(): string - { - return Response::MODEL_BUILD; - } -} From 794c6536fb9b9806d9e0845cd5a177d5c5542d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 6 Mar 2025 17:55:53 +0100 Subject: [PATCH 15/31] Fixbugs --- app/config/collections/projects.php | 6 ++-- .../Http/Deployments/Status/Update.php | 2 +- .../Modules/Functions/Workers/Builds.php | 29 +++++++++---------- .../Sites/Http/Deployments/Status/Update.php | 2 +- .../Services/Sites/SitesCustomServerTest.php | 3 ++ 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 5fd997bb47..bae7dc20dd 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1587,7 +1587,7 @@ return [ '$id' => ID::custom('runtime'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 2048, + 'size' => Database::LENGTH_KEY, 'signed' => true, 'required' => true, 'default' => '', @@ -1598,7 +1598,7 @@ return [ '$id' => ID::custom('status'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 256, + 'size' => 16, 'signed' => true, 'required' => true, 'default' => 'processing', @@ -1609,7 +1609,7 @@ return [ '$id' => ID::custom('buildPath'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 2048, + 'size' => Database::LENGTH_KEY, 'signed' => true, 'required' => false, 'default' => '', diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php index 5b58a972a3..10b4974022 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php @@ -85,7 +85,7 @@ class Update extends Action $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttributes([ 'endTime' => DateTime::now(), - 'duration' => $duration, + 'buildTime' => $duration, 'status' => 'canceled' ])); diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index f66c4b4638..3c7c0df633 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -197,7 +197,6 @@ class Builds extends Action $deployment->setAttribute('startTime', $startTime); $deployment->setAttribute('status', 'processing'); $deployment->setAttribute('runtime', $resource->getAttribute('runtime')); - $deployment->setAttribute('sourceType', \strtolower($deviceForFunctions->getType())); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); $source = $deployment->getAttribute('path', ''); @@ -625,7 +624,7 @@ class Builds extends Action // Get only valid UTF8 part - removes leftover half-multibytes causing SQL errors $logs = \mb_substr($logs, 0, null, 'UTF-8'); - $currentLogs = $deployment->getAttribute('logs', ''); + $currentLogs = $deployment->getAttribute('buildLogs', ''); $streamLogs = \str_replace("\\n", "{APPWRITE_LINEBREAK_PLACEHOLDER}", $logs); foreach (\explode("\n", $streamLogs) as $streamLog) { @@ -640,7 +639,7 @@ class Builds extends Action $currentLogs .= $streamParts[1]; } - $deployment = $deployment->setAttribute('logs', $currentLogs); + $deployment = $deployment->setAttribute('buildLogs', $currentLogs); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); /** @@ -690,7 +689,7 @@ class Builds extends Action /** Update the build document */ $deployment->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); $deployment->setAttribute('endTime', $endTime); - $deployment->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); + $deployment->setAttribute('buildTime', \intval(\ceil($durationEnd - $durationStart))); $deployment->setAttribute('status', 'ready'); $deployment->setAttribute('buildPath', $response['path']); $deployment->setAttribute('buildSize', $response['size']); @@ -699,7 +698,7 @@ class Builds extends Action foreach ($response['output'] as $log) { $logs .= $log['content']; } - $deployment->setAttribute('logs', $logs); + $deployment->setAttribute('buildLogs', $logs); $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); @@ -891,9 +890,9 @@ class Builds extends Action $endTime = DateTime::now(); $durationEnd = \microtime(true); $deployment->setAttribute('endTime', $endTime); - $deployment->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); + $deployment->setAttribute('buildTime', \intval(\ceil($durationEnd - $durationStart))); $deployment->setAttribute('status', 'failed'); - $deployment->setAttribute('logs', "" . $th->getMessage()); + $deployment->setAttribute('buildLogs', "" . $th->getMessage()); $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment); @@ -961,28 +960,28 @@ class Builds extends Action case 'ready': $queue ->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$deployment->getAttribute('duration', 0) * 1000) + ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$deployment->getAttribute('buildTime', 0) * 1000) ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS), 1) // per function - ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$deployment->getAttribute('duration', 0) * 1000); + ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$deployment->getAttribute('buildTime', 0) * 1000); break; case 'failed': $queue ->addMetric(METRIC_BUILDS_FAILED, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$deployment->getAttribute('duration', 0) * 1000) + ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$deployment->getAttribute('buildTime', 0) * 1000) ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsFailed']), 1) // per function - ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsComputeFailed']), (int)$deployment->getAttribute('duration', 0) * 1000); + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsComputeFailed']), (int)$deployment->getAttribute('buildTime', 0) * 1000); break; } $queue ->addMetric(METRIC_BUILDS, 1) // per project ->addMetric(METRIC_BUILDS_STORAGE, $deployment->getAttribute('buildSize', 0)) - ->addMetric(METRIC_BUILDS_COMPUTE, (int)$deployment->getAttribute('duration', 0) * 1000) - ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(METRIC_BUILDS_COMPUTE, (int)$deployment->getAttribute('buildTime', 0) * 1000) + ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildTime', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0)) - ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$deployment->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$deployment->getAttribute('buildTime', 0) * 1000) + ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildTime', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) ->setProject($project) ->trigger(); } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php index d4d836af7d..5a43ae5409 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php @@ -83,7 +83,7 @@ class Update extends Action $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttributes([ 'endTime' => DateTime::now(), - 'duration' => $duration, + 'buildTime' => $duration, 'status' => 'canceled' ])); diff --git a/tests/e2e/Services/Sites/SitesCustomServerTest.php b/tests/e2e/Services/Sites/SitesCustomServerTest.php index c9adc3a116..2547a055e5 100644 --- a/tests/e2e/Services/Sites/SitesCustomServerTest.php +++ b/tests/e2e/Services/Sites/SitesCustomServerTest.php @@ -709,6 +709,9 @@ class SitesCustomServerTest extends Scope }, 100000, 250); $deployment = $this->cancelDeployment($siteId, $deploymentId); + \var_dump($siteId); + \var_dump($deploymentId); + \var_dump($deployment); $this->assertEquals(200, $deployment['headers']['status-code']); $this->assertEquals('canceled', $deployment['body']['status']); From a7fc8cf84b024b896740dd0757684411ccd03f67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 6 Mar 2025 18:17:01 +0100 Subject: [PATCH 16/31] Fix functions --- app/config/collections/projects.php | 22 ------------------- .../Functions/Http/Deployments/Create.php | 2 -- .../Http/Deployments/Duplicate/Create.php | 9 ++++++-- .../Modules/Functions/Workers/Builds.php | 1 - .../Modules/Sites/Http/Deployments/Create.php | 2 -- .../Http/Deployments/Duplicate/Create.php | 10 +++++++-- 6 files changed, 15 insertions(+), 31 deletions(-) diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index bae7dc20dd..0995b9e07a 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1192,17 +1192,6 @@ return [ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('buildInternalId'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], [ 'array' => false, '$id' => ID::custom('entrypoint'), @@ -1583,17 +1572,6 @@ return [ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('runtime'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => true, - 'default' => '', - 'array' => false, - 'filters' => [], - ], [ '$id' => ID::custom('status'), 'type' => Database::VAR_STRING, diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php index 6885ecde6c..940bdc295f 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php @@ -217,7 +217,6 @@ class Create extends Action 'resourceInternalId' => $function->getInternalId(), 'resourceId' => $function->getId(), 'resourceType' => 'functions', - 'buildInternalId' => '', 'entrypoint' => $entrypoint, 'commands' => $commands, 'path' => $path, @@ -248,7 +247,6 @@ class Create extends Action 'resourceInternalId' => $function->getInternalId(), 'resourceId' => $function->getId(), 'resourceType' => 'functions', - 'buildInternalId' => '', 'entrypoint' => $entrypoint, 'commands' => $commands, 'path' => $path, diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php index 624a410dcd..b8e5962b5f 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php @@ -91,12 +91,17 @@ class Create extends Action $deployment = $dbForProject->createDocument('deployments', $deployment->setAttributes([ '$internalId' => '', '$id' => $deploymentId, - 'buildId' => '', - 'buildInternalId' => '', 'path' => $destination, 'entrypoint' => $function->getAttribute('entrypoint'), 'commands' => $function->getAttribute('commands', ''), 'search' => implode(' ', [$deploymentId, $function->getAttribute('entrypoint')]), + 'startTime' => null, + 'endTime' => null, + 'buildTime' => null, + 'buildSize' => null, + 'status' => 'processing', + 'buildPath' => '', + 'buildLogs' => '', ])); $queueForBuilds diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 3c7c0df633..8a39baa87b 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -196,7 +196,6 @@ class Builds extends Action $deployment->setAttribute('startTime', $startTime); $deployment->setAttribute('status', 'processing'); - $deployment->setAttribute('runtime', $resource->getAttribute('runtime')); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); $source = $deployment->getAttribute('path', ''); diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php index fc70f2fe25..baa5158b2b 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php @@ -216,7 +216,6 @@ class Create extends Action 'resourceInternalId' => $site->getInternalId(), 'resourceId' => $site->getId(), 'resourceType' => 'sites', - 'buildInternalId' => '', 'installCommand' => $installCommand, 'buildCommand' => $buildCommand, 'outputDirectory' => $outputDirectory, @@ -265,7 +264,6 @@ class Create extends Action 'resourceInternalId' => $site->getInternalId(), 'resourceId' => $site->getId(), 'resourceType' => 'sites', - 'buildInternalId' => '', 'installCommand' => $installCommand, 'buildCommand' => $buildCommand, 'outputDirectory' => $outputDirectory, diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php index b30c07838f..9ed92e38df 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php @@ -93,14 +93,20 @@ class Create extends Action '$internalId' => '', '$id' => $deploymentId, 'buildId' => '', - 'buildInternalId' => '', 'path' => $destination, 'buildCommand' => $site->getAttribute('buildCommand', ''), 'installCommand' => $site->getAttribute('installCommand', ''), 'outputDirectory' => $site->getAttribute('outputDirectory', ''), 'search' => implode(' ', [$deploymentId]), 'screenshotLight' => '', - 'screenshotDark' => '' + 'screenshotDark' => '', + 'startTime' => null, + 'endTime' => null, + 'buildTime' => null, + 'buildSize' => null, + 'status' => 'processing', + 'buildPath' => '', + 'buildLogs' => '', ])); // Preview deployments for sites From 50ec55d595629680022e09981775119f7d100a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 6 Mar 2025 18:35:57 +0100 Subject: [PATCH 17/31] Fix site tests --- .../Modules/Sites/Http/Deployments/Duplicate/Create.php | 1 - .../Utopia/Database/Validator/Queries/Deployments.php | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php index 9ed92e38df..b78c3f928b 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php @@ -92,7 +92,6 @@ class Create extends Action $deployment = $dbForProject->createDocument('deployments', $deployment->setAttributes([ '$internalId' => '', '$id' => $deploymentId, - 'buildId' => '', 'path' => $destination, 'buildCommand' => $site->getAttribute('buildCommand', ''), 'installCommand' => $site->getAttribute('installCommand', ''), diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Deployments.php b/src/Appwrite/Utopia/Database/Validator/Queries/Deployments.php index 42aed88ef6..55303c4003 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Deployments.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Deployments.php @@ -6,12 +6,14 @@ class Deployments extends Base { public const ALLOWED_ATTRIBUTES = [ 'size', - 'buildId', + 'status', 'activate', 'entrypoint', 'commands', 'type', - 'size' + 'size', + 'buildSize', + 'buildTime' ]; /** From 6a584c83c1b7b073885152031f051276eb460f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 7 Mar 2025 20:14:11 +0100 Subject: [PATCH 18/31] Attribute renaming --- app/config/collections/projects.php | 42 +++++------------- app/controllers/api/vcs.php | 17 ++++++-- .../Platform/Modules/Compute/Base.php | 15 +++++-- .../Functions/Http/Deployments/Create.php | 28 ++++++------ .../Http/Deployments/Duplicate/Create.php | 10 ++--- .../Http/Deployments/Status/Update.php | 6 +-- .../Http/Deployments/Template/Create.php | 2 +- .../Modules/Functions/Workers/Builds.php | 43 +++++++++---------- .../Modules/Sites/Http/Deployments/Create.php | 42 ++++++++++-------- .../Http/Deployments/Duplicate/Create.php | 21 ++++++--- .../Sites/Http/Deployments/Status/Update.php | 6 +-- .../Http/Deployments/Template/Create.php | 13 ++++-- .../Validator/Queries/Deployments.php | 9 ++-- .../Utopia/Response/Model/Deployment.php | 4 +- .../Functions/FunctionsCustomServerTest.php | 2 +- .../Services/Sites/SitesCustomServerTest.php | 2 +- 16 files changed, 136 insertions(+), 126 deletions(-) diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 0995b9e07a..671bf4b6ff 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1205,18 +1205,7 @@ return [ ], [ 'array' => false, - '$id' => ID::custom('commands'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 2048, - 'signed' => true, - 'required' => false, - 'default' => null, - 'filters' => [], - ], - [ - 'array' => false, - '$id' => ID::custom('buildCommand'), + '$id' => ID::custom('buildCommands'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => 16384, @@ -1227,7 +1216,7 @@ return [ ], [ 'array' => false, - '$id' => ID::custom('installCommand'), + '$id' => ID::custom('buildOutput'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => 16384, @@ -1237,18 +1226,7 @@ return [ 'filters' => [], ], [ - 'array' => false, - '$id' => ID::custom('outputDirectory'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 16384, - 'signed' => true, - 'required' => false, - 'default' => null, - 'filters' => [], - ], - [ - '$id' => ID::custom('path'), + '$id' => ID::custom('sourcePath'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => 2048, @@ -1441,7 +1419,7 @@ return [ 'array' => false, ], [ - '$id' => ID::custom('size'), + '$id' => ID::custom('sourceSize'), 'type' => Database::VAR_INTEGER, 'format' => '', 'size' => 0, @@ -1452,7 +1430,7 @@ return [ 'filters' => [], ], [ - '$id' => ID::custom('metadata'), + '$id' => ID::custom('sourceMetadata'), 'type' => Database::VAR_STRING, 'format' => '', 'size' => 16384, // https://tools.ietf.org/html/rfc4288#section-4.2 @@ -1463,7 +1441,7 @@ return [ 'filters' => ['json'], ], [ - '$id' => ID::custom('chunksTotal'), + '$id' => ID::custom('sourceChunksTotal'), 'type' => Database::VAR_INTEGER, 'format' => '', 'size' => 0, @@ -1474,7 +1452,7 @@ return [ 'filters' => [], ], [ - '$id' => ID::custom('chunksUploaded'), + '$id' => ID::custom('sourceChunksUploaded'), 'type' => Database::VAR_INTEGER, 'format' => '', 'size' => 0, @@ -1529,7 +1507,7 @@ return [ 'filters' => [], ], [ - '$id' => ID::custom('startTime'), + '$id' => ID::custom('buildStartAt'), 'type' => Database::VAR_DATETIME, 'format' => '', 'size' => 0, @@ -1540,7 +1518,7 @@ return [ 'filters' => ['datetime'], ], [ - '$id' => ID::custom('endTime'), + '$id' => ID::custom('buildEndAt'), 'type' => Database::VAR_DATETIME, 'format' => '', 'size' => 0, @@ -1551,7 +1529,7 @@ return [ 'filters' => ['datetime'], ], [ - '$id' => ID::custom('buildTime'), + '$id' => ID::custom('buildDuration'), 'type' => Database::VAR_INTEGER, 'format' => '', 'size' => 0, diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 0f277a4661..80dff6ceaf 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -196,6 +196,17 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId $providerRepositoryOwner = $pullRequestResponse['head']['repo']['name']; } + $commands = []; + if (!empty($resource->getAttribute('buildCommand', ''))) { + $commands[] = $resource->getAttribute('buildCommand', ''); + } + if (!empty($resource->getAttribute('installCommand', ''))) { + $commands[] = $resource->getAttribute('installCommand', ''); + } + if (!empty($resource->getAttribute('commands', ''))) { + $commands[] = $resource->getAttribute('commands', ''); + } + $deployment = $dbForProject->createDocument('deployments', new Document([ '$id' => $deploymentId, '$permissions' => [ @@ -207,10 +218,8 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId 'resourceInternalId' => $resourceInternalId, 'resourceType' => $resourceCollection, 'entrypoint' => $resource->getAttribute('entrypoint', ''), - 'commands' => $resource->getAttribute('commands', ''), - 'installCommand' => $resource->getAttribute('installCommand', ''), - 'buildCommand' => $resource->getAttribute('buildCommand', ''), - 'outputDirectory' => $resource->getAttribute('outputDirectory', ''), + 'buildCommands' => \implode(' && ', $commands), + 'buildOutput' => $resource->getAttribute('outputDirectory', ''), 'type' => 'vcs', 'installationId' => $installationId, 'installationInternalId' => $installationInternalId, diff --git a/src/Appwrite/Platform/Modules/Compute/Base.php b/src/Appwrite/Platform/Modules/Compute/Base.php index 7880df189e..2204e171cf 100644 --- a/src/Appwrite/Platform/Modules/Compute/Base.php +++ b/src/Appwrite/Platform/Modules/Compute/Base.php @@ -70,7 +70,7 @@ class Base extends Action 'resourceInternalId' => $function->getInternalId(), 'resourceType' => 'functions', 'entrypoint' => $entrypoint, - 'commands' => $function->getAttribute('commands', ''), + 'buildCommands' => $function->getAttribute('commands', ''), 'type' => 'vcs', 'installationId' => $installation->getId(), 'installationInternalId' => $installation->getInternalId(), @@ -139,6 +139,14 @@ class Base extends Action } } + $commands = []; + if (!empty($site->getAttribute('buildCommand', ''))) { + $commands[] = $site->getAttribute('buildCommand', ''); + } + if (!empty($site->getAttribute('installCommand', ''))) { + $commands[] = $site->getAttribute('installCommand', ''); + } + $deployment = $dbForProject->createDocument('deployments', new Document([ '$id' => $deploymentId, '$permissions' => [ @@ -149,9 +157,8 @@ class Base extends Action 'resourceId' => $site->getId(), 'resourceInternalId' => $site->getInternalId(), 'resourceType' => 'sites', - 'buildCommand' => $site->getAttribute('buildCommand', ''), - 'installCommand' => $site->getAttribute('installCommand', ''), - 'outputDirectory' => $site->getAttribute('outputDirectory', ''), + 'buildCommands' => implode(' && ', $commands), + 'buildOutput' => $site->getAttribute('outputDirectory', ''), 'type' => 'vcs', 'installationId' => $installation->getId(), 'installationInternalId' => $installation->getInternalId(), diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php index 940bdc295f..580959589b 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Create.php @@ -174,8 +174,8 @@ class Create extends Action $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; if (!$deployment->isEmpty()) { - $chunks = $deployment->getAttribute('chunksTotal', 1); - $metadata = $deployment->getAttribute('metadata', []); + $chunks = $deployment->getAttribute('sourceChunksTotal', 1); + $metadata = $deployment->getAttribute('sourceMetadata', []); if ($chunk === -1) { $chunk = $chunks; } @@ -218,16 +218,16 @@ class Create extends Action 'resourceId' => $function->getId(), 'resourceType' => 'functions', 'entrypoint' => $entrypoint, - 'commands' => $commands, - 'path' => $path, - 'size' => $fileSize, + 'buildCommands' => $commands, + 'sourcePath' => $path, + 'sourceSize' => $fileSize, 'search' => implode(' ', [$deploymentId, $entrypoint]), 'activate' => $activate, - 'metadata' => $metadata, + 'sourceMetadata' => $metadata, 'type' => $type ])); } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('size', $fileSize)->setAttribute('metadata', $metadata)); + $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('sourceSize', $fileSize)->setAttribute('sourceMetadata', $metadata)); } // Start the build @@ -248,18 +248,18 @@ class Create extends Action 'resourceId' => $function->getId(), 'resourceType' => 'functions', 'entrypoint' => $entrypoint, - 'commands' => $commands, - 'path' => $path, - 'size' => $fileSize, - 'chunksTotal' => $chunks, - 'chunksUploaded' => $chunksUploaded, + 'buildCommands' => $commands, + 'sourcePath' => $path, + 'sourceSize' => $fileSize, + 'sourceChunksTotal' => $chunks, + 'sourceChunksUploaded' => $chunksUploaded, 'search' => implode(' ', [$deploymentId, $entrypoint]), 'activate' => $activate, - 'metadata' => $metadata, + 'sourceMetadata' => $metadata, 'type' => $type ])); } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('chunksUploaded', $chunksUploaded)->setAttribute('metadata', $metadata)); + $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('sourceChunksUploaded', $chunksUploaded)->setAttribute('sourceMetadata', $metadata)); } } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php index b8e5962b5f..05730c42e9 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php @@ -91,13 +91,13 @@ class Create extends Action $deployment = $dbForProject->createDocument('deployments', $deployment->setAttributes([ '$internalId' => '', '$id' => $deploymentId, - 'path' => $destination, + 'sourcePath' => $destination, 'entrypoint' => $function->getAttribute('entrypoint'), - 'commands' => $function->getAttribute('commands', ''), + 'buildCommands' => $function->getAttribute('commands', ''), 'search' => implode(' ', [$deploymentId, $function->getAttribute('entrypoint')]), - 'startTime' => null, - 'endTime' => null, - 'buildTime' => null, + 'buildStartAt' => null, + 'buildEndAt' => null, + 'buildDuration' => null, 'buildSize' => null, 'status' => 'processing', 'buildPath' => '', diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php index 10b4974022..6c8cacdcfd 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Status/Update.php @@ -79,13 +79,13 @@ class Update extends Action throw new Exception(Exception::BUILD_ALREADY_COMPLETED); } - $startTime = new \DateTime($deployment->getAttribute('startTime')); + $startTime = new \DateTime($deployment->getAttribute('buildStartAt')); $endTime = new \DateTime('now'); $duration = $endTime->getTimestamp() - $startTime->getTimestamp(); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttributes([ - 'endTime' => DateTime::now(), - 'buildTime' => $duration, + 'buildEndAt' => DateTime::now(), + 'buildDuration' => $duration, 'status' => 'canceled' ])); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php index 9a61693e15..1f6fa364f3 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Template/Create.php @@ -130,7 +130,7 @@ class Create extends Base 'resourceInternalId' => $function->getInternalId(), 'resourceType' => 'functions', 'entrypoint' => $function->getAttribute('entrypoint', ''), - 'commands' => $function->getAttribute('commands', ''), + 'buildCommands' => $function->getAttribute('commands', ''), 'type' => 'manual', 'search' => implode(' ', [$deploymentId, $function->getAttribute('entrypoint', '')]), 'activate' => $activate, diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 8a39baa87b..698b0de774 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -194,11 +194,11 @@ class Builds extends Action $deploymentId = $deployment->getId(); - $deployment->setAttribute('startTime', $startTime); + $deployment->setAttribute('buildStartAt', $startTime); $deployment->setAttribute('status', 'processing'); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); - $source = $deployment->getAttribute('path', ''); + $source = $deployment->getAttribute('sourcePath', ''); $installationId = $deployment->getAttribute('installationId', ''); $providerRepositoryId = $deployment->getAttribute('providerRepositoryId', ''); $providerCommitHash = $deployment->getAttribute('providerCommitHash', ''); @@ -267,8 +267,8 @@ class Builds extends Action $directorySize = $device->getFileSize($source); $deployment - ->setAttribute('path', $source) - ->setAttribute('size', $directorySize); + ->setAttribute('sourcePath', $source) + ->setAttribute('sourceSize', $directorySize); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); } } elseif ($isVcsEnabled) { @@ -427,8 +427,8 @@ class Builds extends Action $directorySize = $device->getFileSize($source); $deployment - ->setAttribute('path', $source) - ->setAttribute('size', $directorySize); + ->setAttribute('sourcePath', $source) + ->setAttribute('sourceSize', $directorySize); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); $this->runGitAction('processing', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform); @@ -686,9 +686,9 @@ class Builds extends Action } /** Update the build document */ - $deployment->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); - $deployment->setAttribute('endTime', $endTime); - $deployment->setAttribute('buildTime', \intval(\ceil($durationEnd - $durationStart))); + $deployment->setAttribute('buildStartAt', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); + $deployment->setAttribute('buildEndAt', $endTime); + $deployment->setAttribute('buildDuration', \intval(\ceil($durationEnd - $durationStart))); $deployment->setAttribute('status', 'ready'); $deployment->setAttribute('buildPath', $response['path']); $deployment->setAttribute('buildSize', $response['size']); @@ -888,8 +888,8 @@ class Builds extends Action $endTime = DateTime::now(); $durationEnd = \microtime(true); - $deployment->setAttribute('endTime', $endTime); - $deployment->setAttribute('buildTime', \intval(\ceil($durationEnd - $durationStart))); + $deployment->setAttribute('buildEndAt', $endTime); + $deployment->setAttribute('buildDuration', \intval(\ceil($durationEnd - $durationStart))); $deployment->setAttribute('status', 'failed'); $deployment->setAttribute('buildLogs', "" . $th->getMessage()); @@ -959,28 +959,28 @@ class Builds extends Action case 'ready': $queue ->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$deployment->getAttribute('buildTime', 0) * 1000) + ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$deployment->getAttribute('buildDuration', 0) * 1000) ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS), 1) // per function - ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$deployment->getAttribute('buildTime', 0) * 1000); + ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$deployment->getAttribute('buildDuration', 0) * 1000); break; case 'failed': $queue ->addMetric(METRIC_BUILDS_FAILED, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$deployment->getAttribute('buildTime', 0) * 1000) + ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$deployment->getAttribute('buildDuration', 0) * 1000) ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsFailed']), 1) // per function - ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsComputeFailed']), (int)$deployment->getAttribute('buildTime', 0) * 1000); + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsComputeFailed']), (int)$deployment->getAttribute('buildDuration', 0) * 1000); break; } $queue ->addMetric(METRIC_BUILDS, 1) // per project ->addMetric(METRIC_BUILDS_STORAGE, $deployment->getAttribute('buildSize', 0)) - ->addMetric(METRIC_BUILDS_COMPUTE, (int)$deployment->getAttribute('buildTime', 0) * 1000) - ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildTime', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(METRIC_BUILDS_COMPUTE, (int)$deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0)) - ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$deployment->getAttribute('buildTime', 0) * 1000) - ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildTime', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(str_replace($key, $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) ->setProject($project) ->trigger(); } @@ -1012,7 +1012,7 @@ class Builds extends Action protected function getCommand(Document $resource, Document $deployment): string { if ($resource->getCollection() === 'functions') { - return $deployment->getAttribute('commands', ''); + return $deployment->getAttribute('buildCommands', ''); } elseif ($resource->getCollection() === 'sites') { $commands = []; @@ -1033,8 +1033,7 @@ class Builds extends Action } $commands[] = $envCommand; - $commands[] = $deployment->getAttribute('installCommand', ''); - $commands[] = $deployment->getAttribute('buildCommand', ''); + $commands[] = $deployment->getAttribute('buildCommands', ''); $commands[] = $bundleCommand; $commands = array_filter($commands, fn ($command) => !empty($command)); diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php index baa5158b2b..3405f8a03c 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php @@ -173,8 +173,8 @@ class Create extends Action $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; if (!$deployment->isEmpty()) { - $chunks = $deployment->getAttribute('chunksTotal', 1); - $metadata = $deployment->getAttribute('metadata', []); + $chunks = $deployment->getAttribute('sourceChunksTotal', 1); + $metadata = $deployment->getAttribute('sourceMetadata', []); if ($chunk === -1) { $chunk = $chunks; } @@ -188,6 +188,14 @@ class Create extends Action $type = $request->getHeader('x-sdk-language') === 'cli' ? 'cli' : 'manual'; + $commands = []; + if (!empty($installCommand)) { + $commands[] = $installCommand; + } + if (!empty($buildCommand)) { + $commands[] = $buildCommand; + } + if ($chunksUploaded === $chunks) { if ($activate) { // Remove deploy for all other deployments. @@ -216,14 +224,13 @@ class Create extends Action 'resourceInternalId' => $site->getInternalId(), 'resourceId' => $site->getId(), 'resourceType' => 'sites', - 'installCommand' => $installCommand, - 'buildCommand' => $buildCommand, - 'outputDirectory' => $outputDirectory, - 'path' => $path, - 'size' => $fileSize, + 'buildCommands' => $commands, + 'buildOutput' => $outputDirectory, + 'sourcePath' => $path, + 'sourceSize' => $fileSize, 'search' => implode(' ', [$deploymentId]), 'activate' => $activate, - 'metadata' => $metadata, + 'sourceMetadata' => $metadata, 'type' => $type ])); @@ -244,7 +251,7 @@ class Create extends Action ])) ); } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('size', $fileSize)->setAttribute('metadata', $metadata)); + $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('sourceSize', $fileSize)->setAttribute('sourceMetadata', $metadata)); } // Start the build @@ -264,16 +271,15 @@ class Create extends Action 'resourceInternalId' => $site->getInternalId(), 'resourceId' => $site->getId(), 'resourceType' => 'sites', - 'installCommand' => $installCommand, - 'buildCommand' => $buildCommand, - 'outputDirectory' => $outputDirectory, - 'path' => $path, - 'size' => $fileSize, - 'chunksTotal' => $chunks, - 'chunksUploaded' => $chunksUploaded, + 'buildCommands' => $commands, + 'buildOutput' => $outputDirectory, + 'sourcePath' => $path, + 'sourceSize' => $fileSize, + 'sourceChunksTotal' => $chunks, + 'sourceChunksUploaded' => $chunksUploaded, 'search' => implode(' ', [$deploymentId]), 'activate' => $activate, - 'metadata' => $metadata, + 'sourceMetadata' => $metadata, 'type' => $type ])); @@ -294,7 +300,7 @@ class Create extends Action ])) ); } else { - $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('chunksUploaded', $chunksUploaded)->setAttribute('metadata', $metadata)); + $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('sourceChunksUploaded', $chunksUploaded)->setAttribute('sourceMetadata', $metadata)); } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php index b78c3f928b..c2e6df64dd 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php @@ -88,20 +88,27 @@ class Create extends Action $destination = $deviceForSites->getPath($deploymentId . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); $deviceForSites->transfer($path, $destination, $deviceForSites); + $commands = []; + if (!empty($site->getAttribute('buildCommand', ''))) { + $commands[] = $site->getAttribute('buildCommand', ''); + } + if (!empty($site->getAttribute('installCommand', ''))) { + $commands[] = $site->getAttribute('installCommand', ''); + } + $deployment->removeAttribute('$internalId'); $deployment = $dbForProject->createDocument('deployments', $deployment->setAttributes([ '$internalId' => '', '$id' => $deploymentId, - 'path' => $destination, - 'buildCommand' => $site->getAttribute('buildCommand', ''), - 'installCommand' => $site->getAttribute('installCommand', ''), - 'outputDirectory' => $site->getAttribute('outputDirectory', ''), + 'sourcePath' => $destination, + 'buildCommands' => \implode(' && ', $commands), + 'buildOutput' => $site->getAttribute('outputDirectory', ''), 'search' => implode(' ', [$deploymentId]), 'screenshotLight' => '', 'screenshotDark' => '', - 'startTime' => null, - 'endTime' => null, - 'buildTime' => null, + 'buildStartAt' => null, + 'buildEndAt' => null, + 'buildDuration' => null, 'buildSize' => null, 'status' => 'processing', 'buildPath' => '', diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php index 5a43ae5409..632ad51c9b 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Status/Update.php @@ -77,13 +77,13 @@ class Update extends Action throw new Exception(Exception::BUILD_ALREADY_COMPLETED); } - $startTime = new \DateTime($deployment->getAttribute('startTime')); + $startTime = new \DateTime($deployment->getAttribute('buildStartAt')); $endTime = new \DateTime('now'); $duration = $endTime->getTimestamp() - $startTime->getTimestamp(); $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttributes([ - 'endTime' => DateTime::now(), - 'buildTime' => $duration, + 'buildEndAt' => DateTime::now(), + 'buildDuration' => $duration, 'status' => 'canceled' ])); diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php index 5fa7a68e82..7ab20cecb0 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Template/Create.php @@ -121,6 +121,14 @@ class Create extends Base return; } + $commands = []; + if (!empty($site->getAttribute('installCommand', ''))) { + $commands[] = $site->getAttribute('installCommand', ''); + } + if (!empty($site->getAttribute('buildCommand', ''))) { + $commands[] = $site->getAttribute('buildCommand', ''); + } + $deploymentId = ID::unique(); $deployment = $dbForProject->createDocument('deployments', new Document([ '$id' => $deploymentId, @@ -132,9 +140,8 @@ class Create extends Base 'resourceId' => $site->getId(), 'resourceInternalId' => $site->getInternalId(), 'resourceType' => 'sites', - 'installCommand' => $site->getAttribute('installCommand', ''), - 'buildCommand' => $site->getAttribute('buildCommand', ''), - 'outputDirectory' => $site->getAttribute('outputDirectory', ''), + 'buildCommands' => \implode(' && ', $commands), + 'buildOutput' => $site->getAttribute('outputDirectory', ''), 'type' => 'manual', 'search' => implode(' ', [$deploymentId]), 'activate' => $activate, diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Deployments.php b/src/Appwrite/Utopia/Database/Validator/Queries/Deployments.php index 55303c4003..75035ad501 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Deployments.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Deployments.php @@ -5,15 +5,12 @@ namespace Appwrite\Utopia\Database\Validator\Queries; class Deployments extends Base { public const ALLOWED_ATTRIBUTES = [ - 'size', + 'buildSize', + 'sourceSize', + 'buildDuration', 'status', 'activate', - 'entrypoint', - 'commands', 'type', - 'size', - 'buildSize', - 'buildTime' ]; /** diff --git a/src/Appwrite/Utopia/Response/Model/Deployment.php b/src/Appwrite/Utopia/Response/Model/Deployment.php index f42ce6d334..2b14977a7e 100644 --- a/src/Appwrite/Utopia/Response/Model/Deployment.php +++ b/src/Appwrite/Utopia/Response/Model/Deployment.php @@ -52,7 +52,7 @@ class Deployment extends Model 'default' => '', 'example' => 'index.js', ]) - ->addRule('size', [ + ->addRule('sourceSize', [ 'type' => self::TYPE_INTEGER, 'description' => 'The code size in bytes.', 'default' => 0, @@ -100,7 +100,7 @@ class Deployment extends Model 'default' => '', 'example' => 'Compiling source files...', ]) - ->addRule('buildTime', [ + ->addRule('buildDuration', [ 'type' => self::TYPE_INTEGER, 'description' => 'The current build time in seconds.', 'default' => 0, diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 2056d1f689..c79fd59923 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -832,7 +832,7 @@ class FunctionsCustomServerTest extends Scope $deployment = $this->getDeployment($data['functionId'], $data['deploymentId']); $this->assertEquals(200, $deployment['headers']['status-code']); - $this->assertGreaterThan(0, $deployment['body']['buildTime']); + $this->assertGreaterThan(0, $deployment['body']['buildDuration']); $this->assertNotEmpty($deployment['body']['status']); $this->assertNotEmpty($deployment['body']['buildLogs']); $this->assertArrayHasKey('size', $deployment['body']); diff --git a/tests/e2e/Services/Sites/SitesCustomServerTest.php b/tests/e2e/Services/Sites/SitesCustomServerTest.php index 2547a055e5..57f5f56bf5 100644 --- a/tests/e2e/Services/Sites/SitesCustomServerTest.php +++ b/tests/e2e/Services/Sites/SitesCustomServerTest.php @@ -989,7 +989,7 @@ class SitesCustomServerTest extends Scope $deployment = $this->getDeployment($siteId, $deploymentId); $this->assertEquals(200, $deployment['headers']['status-code']); - $this->assertGreaterThan(0, $deployment['body']['buildTime']); + $this->assertGreaterThan(0, $deployment['body']['buildDuration']); $this->assertNotEmpty($deployment['body']['status']); $this->assertNotEmpty($deployment['body']['buildLogs']); $this->assertArrayHasKey('size', $deployment['body']); From c1ee2f2b411fb63056bfebec1569f6d441ea5f6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 7 Mar 2025 21:04:24 +0100 Subject: [PATCH 19/31] Failing test fixes --- app/config/collections/projects.php | 33 +++++++++++-- .../Functions/Http/Deployments/Delete.php | 4 +- .../Http/Deployments/Download/Get.php | 2 +- .../Http/Deployments/Duplicate/Create.php | 2 +- .../Modules/Sites/Http/Deployments/Create.php | 4 +- .../Modules/Sites/Http/Deployments/Delete.php | 4 +- .../Sites/Http/Deployments/Download/Get.php | 2 +- .../Http/Deployments/Duplicate/Create.php | 2 +- src/Appwrite/Platform/Workers/Deletes.php | 2 +- .../Functions/FunctionsCustomServerTest.php | 46 ++++++------------- .../Services/Sites/SitesCustomServerTest.php | 23 ++++------ 11 files changed, 65 insertions(+), 59 deletions(-) diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 671bf4b6ff..fb8af40348 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1240,7 +1240,7 @@ return [ '$id' => ID::custom('type'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 2048, + 'size' => 32, 'signed' => true, 'required' => true, 'default' => null, @@ -1607,9 +1607,23 @@ return [ 'orders' => [], ], [ - '$id' => ID::custom('_key_size'), + '$id' => ID::custom('_key_sourceSize'), 'type' => Database::INDEX_KEY, - 'attributes' => ['size'], + 'attributes' => ['sourceSize'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => ID::custom('_key_buildSize'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['buildSize'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => ID::custom('_key_buildDuration'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['buildDuration'], 'lengths' => [], 'orders' => [Database::ORDER_ASC], ], @@ -1620,6 +1634,19 @@ return [ 'lengths' => [], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => ID::custom('_key_type'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['type'], + 'lengths' => [32], + 'orders' => [Database::ORDER_ASC], + ], [ + '$id' => ID::custom('_key_status'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['status'], + 'lengths' => [16], + 'orders' => [Database::ORDER_ASC], + ], ], ], diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Delete.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Delete.php index 5bde7903d9..c02c470980 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Delete.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Delete.php @@ -83,8 +83,8 @@ class Delete extends Action throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove deployment from DB'); } - if (!empty($deployment->getAttribute('path', ''))) { - if (!($deviceForFunctions->delete($deployment->getAttribute('path', '')))) { + if (!empty($deployment->getAttribute('sourcePath', ''))) { + if (!($deviceForFunctions->delete($deployment->getAttribute('sourcePath', '')))) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove deployment from storage'); } } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php index d528bc9995..4fa279a3d7 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Download/Get.php @@ -85,7 +85,7 @@ class Get extends Action $device = $deviceForBuilds; break; case 'source': - $path = $deployment->getAttribute('path', ''); + $path = $deployment->getAttribute('sourcePath', ''); $device = $deviceForFunctions; break; } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php index 05730c42e9..b942ba9ab7 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Deployments/Duplicate/Create.php @@ -77,7 +77,7 @@ class Create extends Action throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } - $path = $deployment->getAttribute('path'); + $path = $deployment->getAttribute('sourcePath'); if (empty($path) || !$deviceForFunctions->exists($path)) { throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php index 3405f8a03c..a899e1aa3e 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php @@ -224,7 +224,7 @@ class Create extends Action 'resourceInternalId' => $site->getInternalId(), 'resourceId' => $site->getId(), 'resourceType' => 'sites', - 'buildCommands' => $commands, + 'buildCommands' => \implode(' && ', $commands), 'buildOutput' => $outputDirectory, 'sourcePath' => $path, 'sourceSize' => $fileSize, @@ -271,7 +271,7 @@ class Create extends Action 'resourceInternalId' => $site->getInternalId(), 'resourceId' => $site->getId(), 'resourceType' => 'sites', - 'buildCommands' => $commands, + 'buildCommands' => \implode(' && ', $commands), 'buildOutput' => $outputDirectory, 'sourcePath' => $path, 'sourceSize' => $fileSize, diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Delete.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Delete.php index fe16dafae1..7065ac1917 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Delete.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Delete.php @@ -83,8 +83,8 @@ class Delete extends Action throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove deployment from DB'); } - if (!empty($deployment->getAttribute('path', ''))) { - if (!($deviceForSites->delete($deployment->getAttribute('path', '')))) { + if (!empty($deployment->getAttribute('sourcePath', ''))) { + if (!($deviceForSites->delete($deployment->getAttribute('sourcePath', '')))) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove deployment from storage'); } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php index 1d84288203..6d969f883f 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php @@ -84,7 +84,7 @@ class Get extends Action $device = $deviceForBuilds; break; case 'source': - $path = $deployment->getAttribute('path', ''); + $path = $deployment->getAttribute('sourcePath', ''); $device = $deviceForSites; break; } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php index c2e6df64dd..f32056cbe9 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/Duplicate/Create.php @@ -78,7 +78,7 @@ class Create extends Action throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } - $path = $deployment->getAttribute('path'); + $path = $deployment->getAttribute('sourcePath'); if (empty($path) || !$deviceForSites->exists($path)) { throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); } diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index fe12c90c4a..a875f9e940 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -976,7 +976,7 @@ class Deletes extends Action private function deleteDeploymentFiles(Device $device, Document $deployment): void { $deploymentId = $deployment->getId(); - $deploymentPath = $deployment->getAttribute('path', ''); + $deploymentPath = $deployment->getAttribute('sourcePath', ''); if (empty($deploymentPath)) { Console::info("No deployment files for deployment " . $deploymentId); diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index c79fd59923..e9e7a5c7d8 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -382,7 +382,7 @@ class FunctionsCustomServerTest extends Scope $lastDeployment = $deployments['body']['deployments'][0]; $this->assertNotEmpty($lastDeployment['$id']); - $this->assertEquals(0, $lastDeployment['size']); + $this->assertEquals(0, $lastDeployment['sourceSize']); $deploymentId = $lastDeployment['$id']; @@ -598,10 +598,10 @@ class FunctionsCustomServerTest extends Scope $this->assertNotEmpty($largeTag['body']['$id']); $this->assertEquals(true, (new DatetimeValidator())->isValid($largeTag['body']['$createdAt'])); $this->assertEquals('index.php', $largeTag['body']['entrypoint']); - $this->assertGreaterThan(1024 * 1024 * 5, $largeTag['body']['size']); // ~7MB video file - $this->assertLessThan(1024 * 1024 * 10, $largeTag['body']['size']); // ~7MB video file + $this->assertGreaterThan(1024 * 1024 * 5, $largeTag['body']['sourceSize']); // ~7MB video file + $this->assertLessThan(1024 * 1024 * 10, $largeTag['body']['sourceSize']); // ~7MB video file - $deploymentSize = $largeTag['body']['size']; + $deploymentSize = $largeTag['body']['sourceSize']; $deploymentId = $largeTag['body']['$id']; $this->assertEventually(function () use ($functionId, $deploymentId, $deploymentSize) { @@ -609,7 +609,7 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(200, $deployment['headers']['status-code']); $this->assertEquals('ready', $deployment['body']['status']); - $this->assertEquals($deploymentSize, $deployment['body']['size']); + $this->assertEquals($deploymentSize, $deployment['body']['sourceSize']); $this->assertGreaterThan(1024 * 1024 * 10, $deployment['body']['buildSize']); // ~7MB video file + 10MB sample file }, 500000, 1000); @@ -655,7 +655,7 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals($deployments['body']['total'], 3); $this->assertIsArray($deployments['body']['deployments']); $this->assertCount(3, $deployments['body']['deployments']); - $this->assertArrayHasKey('size', $deployments['body']['deployments'][0]); + $this->assertArrayHasKey('sourceSize', $deployments['body']['deployments'][0]); $this->assertArrayHasKey('buildSize', $deployments['body']['deployments'][0]); $deployments = $this->listDeployments($functionId, [ @@ -676,24 +676,6 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals($deployments['headers']['status-code'], 200); $this->assertCount(2, $deployments['body']['deployments']); - $deployments = $this->listDeployments($functionId, [ - 'queries' => [ - Query::equal('entrypoint', ['index.php'])->toString(), - ], - ]); - - $this->assertEquals($deployments['headers']['status-code'], 200); - $this->assertCount(3, $deployments['body']['deployments']); - - $deployments = $this->listDeployments($functionId, [ - 'queries' => [ - Query::equal('entrypoint', ['index.js'])->toString(), - ], - ]); - - $this->assertEquals($deployments['headers']['status-code'], 200); - $this->assertCount(0, $deployments['body']['deployments']); - $deployments = $this->listDeployments($functionId, [ 'search' => 'php-8.0' ]); @@ -744,7 +726,7 @@ class FunctionsCustomServerTest extends Scope $functionId, [ 'queries' => [ - Query::greaterThan('size', 10000)->toString(), + Query::greaterThan('sourceSize', 10000)->toString(), ], ] ); @@ -756,7 +738,7 @@ class FunctionsCustomServerTest extends Scope $functionId, [ 'queries' => [ - Query::greaterThan('size', 0)->toString(), + Query::greaterThan('sourceSize', 0)->toString(), ], ] ); @@ -768,7 +750,7 @@ class FunctionsCustomServerTest extends Scope $functionId, [ 'queries' => [ - Query::greaterThan('size', -100)->toString(), + Query::greaterThan('sourceSize', -100)->toString(), ], ] ); @@ -789,16 +771,16 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(200, $deployments['headers']['status-code']); $this->assertGreaterThanOrEqual(1, $deployments['body']['total']); $this->assertNotEmpty($deployments['body']['deployments'][0]['$id']); - $this->assertNotEmpty($deployments['body']['deployments'][0]['size']); + $this->assertNotEmpty($deployments['body']['deployments'][0]['sourceSize']); $deploymentId = $deployments['body']['deployments'][0]['$id']; - $deploymentSize = $deployments['body']['deployments'][0]['size']; + $deploymentSize = $deployments['body']['deployments'][0]['sourceSize']; $deployments = $this->listDeployments( $functionId, [ 'queries' => [ - Query::equal('size', [$deploymentSize])->toString(), + Query::equal('sourceSize', [$deploymentSize])->toString(), ], ] ); @@ -815,7 +797,7 @@ class FunctionsCustomServerTest extends Scope if (!empty($matchingDeployment)) { $deployment = reset($matchingDeployment); - $this->assertEquals($deploymentSize, $deployment['size']); + $this->assertEquals($deploymentSize, $deployment['sourceSize']); } return $data; @@ -835,7 +817,7 @@ class FunctionsCustomServerTest extends Scope $this->assertGreaterThan(0, $deployment['body']['buildDuration']); $this->assertNotEmpty($deployment['body']['status']); $this->assertNotEmpty($deployment['body']['buildLogs']); - $this->assertArrayHasKey('size', $deployment['body']); + $this->assertArrayHasKey('sourceSize', $deployment['body']); $this->assertArrayHasKey('buildSize', $deployment['body']); /** diff --git a/tests/e2e/Services/Sites/SitesCustomServerTest.php b/tests/e2e/Services/Sites/SitesCustomServerTest.php index 57f5f56bf5..b52847f665 100644 --- a/tests/e2e/Services/Sites/SitesCustomServerTest.php +++ b/tests/e2e/Services/Sites/SitesCustomServerTest.php @@ -598,7 +598,7 @@ class SitesCustomServerTest extends Scope $lastDeployment = $deployments['body']['deployments'][0]; $this->assertNotEmpty($lastDeployment['$id']); - $this->assertEquals(0, $lastDeployment['size']); + $this->assertEquals(0, $lastDeployment['sourceSize']); $deploymentId = $lastDeployment['$id']; @@ -709,9 +709,6 @@ class SitesCustomServerTest extends Scope }, 100000, 250); $deployment = $this->cancelDeployment($siteId, $deploymentId); - \var_dump($siteId); - \var_dump($deploymentId); - \var_dump($deployment); $this->assertEquals(200, $deployment['headers']['status-code']); $this->assertEquals('canceled', $deployment['body']['status']); @@ -815,7 +812,7 @@ class SitesCustomServerTest extends Scope $this->assertEquals($deployments['body']['total'], 2); $this->assertIsArray($deployments['body']['deployments']); $this->assertCount(2, $deployments['body']['deployments']); - $this->assertArrayHasKey('size', $deployments['body']['deployments'][0]); + $this->assertArrayHasKey('sourceSize', $deployments['body']['deployments'][0]); $this->assertArrayHasKey('buildSize', $deployments['body']['deployments'][0]); $deployments = $this->listDeployments($siteId, [ @@ -876,7 +873,7 @@ class SitesCustomServerTest extends Scope $siteId, [ 'queries' => [ - Query::greaterThan('size', 10000)->toString(), + Query::greaterThan('sourceSize', 10000)->toString(), ], ] ); @@ -888,7 +885,7 @@ class SitesCustomServerTest extends Scope $siteId, [ 'queries' => [ - Query::greaterThan('size', 0)->toString(), + Query::greaterThan('sourceSize', 0)->toString(), ], ] ); @@ -900,7 +897,7 @@ class SitesCustomServerTest extends Scope $siteId, [ 'queries' => [ - Query::greaterThan('size', -100)->toString(), + Query::greaterThan('sourceSize', -100)->toString(), ], ] ); @@ -921,16 +918,16 @@ class SitesCustomServerTest extends Scope $this->assertEquals(200, $deployments['headers']['status-code']); $this->assertGreaterThanOrEqual(1, $deployments['body']['total']); $this->assertNotEmpty($deployments['body']['deployments'][0]['$id']); - $this->assertNotEmpty($deployments['body']['deployments'][0]['size']); + $this->assertNotEmpty($deployments['body']['deployments'][0]['sourceSize']); $deploymentId = $deployments['body']['deployments'][0]['$id']; - $deploymentSize = $deployments['body']['deployments'][0]['size']; + $deploymentSize = $deployments['body']['deployments'][0]['sourceSize']; $deployments = $this->listDeployments( $siteId, [ 'queries' => [ - Query::equal('size', [$deploymentSize])->toString(), + Query::equal('sourceSize', [$deploymentSize])->toString(), ], ] ); @@ -947,7 +944,7 @@ class SitesCustomServerTest extends Scope if (!empty($matchingDeployment)) { $deployment = reset($matchingDeployment); - $this->assertEquals($deploymentSize, $deployment['size']); + $this->assertEquals($deploymentSize, $deployment['sourceSize']); } $this->cleanupDeployment($siteId, $deploymentIdActive); @@ -992,7 +989,7 @@ class SitesCustomServerTest extends Scope $this->assertGreaterThan(0, $deployment['body']['buildDuration']); $this->assertNotEmpty($deployment['body']['status']); $this->assertNotEmpty($deployment['body']['buildLogs']); - $this->assertArrayHasKey('size', $deployment['body']); + $this->assertArrayHasKey('sourceSize', $deployment['body']); $this->assertArrayHasKey('buildSize', $deployment['body']); /** From a5457da91c4cf98c622d72552880eee851bceae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 7 Mar 2025 21:12:13 +0100 Subject: [PATCH 20/31] Linter fix --- app/config/collections/projects.php | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index fb8af40348..0612b50566 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -1635,18 +1635,18 @@ return [ 'orders' => [Database::ORDER_ASC], ], [ - '$id' => ID::custom('_key_type'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['type'], - 'lengths' => [32], - 'orders' => [Database::ORDER_ASC], - ], [ - '$id' => ID::custom('_key_status'), - 'type' => Database::INDEX_KEY, - 'attributes' => ['status'], - 'lengths' => [16], - 'orders' => [Database::ORDER_ASC], - ], + '$id' => ID::custom('_key_type'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['type'], + 'lengths' => [32], + 'orders' => [Database::ORDER_ASC], + ], [ + '$id' => ID::custom('_key_status'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['status'], + 'lengths' => [16], + 'orders' => [Database::ORDER_ASC], + ], ], ], From 5e30fc535058f392b37954ceaa13cec948e4dc05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 7 Mar 2025 21:20:37 +0100 Subject: [PATCH 21/31] Fix graphQL test --- tests/e2e/Services/GraphQL/Base.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/GraphQL/Base.php b/tests/e2e/Services/GraphQL/Base.php index 5c38a10472..121d40156e 100644 --- a/tests/e2e/Services/GraphQL/Base.php +++ b/tests/e2e/Services/GraphQL/Base.php @@ -1568,7 +1568,7 @@ trait Base _id buildId entrypoint - size + buildSize status buildLogs } From 1d983a949e0e616a12a810a28b2d9b661d75c05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 7 Mar 2025 22:36:13 +0100 Subject: [PATCH 22/31] Merge site & function specs --- app/config/runtimes/specifications.php | 51 -------- .../{frameworks => }/specifications.php | 0 app/controllers/general.php | 2 +- app/init.php | 3 +- src/Appwrite/Functions/Specification.php | 16 --- .../Modules/Compute}/Specification.php | 2 +- .../Compute/Validator/Specification.php} | 8 +- .../Functions/Http/Executions/Create.php | 2 +- .../Functions/Http/Functions/Create.php | 6 +- .../Functions/Http/Functions/Update.php | 8 +- .../Functions/Http/Specifications/XList.php | 20 ++-- .../Modules/Functions/Workers/Builds.php | 2 +- .../Modules/Sites/Http/Sites/Create.php | 6 +- .../Modules/Sites/Http/Sites/Update.php | 8 +- .../Sites/Http/Specifications/XList.php | 76 ++++++++++++ src/Appwrite/Platform/Workers/Functions.php | 2 +- .../Validator/FrameworkSpecification.php | 112 ------------------ 17 files changed, 108 insertions(+), 216 deletions(-) delete mode 100644 app/config/runtimes/specifications.php rename app/config/{frameworks => }/specifications.php (100%) delete mode 100644 src/Appwrite/Functions/Specification.php rename src/Appwrite/{Sites => Platform/Modules/Compute}/Specification.php (94%) rename src/Appwrite/{Functions/Validator/RuntimeSpecification.php => Platform/Modules/Compute/Validator/Specification.php} (88%) create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Specifications/XList.php delete mode 100644 src/Appwrite/Sites/Validator/FrameworkSpecification.php diff --git a/app/config/runtimes/specifications.php b/app/config/runtimes/specifications.php deleted file mode 100644 index d3625db8a2..0000000000 --- a/app/config/runtimes/specifications.php +++ /dev/null @@ -1,51 +0,0 @@ - [ - 'slug' => Specification::S_05VCPU_512MB, - 'memory' => 512, - 'cpus' => 0.5 - ], - Specification::S_1VCPU_512MB => [ - 'slug' => Specification::S_1VCPU_512MB, - 'memory' => 512, - 'cpus' => 1 - ], - Specification::S_1VCPU_1GB => [ - 'slug' => Specification::S_1VCPU_1GB, - 'memory' => 1024, - 'cpus' => 1 - ], - Specification::S_2VCPU_2GB => [ - 'slug' => Specification::S_2VCPU_2GB, - 'memory' => 2048, - 'cpus' => 2 - ], - Specification::S_2VCPU_4GB => [ - 'slug' => Specification::S_2VCPU_4GB, - 'memory' => 4096, - 'cpus' => 2 - ], - Specification::S_4VCPU_4GB => [ - 'slug' => Specification::S_4VCPU_4GB, - 'memory' => 4096, - 'cpus' => 4 - ], - Specification::S_4VCPU_8GB => [ - 'slug' => Specification::S_4VCPU_8GB, - 'memory' => 8192, - 'cpus' => 4 - ], - Specification::S_8VCPU_4GB => [ - 'slug' => Specification::S_8VCPU_4GB, - 'memory' => 4096, - 'cpus' => 8 - ], - Specification::S_8VCPU_8GB => [ - 'slug' => Specification::S_8VCPU_8GB, - 'memory' => 8192, - 'cpus' => 8 - ] -]; diff --git a/app/config/frameworks/specifications.php b/app/config/specifications.php similarity index 100% rename from app/config/frameworks/specifications.php rename to app/config/specifications.php diff --git a/app/controllers/general.php b/app/controllers/general.php index d4728fb5af..1ebb1c80b3 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -174,7 +174,7 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw }; $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); - $spec = Config::getParam('runtime-specifications')[$resource->getAttribute('specification', APP_COMPUTE_SPECIFICATION_DEFAULT)]; + $spec = Config::getParam('specifications')[$resource->getAttribute('specification', APP_COMPUTE_SPECIFICATION_DEFAULT)]; $runtime = match ($type) { 'function' => $runtimes[$resource->getAttribute('runtime')] ?? null, diff --git a/app/init.php b/app/init.php index 3291b160b9..90811013af 100644 --- a/app/init.php +++ b/app/init.php @@ -389,8 +389,7 @@ Config::load('storage-logos', __DIR__ . '/config/storage/logos.php'); Config::load('storage-mimes', __DIR__ . '/config/storage/mimes.php'); Config::load('storage-inputs', __DIR__ . '/config/storage/inputs.php'); Config::load('storage-outputs', __DIR__ . '/config/storage/outputs.php'); -Config::load('runtime-specifications', __DIR__ . '/config/runtimes/specifications.php'); -Config::load('framework-specifications', __DIR__ . '/config/frameworks/specifications.php'); +Config::load('specifications', __DIR__ . '/config/specifications.php'); Config::load('function-templates', __DIR__ . '/config/function-templates.php'); Config::load('site-templates', __DIR__ . '/config/site-templates.php'); diff --git a/src/Appwrite/Functions/Specification.php b/src/Appwrite/Functions/Specification.php deleted file mode 100644 index 50a3c02b62..0000000000 --- a/src/Appwrite/Functions/Specification.php +++ /dev/null @@ -1,16 +0,0 @@ -specifications as $size => $values) { if ($values['cpus'] <= $this->maxCpus && $values['memory'] <= $this->maxMemory) { - if (!empty($this->plan) && array_key_exists('runtimeSpecifications', $this->plan)) { - if (!\in_array($size, $this->plan['runtimeSpecifications'])) { + if (!empty($this->plan) && array_key_exists('specifications', $this->plan)) { + if (!\in_array($size, $this->plan['specifications'])) { continue; } } diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php index f522e2dc62..d59a3322a4 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php @@ -140,7 +140,7 @@ class Create extends Base $version = $function->getAttribute('version', 'v2'); $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); - $spec = Config::getParam('runtime-specifications')[$function->getAttribute('specification', APP_COMPUTE_SPECIFICATION_DEFAULT)]; + $spec = Config::getParam('specifications')[$function->getAttribute('specification', APP_COMPUTE_SPECIFICATION_DEFAULT)]; $runtime = (isset($runtimes[$function->getAttribute('runtime', '')])) ? $runtimes[$function->getAttribute('runtime', '')] : null; diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php index 0507c68068..d2abb97b7e 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Create.php @@ -2,10 +2,10 @@ namespace Appwrite\Platform\Modules\Functions\Http\Functions; +use Appwrite\Compute\Validator\Specification; use Appwrite\Event\Event; use Appwrite\Event\Validator\FunctionEvent; use Appwrite\Extend\Exception; -use Appwrite\Functions\Validator\RuntimeSpecification; use Appwrite\Platform\Modules\Compute\Base; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; @@ -85,9 +85,9 @@ class Create extends Base ->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the function.', true) ->param('providerSilentMode', false, new Boolean(), 'Is the VCS (Version Control System) connection in silent mode for the repo linked to the function? In silent mode, comments will not be made on commits and pull requests.', true) ->param('providerRootDirectory', '', new Text(128, 0), 'Path to function code in the linked repo.', true) - ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new RuntimeSpecification( + ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new Specification( $plan, - Config::getParam('runtime-specifications', []), + Config::getParam('specifications', []), App::getEnv('_APP_COMPUTE_CPUS', APP_COMPUTE_CPUS_DEFAULT), App::getEnv('_APP_COMPUTE_MEMORY', APP_COMPUTE_MEMORY_DEFAULT) ), 'Runtime specification for the function and builds.', true, ['plan']) diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php index 9244b10c8c..cb4d2d553a 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Functions/Update.php @@ -2,11 +2,11 @@ namespace Appwrite\Platform\Modules\Functions\Http\Functions; +use Appwrite\Compute\Validator\Specification; use Appwrite\Event\Build; use Appwrite\Event\Event; use Appwrite\Event\Validator\FunctionEvent; use Appwrite\Extend\Exception; -use Appwrite\Functions\Validator\RuntimeSpecification; use Appwrite\Platform\Modules\Compute\Base; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; @@ -89,9 +89,9 @@ class Update extends Base ->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the function', true) ->param('providerSilentMode', false, new Boolean(), 'Is the VCS (Version Control System) connection in silent mode for the repo linked to the function? In silent mode, comments will not be made on commits and pull requests.', true) ->param('providerRootDirectory', '', new Text(128, 0), 'Path to function code in the linked repo.', true) - ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new RuntimeSpecification( + ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new Specification( $plan, - Config::getParam('runtime-specifications', []), + Config::getParam('specifications', []), App::getEnv('_APP_COMPUTE_CPUS', APP_COMPUTE_CPUS_DEFAULT), App::getEnv('_APP_COMPUTE_MEMORY', APP_COMPUTE_MEMORY_DEFAULT) ), 'Runtime specification for the function and builds.', true, ['plan']) @@ -206,8 +206,6 @@ class Update extends Base $live = false; } - $spec = Config::getParam('runtime-specifications')[$specification] ?? []; - // Enforce Cold Start if spec limits change. if ($function->getAttribute('specification') !== $specification && !empty($function->getAttribute('deployment'))) { $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Specifications/XList.php b/src/Appwrite/Platform/Modules/Functions/Http/Specifications/XList.php index dbaf3daa04..375c99966d 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Specifications/XList.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Specifications/XList.php @@ -19,7 +19,7 @@ class XList extends Base public static function getName() { - return 'listFunctionsSpecifications'; + return 'listSpecifications'; } public function __construct() @@ -28,7 +28,7 @@ class XList extends Base ->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) ->setHttpPath('/v1/functions/specifications') ->groups(['api', 'functions']) - ->desc('List available function runtime specifications') + ->desc('List specifications') ->label('scope', 'functions.read') ->label('resourceType', RESOURCE_TYPE_FUNCTIONS) ->label('sdk', new Method( @@ -52,25 +52,25 @@ class XList extends Base public function action(Response $response, array $plan) { - $allRuntimeSpecs = Config::getParam('runtime-specifications', []); + $allSpecs = Config::getParam('specifications', []); - $runtimeSpecs = []; - foreach ($allRuntimeSpecs as $spec) { + $specs = []; + foreach ($allSpecs as $spec) { $spec['enabled'] = true; - if (array_key_exists('runtimeSpecifications', $plan)) { - $spec['enabled'] = in_array($spec['slug'], $plan['runtimeSpecifications']); + if (array_key_exists('specifications', $plan)) { + $spec['enabled'] = in_array($spec['slug'], $plan['specifications']); } // Only add specs that are within the limits set by environment variables if ($spec['cpus'] <= System::getEnv('_APP_COMPUTE_CPUS', 1) && $spec['memory'] <= System::getEnv('_APP_COMPUTE_MEMORY', 512)) { - $runtimeSpecs[] = $spec; + $specs[] = $spec; } } $response->dynamic(new Document([ - 'specifications' => $runtimeSpecs, - 'total' => count($runtimeSpecs) + 'specifications' => $specs, + 'total' => count($specs) ]), Response::MODEL_SPECIFICATION_LIST); } } diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 53556f60ba..ba922b89c0 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -172,7 +172,7 @@ class Builds extends Action $version = $this->getVersion($resource); $runtime = $this->getRuntime($resource, $version); - $spec = Config::getParam('runtime-specifications')[$resource->getAttribute('specification', APP_COMPUTE_SPECIFICATION_DEFAULT)]; + $spec = Config::getParam('specifications')[$resource->getAttribute('specification', APP_COMPUTE_SPECIFICATION_DEFAULT)]; if ($resource->getCollection() === 'functions' && \is_null($runtime)) { throw new \Exception('Runtime "' . $resource->getAttribute('runtime', '') . '" is not supported'); diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php index c39f618691..eb1c5b38f2 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php @@ -2,13 +2,13 @@ namespace Appwrite\Platform\Modules\Sites\Http\Sites; +use Appwrite\Compute\Validator\Specification; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\Platform\Modules\Compute\Base; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; -use Appwrite\Sites\Validator\FrameworkSpecification; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Response; use Utopia\App; @@ -77,9 +77,9 @@ class Create extends Base ->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the site.', true) ->param('providerSilentMode', false, new Boolean(), 'Is the VCS (Version Control System) connection in silent mode for the repo linked to the site? In silent mode, comments will not be made on commits and pull requests.', true) ->param('providerRootDirectory', '', new Text(128, 0), 'Path to site code in the linked repo.', true) - ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new FrameworkSpecification( + ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new Specification( $plan, - Config::getParam('framework-specifications', []), + Config::getParam('specifications', []), App::getEnv('_APP_COMPUTE_CPUS', APP_COMPUTE_CPUS_DEFAULT), App::getEnv('_APP_COMPUTE_MEMORY', APP_COMPUTE_MEMORY_DEFAULT) ), 'Framework specification for the site and builds.', true, ['plan']) diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php index f9dbdf604c..996a8209b5 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Http\Sites; +use Appwrite\Compute\Validator\Specification; use Appwrite\Event\Build; use Appwrite\Event\Event; use Appwrite\Extend\Exception; @@ -9,7 +10,6 @@ use Appwrite\Platform\Modules\Compute\Base; use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; -use Appwrite\Sites\Validator\FrameworkSpecification; use Appwrite\Utopia\Response; use Executor\Executor; use Utopia\App; @@ -81,9 +81,9 @@ class Update extends Base ->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the site.', true) ->param('providerSilentMode', false, new Boolean(), 'Is the VCS (Version Control System) connection in silent mode for the repo linked to the site? In silent mode, comments will not be made on commits and pull requests.', true) ->param('providerRootDirectory', '', new Text(128, 0), 'Path to site code in the linked repo.', true) - ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new FrameworkSpecification( + ->param('specification', APP_COMPUTE_SPECIFICATION_DEFAULT, fn (array $plan) => new Specification( $plan, - Config::getParam('framework-specifications', []), + Config::getParam('specifications', []), App::getEnv('_APP_COMPUTE_CPUS', APP_COMPUTE_CPUS_DEFAULT), App::getEnv('_APP_COMPUTE_MEMORY', APP_COMPUTE_MEMORY_DEFAULT) ), 'Framework specification for the site and builds.', true, ['plan']) @@ -204,8 +204,6 @@ class Update extends Base $live = false; } - $spec = Config::getParam('framework-specifications')[$specification] ?? []; - // Enforce Cold Start if spec limits change. if ($site->getAttribute('specification') !== $specification && !empty($site->getAttribute('deploymentId'))) { $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Specifications/XList.php b/src/Appwrite/Platform/Modules/Sites/Http/Specifications/XList.php new file mode 100644 index 0000000000..63ee7bec10 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Specifications/XList.php @@ -0,0 +1,76 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/sites/specifications') + ->groups(['api', 'sites']) + ->desc('List specifications') + ->label('scope', 'sites.read') + ->label('resourceType', RESOURCE_TYPE_SITES) + ->label('sdk', new Method( + namespace: 'sites', + name: 'listSpecifications', + description: <<inject('response') + ->inject('plan') + ->callback([$this, 'action']); + } + + public function action(Response $response, array $plan) + { + $allSpecs = Config::getParam('specifications', []); + + $specs = []; + foreach ($allSpecs as $spec) { + $spec['enabled'] = true; + + if (array_key_exists('specifications', $plan)) { + $spec['enabled'] = in_array($spec['slug'], $plan['specifications']); + } + + // Only add specs that are within the limits set by environment variables + if ($spec['cpus'] <= System::getEnv('_APP_COMPUTE_CPUS', 1) && $spec['memory'] <= System::getEnv('_APP_COMPUTE_MEMORY', 512)) { + $specs[] = $spec; + } + } + + $response->dynamic(new Document([ + 'specifications' => $specs, + 'total' => count($specs) + ]), Response::MODEL_SPECIFICATION_LIST); + } +} diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index f6d441ef3c..b0a9512f8e 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -327,7 +327,7 @@ class Functions extends Action $user ??= new Document(); $functionId = $function->getId(); $deploymentId = $function->getAttribute('deployment', ''); - $spec = Config::getParam('runtime-specifications')[$function->getAttribute('specification', APP_COMPUTE_SPECIFICATION_DEFAULT)]; + $spec = Config::getParam('specifications')[$function->getAttribute('specification', APP_COMPUTE_SPECIFICATION_DEFAULT)]; $log->addTag('deploymentId', $deploymentId); diff --git a/src/Appwrite/Sites/Validator/FrameworkSpecification.php b/src/Appwrite/Sites/Validator/FrameworkSpecification.php deleted file mode 100644 index b8dc3b5cfd..0000000000 --- a/src/Appwrite/Sites/Validator/FrameworkSpecification.php +++ /dev/null @@ -1,112 +0,0 @@ -plan = $plan; - $this->specifications = $specifications; - $this->maxCpus = $maxCpus; - $this->maxMemory = $maxMemory; - } - - /** - * Get Allowed Specifications. - * - * Get allowed specifications taking into account the limits set by the environment variables and the plan. - * - * @return array - */ - public function getAllowedSpecifications(): array - { - $allowedSpecifications = []; - - foreach ($this->specifications as $size => $values) { - if ($values['cpus'] <= $this->maxCpus && $values['memory'] <= $this->maxMemory) { - if (!empty($this->plan) && array_key_exists('frameworkSpecifications', $this->plan)) { - if (!\in_array($size, $this->plan['frameworkSpecifications'])) { - continue; - } - } - - $allowedSpecifications[] = $size; - } - } - - return $allowedSpecifications; - } - - /** - * Get Description. - * - * Returns validator description. - * - * @return string - */ - public function getDescription(): string - { - return 'Specification must be one of: ' . implode(', ', $this->getAllowedSpecifications()); - } - - /** - * Is valid. - * - * Returns true if valid or false if not. - * - * @param mixed $value - * - * @return bool - */ - public function isValid($value): bool - { - if (empty($value)) { - return false; - } - - if (!\is_string($value)) { - return false; - } - - if (!\in_array($value, $this->getAllowedSpecifications())) { - return false; - } - - return true; - } - - /** - * Is array. - * - * Function will return true if object is array. - * - * @return bool - */ - public function isArray(): bool - { - return false; - } - - /** - * Get Type. - * - * Returns validator type. - * - * @return string - */ - public function getType(): string - { - return self::TYPE_STRING; - } -} From c6a862a0a31a6db5141e0599c0579c9220524eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 7 Mar 2025 23:17:15 +0100 Subject: [PATCH 23/31] Specifications test, fix failures --- app/config/specifications.php | 2 +- app/init.php | 2 +- .../Modules/Compute/Specification.php | 2 +- .../Compute/Validator/Specification.php | 2 +- .../Functions/Http/Functions/Create.php | 2 +- .../Functions/Http/Functions/Update.php | 2 +- .../Modules/Sites/Http/Sites/Create.php | 2 +- .../Modules/Sites/Http/Sites/Update.php | 2 +- .../Platform/Modules/Sites/Services/Http.php | 3 ++ tests/e2e/General/UsageTest.php | 2 +- .../e2e/Services/Functions/FunctionsBase.php | 9 ++++ .../Functions/FunctionsCustomServerTest.php | 43 +++++++++++++++++-- tests/e2e/Services/Sites/SitesBase.php | 9 ++++ .../Services/Sites/SitesCustomServerTest.php | 39 ++++++++++++++++- 14 files changed, 107 insertions(+), 14 deletions(-) diff --git a/app/config/specifications.php b/app/config/specifications.php index 63ca3ea423..82f5633a96 100644 --- a/app/config/specifications.php +++ b/app/config/specifications.php @@ -1,6 +1,6 @@ [ diff --git a/app/init.php b/app/init.php index 90811013af..6b57035ce9 100644 --- a/app/init.php +++ b/app/init.php @@ -36,13 +36,13 @@ use Appwrite\Event\Realtime; use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; -use Appwrite\Functions\Specification; use Appwrite\GraphQL\Promises\Adapter\Swoole; use Appwrite\GraphQL\Schema; use Appwrite\Hooks\Hooks; use Appwrite\Network\Validator\Email; use Appwrite\Network\Validator\Origin; use Appwrite\OpenSSL\OpenSSL; +use Appwrite\Platform\Modules\Compute\Specification; use Appwrite\PubSub\Adapter\Redis as PubSub; use Appwrite\URL\URL as AppwriteURL; use Appwrite\Utopia\Request; diff --git a/src/Appwrite/Platform/Modules/Compute/Specification.php b/src/Appwrite/Platform/Modules/Compute/Specification.php index 8c9e9b9aad..cf48896a7b 100644 --- a/src/Appwrite/Platform/Modules/Compute/Specification.php +++ b/src/Appwrite/Platform/Modules/Compute/Specification.php @@ -1,6 +1,6 @@ addAction(ListUsage::getName(), new ListUsage()); $this->addAction(GetUsage::getName(), new GetUsage()); + + $this->addAction(ListSpecifications::getName(), new ListSpecifications()); } } diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 00dc790869..a74101f50d 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -2,7 +2,7 @@ namespace Tests\E2E\General; -use Appwrite\Functions\Specification; +use Appwrite\Platform\Modules\Compute\Specification; use Appwrite\Tests\Retry; use CURLFile; use DateTime; diff --git a/tests/e2e/Services/Functions/FunctionsBase.php b/tests/e2e/Services/Functions/FunctionsBase.php index 9f0a5903f3..85e881e5eb 100644 --- a/tests/e2e/Services/Functions/FunctionsBase.php +++ b/tests/e2e/Services/Functions/FunctionsBase.php @@ -389,4 +389,13 @@ trait FunctionsBase return $deployment; } + + protected function listSpecifications(): mixed + { + $specifications = $this->client->call(Client::METHOD_GET, '/functions/specifications', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + return $specifications; + } } diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 2056d1f689..472d6f430e 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -2,7 +2,7 @@ namespace Tests\E2E\Services\Functions; -use Appwrite\Functions\Specification; +use Appwrite\Platform\Modules\Compute\Specification; use Appwrite\Tests\Retry; use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; @@ -22,6 +22,41 @@ class FunctionsCustomServerTest extends Scope use ProjectCustom; use SideServer; + public function testListSpecs(): void + { + $specifications = $this->listSpecifications(); + $this->assertEquals(200, $specifications['headers']['status-code']); + $this->assertGreaterThan(0, $specifications['body']['total']); + $this->assertArrayHasKey(0, $specifications['body']['specifications']); + $this->assertArrayHasKey('memory', $specifications['body']['specifications'][0]); + $this->assertArrayHasKey('cpus', $specifications['body']['specifications'][0]); + $this->assertArrayHasKey('enabled', $specifications['body']['specifications'][0]); + $this->assertArrayHasKey('slug', $specifications['body']['specifications'][0]); + + $function = $this->createFunction([ + 'functionId' => ID::unique(), + 'name' => 'Specs function', + 'runtime' => 'php-8.0', + 'specification' => $specifications['body']['specifications'][0]['slug'] + ]); + $this->assertEquals(201, $function['headers']['status-code']); + $this->assertEquals($specifications['body']['specifications'][0]['slug'], $function['body']['specification']); + + $function = $this->getFunction($function['body']['$id']); + $this->assertEquals(200, $function['headers']['status-code']); + $this->assertEquals($specifications['body']['specifications'][0]['slug'], $function['body']['specification']); + + $this->cleanupFunction($function['body']['$id']); + + $function = $this->createFunction([ + 'functionId' => ID::unique(), + 'name' => 'Specs function', + 'runtime' => 'php-8.0', + 'specification' => 'cheap-please' + ]); + $this->assertEquals(400, $function['headers']['status-code']); + } + public function testCreateFunction(): array { /** @@ -2006,14 +2041,14 @@ class FunctionsCustomServerTest extends Scope $this->assertEquals(201, $execution['headers']['status-code']); $this->assertStringContainsString('APPWRITE_FUNCTION_ID', $execution['body']['responseBody']); - $site = $this->updateFunction($functionId, [ + $function = $this->updateFunction($functionId, [ 'runtime' => 'node-18.0', 'name' => 'Duplicate Deployment Test', 'entrypoint' => 'index.js', 'commands' => 'rm index.js && mv maintenance.js index.js' ]); - $this->assertEquals(200, $site['headers']['status-code']); - $this->assertStringContainsString('maintenance.js', $site['body']['commands']); + $this->assertEquals(200, $function['headers']['status-code']); + $this->assertStringContainsString('maintenance.js', $function['body']['commands']); $deploymentId2 = $this->setupDuplicateDeployment($functionId, $deploymentId1); $this->assertNotEmpty($deploymentId2); diff --git a/tests/e2e/Services/Sites/SitesBase.php b/tests/e2e/Services/Sites/SitesBase.php index a7e0293409..78c65afe98 100644 --- a/tests/e2e/Services/Sites/SitesBase.php +++ b/tests/e2e/Services/Sites/SitesBase.php @@ -414,4 +414,13 @@ trait SitesBase return $deployment; } + + protected function listSpecifications(): mixed + { + $specifications = $this->client->call(Client::METHOD_GET, '/sites/specifications', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + return $specifications; + } } diff --git a/tests/e2e/Services/Sites/SitesCustomServerTest.php b/tests/e2e/Services/Sites/SitesCustomServerTest.php index bd6236291c..891882b1f9 100644 --- a/tests/e2e/Services/Sites/SitesCustomServerTest.php +++ b/tests/e2e/Services/Sites/SitesCustomServerTest.php @@ -2,7 +2,7 @@ namespace Tests\E2E\Services\Sites; -use Appwrite\Sites\Specification; +use Appwrite\Platform\Modules\Compute\Specification; use Appwrite\Tests\Retry; use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; @@ -20,6 +20,43 @@ class SitesCustomServerTest extends Scope use ProjectCustom; use SideServer; + public function testListSpecs(): void + { + $specifications = $this->listSpecifications(); + $this->assertEquals(200, $specifications['headers']['status-code']); + $this->assertGreaterThan(0, $specifications['body']['total']); + $this->assertArrayHasKey(0, $specifications['body']['specifications']); + $this->assertArrayHasKey('memory', $specifications['body']['specifications'][0]); + $this->assertArrayHasKey('cpus', $specifications['body']['specifications'][0]); + $this->assertArrayHasKey('enabled', $specifications['body']['specifications'][0]); + $this->assertArrayHasKey('slug', $specifications['body']['specifications'][0]); + + $site = $this->createSite([ + 'buildRuntime' => 'node-22', + 'framework' => 'other', + 'name' => 'Specs site', + 'siteId' => ID::unique(), + 'specification' => $specifications['body']['specifications'][0]['slug'] + ]); + $this->assertEquals(201, $site['headers']['status-code']); + $this->assertEquals($specifications['body']['specifications'][0]['slug'], $site['body']['specification']); + + $site = $this->getSite($site['body']['$id']); + $this->assertEquals(200, $site['headers']['status-code']); + $this->assertEquals($specifications['body']['specifications'][0]['slug'], $site['body']['specification']); + + $this->cleanupSite($site['body']['$id']); + + $site = $this->createSite([ + 'buildRuntime' => 'node-22', + 'framework' => 'other', + 'name' => 'Specs site', + 'siteId' => ID::unique(), + 'specification' => 'cheap-please' + ]); + $this->assertEquals(400, $site['headers']['status-code']); + } + public function testCreateSite(): void { /** From 43cd7ac8b3d5de5cc84fef0022466cd4bff0351a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 7 Mar 2025 23:45:19 +0100 Subject: [PATCH 24/31] Linter fix --- app/controllers/general.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 4067a6e597..b2aef81673 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -120,7 +120,7 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw if (\str_starts_with($path, '/.well-known/acme-challenge')) { return false; } - + $type = $rule->getAttribute('type', ''); if ($type === 'deployment') { From 06020e2b621f6976386ae0ae8baabb841e877a6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sat, 8 Mar 2025 12:57:55 +0100 Subject: [PATCH 25/31] Add preview auth tests, finish todos, improve quality --- app/controllers/general.php | 50 ++--- src/Appwrite/Auth/Key.php | 10 +- .../Modules/Functions/Workers/Builds.php | 3 +- tests/e2e/Client.php | 4 +- .../Services/Sites/SitesCustomServerTest.php | 174 +++++++++++++++++- 5 files changed, 215 insertions(+), 26 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index b2aef81673..bebfd89e4b 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -174,46 +174,54 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw return true; } - // Preview authorization (validate) - if ($type === 'deployment') { + // Ensure preview authorization + $requirePreview = \is_null($apiKey) || !$apiKey->isPreviewAuthDisabled(); + if ($isPreview && $requirePreview) { $cookie = $request->getCookie(Auth::$cookieNamePreview, ''); - $ok = true; + $authorized = false; - if (empty($cookie)) { - $ok = false; - } else { + // Security checks to mark authorized true + if (!empty($cookie)) { $jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 3600, 0); $payload = []; try { $payload = $jwt->decode($cookie); } catch (JWTException $error) { - $ok = false; + // Authorized remains false } - $jwtUserId = $payload['userId'] ?? ''; - if (empty($jwtUserId)) { - $ok = false; - } else { - $user = $dbForPlatform->getDocument('users', $jwtUserId); - if ($user->isEmpty()) { - $ok = false; + $userExists = false; + $userId = $payload['userId'] ?? ''; + if (!empty($userId)) { + $user = Authorization::skip(fn () => $dbForPlatform->getDocument('users', $userId)); + if (!$user->isEmpty() && $user->getAttribute('status', false)) { + $userExists = true; } } + $sessionExists = false; $jwtSessionId = $payload['sessionId'] ?? ''; - if (empty($jwtSessionId)) { - $ok = false; - } else { - if (empty($user->find('$id', $jwtSessionId, 'sessions'))) { - $ok = false; + if (!empty($jwtSessionId) && !empty($user->find('$id', $jwtSessionId, 'sessions'))) { + $sessionExists = true; + } + + $membershipExists = false; + $project = Authorization::skip(fn () => $dbForPlatform->getDocument('projects', $projectId)); + if (!$project->isEmpty()) { + $teamId = $project->getAttribute('teamId', ''); + $membership = $user->find('teamId', $teamId, 'memberships'); + if (!empty($membership)) { + $membershipExists = true; } } - // TODO: Ensure user has access to projectId + if ($userExists && $sessionExists && $membershipExists) { + $authorized = true; + } } - if ($ok === false) { + if (!$authorized) { $url = (System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https') . "://" . System::getEnv('_APP_DOMAIN'); $response ->addHeader('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') diff --git a/src/Appwrite/Auth/Key.php b/src/Appwrite/Auth/Key.php index 83f8dd408d..89c28c4727 100644 --- a/src/Appwrite/Auth/Key.php +++ b/src/Appwrite/Auth/Key.php @@ -23,6 +23,7 @@ class Key protected bool $hostnameOverride = false, protected bool $bannerDisabled = false, protected bool $projectCheckDisabled = false, + protected bool $previewAuthDisabled = false, ) { } @@ -73,6 +74,11 @@ class Key return $this->bannerDisabled; } + public function isPreviewAuthDisabled(): bool + { + return $this->previewAuthDisabled; + } + public function isProjectCheckDisabled(): bool { return $this->projectCheckDisabled; @@ -132,6 +138,7 @@ class Key $hostnameOverride = $payload['hostnameOverride'] ?? false; $bannerDisabled = $payload['bannerDisabled'] ?? false; $projectCheckDisabled = $payload['projectCheckDisabled'] ?? false; + $previewAuthDisabled = $payload['previewAuthDisabled'] ?? false; $scopes = \array_merge($payload['scopes'] ?? [], $scopes); if (!$projectCheckDisabled && $projectId !== $project->getId()) { @@ -148,7 +155,8 @@ class Key $disabledMetrics, $hostnameOverride, $bannerDisabled, - $projectCheckDisabled + $projectCheckDisabled, + $previewAuthDisabled ); case API_KEY_STANDARD: $key = $project->find( diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 53556f60ba..29eea1470e 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -762,7 +762,8 @@ class Builds extends Action $apiKey = $jwtObj->encode([ 'hostnameOverride' => true, 'bannerDisabled' => true, - 'projectCheckDisabled' => true + 'projectCheckDisabled' => true, + 'previewAuthDisabled' => true, ]); // TODO: @Meldiron if becomes too slow, do concurrently diff --git a/tests/e2e/Client.php b/tests/e2e/Client.php index dc80808b14..278d1cd0d5 100644 --- a/tests/e2e/Client.php +++ b/tests/e2e/Client.php @@ -164,7 +164,7 @@ class Client * @return array * @throws Exception */ - public function call(string $method, string $path = '', array $headers = [], mixed $params = [], bool $decode = true): array + public function call(string $method, string $path = '', array $headers = [], mixed $params = [], bool $decode = true, bool $followRedirects = true): array { $headers = array_merge($this->headers, $headers); $ch = curl_init($this->endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : '')); @@ -192,7 +192,7 @@ class Client curl_setopt($ch, CURLOPT_PATH_AS_IS, 1); curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, $followRedirects); curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'); curl_setopt($ch, CURLOPT_HTTPHEADER, $formattedHeaders); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0); diff --git a/tests/e2e/Services/Sites/SitesCustomServerTest.php b/tests/e2e/Services/Sites/SitesCustomServerTest.php index bd6236291c..4bb09ad4dd 100644 --- a/tests/e2e/Services/Sites/SitesCustomServerTest.php +++ b/tests/e2e/Services/Sites/SitesCustomServerTest.php @@ -2,6 +2,7 @@ namespace Tests\E2E\Services\Sites; +use Ahc\Jwt\JWT; use Appwrite\Sites\Specification; use Appwrite\Tests\Retry; use Tests\E2E\Client; @@ -1482,8 +1483,18 @@ class SitesCustomServerTest extends Scope $proxyClient = new Client(); $proxyClient->setEndpoint('http://' . $previewDomain); - $response = $proxyClient->call(Client::METHOD_GET, '/'); + $response = $proxyClient->call(Client::METHOD_GET, '/', followRedirects: false); + $this->assertEquals(301, $response['headers']['status-code']); + $this->assertStringContainsString('/console/auth/preview', $response['headers']['location']); + $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 900, 0); + $apiKey = $jwtObj->encode([ + 'projectCheckDisabled' => true, + 'previewAuthDisabled' => true, + ]); + $response = $proxyClient->call(Client::METHOD_GET, '/', followRedirects: false, headers: [ + 'x-appwrite-key' => API_KEY_DYNAMIC . '_' . $apiKey, + ]); $this->assertEquals(200, $response['headers']['status-code']); $this->assertStringContainsString("Hello Appwrite", $response['body']); $this->assertStringContainsString("Preview by", $response['body']); @@ -1839,4 +1850,165 @@ class SitesCustomServerTest extends Scope $this->cleanupSite($siteId); } + + public function testPreviewDomain(): void + { + $siteId = $this->setupSite([ + 'buildRuntime' => 'node-22', + 'framework' => 'other', + 'name' => 'Authorized preview site', + 'siteId' => ID::unique(), + 'adapter' => 'static', + ]); + $this->assertNotEmpty($siteId); + + $deploymentId = $this->setupDeployment($siteId, [ + 'code' => $this->packageSite('static'), + 'activate' => true + ]); + $this->assertNotEmpty($deploymentId); + + $domain = $this->getDeploymentDomain($deploymentId); + $this->assertNotEmpty($domain); + $proxyClient = new Client(); + $proxyClient->setEndpoint('http://' . $domain); + + $response = $proxyClient->call(Client::METHOD_GET, '/contact', followRedirects: false); + $this->assertEquals(301, $response['headers']['status-code']); + $this->assertStringContainsString('/console/auth/preview', $response['headers']['location']); + $this->assertStringContainsString('projectId=' . $this->getProject()['$id'], $response['headers']['location']); + $this->assertStringContainsString('origin=', $response['headers']['location']); + $this->assertStringContainsString('path=%2Fcontact', $response['headers']['location']); + + $session = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => 'console', + ]), [ + 'email' => $this->getRoot()['email'], + 'password' => 'password' + ]); + $this->assertEquals(201, $session['headers']['status-code']); + $this->assertNotEmpty($session['cookies']['a_session_console']); + $this->assertNotEmpty($session['body']['$id']); + $cookie = 'a_session_console=' . $session['cookies']['a_session_console']; + + $jwt = $this->client->call(Client::METHOD_POST, '/account/jwts', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'cookie' => $cookie, + 'x-appwrite-project' => 'console', + ]), []); + $this->assertEquals(201, $jwt['headers']['status-code']); + $this->assertNotEmpty($jwt['body']['jwt']); + + $response = $proxyClient->call(Client::METHOD_GET, '/_appwrite/authorize', params: [ + 'jwt' => $jwt['body']['jwt'], + 'path' => '/contact' + ], followRedirects: false); + $this->assertEquals(301, $response['headers']['status-code']); + $this->assertArrayHasKey('set-cookie', $response['headers']); + $this->assertStringContainsString('a_jwt_console=', $response['headers']['set-cookie']); + $this->assertStringContainsString('httponly', $response['headers']['set-cookie']); + $this->assertStringContainsString('domain=' . $domain, $response['headers']['set-cookie']); + $this->assertStringContainsString('path=/', $response['headers']['set-cookie']); + $this->assertNotEmpty($response['cookies']['a_jwt_console']); + $this->assertEquals($jwt['body']['jwt'], $response['cookies']['a_jwt_console']); + + $response = $proxyClient->call(Client::METHOD_GET, '/contact', headers: [ + 'cookie' => 'a_jwt_console=' . $jwt['body']['jwt'] + ], followRedirects: false); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertStringContainsString("Contact page", $response['body']); + $this->assertStringContainsString("Preview by", $response['body']); + + // Failure: Session missing (old bad, new ok) + $session = $this->client->call(Client::METHOD_DELETE, '/account/sessions/current', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'cookie' => $cookie, + 'x-appwrite-project' => 'console', + ]), []); + $this->assertEquals(204, $session['headers']['status-code']); + + $response = $proxyClient->call(Client::METHOD_GET, '/contact', headers: [ + 'cookie' => 'a_jwt_console=' . $jwt['body']['jwt'] + ], followRedirects: false); + $this->assertEquals(301, $response['headers']['status-code']); + $this->assertStringContainsString('/console/auth/preview', $response['headers']['location']); + + // Failure: User missing + $cookie = 'a_session_console=' .$this->getRoot()['session']; + $jwt = $this->client->call(Client::METHOD_POST, '/account/jwts', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'cookie' => $cookie, + 'x-appwrite-project' => 'console', + ]), []); + $this->assertEquals(201, $jwt['headers']['status-code']); + $this->assertNotEmpty($jwt['body']['jwt']); + + $response = $proxyClient->call(Client::METHOD_GET, '/contact', headers: [ + 'cookie' => 'a_jwt_console=' . $jwt['body']['jwt'] + ], followRedirects: false); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertStringContainsString("Contact page", $response['body']); + $this->assertStringContainsString("Preview by", $response['body']); + + $user = $this->client->call(Client::METHOD_PATCH, '/account/status', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'cookie' => $cookie, + 'x-appwrite-project' => 'console', + ]), []); + $this->assertEquals(200, $user['headers']['status-code']); + $this->assertFalse($user['body']['status']); + + $response = $proxyClient->call(Client::METHOD_GET, '/contact', headers: [ + 'cookie' => 'a_jwt_console=' . $jwt['body']['jwt'] + ], followRedirects: false); + $this->assertEquals(301, $response['headers']['status-code']); + $this->assertStringContainsString('/console/auth/preview', $response['headers']['location']); + + // Failure: Membership missing + $user = $this->client->call(Client::METHOD_POST, '/account', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => 'console', + ], [ + 'userId' => ID::unique(), + 'email' => 'newuser@appwrite.io', + 'password' => 'password' + ]); + $this->assertEquals(201, $user['headers']['status-code']); + + $session = $this->client->call(Client::METHOD_POST, '/account/sessions/email', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => 'console', + ], [ + 'email' => 'newuser@appwrite.io', + 'password' => 'password', + ]); + $this->assertEquals(201, $session['headers']['status-code']); + $this->assertNotEmpty($session['cookies']['a_session_console']); + $cookie = 'a_session_console=' . $session['cookies']['a_session_console']; + + $jwt = $this->client->call(Client::METHOD_POST, '/account/jwts', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'cookie' => $cookie, + 'x-appwrite-project' => 'console', + ]), []); + $this->assertEquals(201, $jwt['headers']['status-code']); + $this->assertNotEmpty($jwt['body']['jwt']); + + $response = $proxyClient->call(Client::METHOD_GET, '/contact', headers: [ + 'cookie' => 'a_jwt_console=' . $jwt['body']['jwt'] + ], followRedirects: false); + $this->assertEquals(301, $response['headers']['status-code']); + $this->assertStringContainsString('/console/auth/preview', $response['headers']['location']); + + $this->cleanupSite($siteId); + } } From f85bf337b702fc7a338ab15f89872fc8358b71ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sat, 8 Mar 2025 13:03:23 +0100 Subject: [PATCH 26/31] Migrate authorize endpoint to http lib --- app/controllers/general.php | 44 +++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index bebfd89e4b..48cacaaec7 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -158,22 +158,6 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw $protocol = $request->getProtocol(); - // Preview authorization (configure) - if (\str_starts_with($path, '/_appwrite/authorize')) { - $jwt = $request->getParam('jwt', ''); - $path = $request->getParam('path', ''); - - $duration = 60 * 60 * 24; // 1 day in seconds - $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)); - - $response - ->addCookie(Auth::$cookieNamePreview, $jwt, (new \DateTime($expire))->getTimestamp(), '/', $host, ('https' === $protocol), true, null) - ->addHeader('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') - ->addHeader('Pragma', 'no-cache') - ->redirect($protocol . '://' . $host . $path); - return true; - } - // Ensure preview authorization $requirePreview = \is_null($apiKey) || !$apiKey->isPreviewAuthDisabled(); if ($isPreview && $requirePreview) { @@ -1359,6 +1343,34 @@ App::get('/v1/ping') $response->text('Pong!'); }); +// Preview authorization +App::get('/_appwrite/authorize') + ->inject('request') + ->inject('response') + ->inject('previewHostname') + ->action(function (Request $request, Response $response, string $previewHostname) { + + $host = $request->getHostname() ?? ''; + if (!empty($previewHostname)) { + $host = $previewHostname; + } + + $referrer = $request->getReferer(); + $protocol = \parse_url($request->getOrigin($referrer), PHP_URL_SCHEME); + + $jwt = $request->getParam('jwt', ''); + $path = $request->getParam('path', ''); + + $duration = 60 * 60 * 24; // 1 day in seconds + $expire = DateTime::formatTz(DateTime::addSeconds(new \DateTime(), $duration)); + + $response + ->addCookie(Auth::$cookieNamePreview, $jwt, (new \DateTime($expire))->getTimestamp(), '/', $host, ('https' === $protocol), true, null) + ->addHeader('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') + ->addHeader('Pragma', 'no-cache') + ->redirect($protocol . '://' . $host . $path); + }); + App::wildcard() ->groups(['api']) ->label('scope', 'global') From 232f99c6c9d27e0e581d8fe486f8a8cde1f04659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sat, 8 Mar 2025 16:50:39 +0100 Subject: [PATCH 27/31] Finalize PR, review changes, fix tests --- app/config/frameworks.php | 38 ++--- app/controllers/api/vcs.php | 104 +++++++------ composer.lock | 78 +++++++++- docker-compose.yml | 2 +- .../Modules/Functions/Workers/Builds.php | 36 ++--- .../Modules/Sites/Http/Sites/Create.php | 2 +- .../Modules/Sites/Http/Sites/Update.php | 2 +- src/Appwrite/Utopia/Response.php | 32 ++-- ...rkDetection.php => DetectionFramework.php} | 6 +- ...timeDetection.php => DetectionRuntime.php} | 6 +- ...ry.php => ProviderRepositoryFramework.php} | 8 +- ...tory.php => ProviderRepositoryRuntime.php} | 10 +- src/Executor/Executor.php | 6 +- .../Services/Sites/SitesCustomServerTest.php | 144 +++++++++++++++++- .../e2e/Services/VCS/VCSConsoleClientTest.php | 55 ++++--- .../sites/astro-static/astro.config.mjs | 4 + .../resources/sites/astro-static/package.json | 14 ++ .../sites/astro-static/src/pages/index.astro | 10 ++ .../sites/static-single-file/main.html | 10 ++ 19 files changed, 396 insertions(+), 171 deletions(-) rename src/Appwrite/Utopia/Response/Model/{FrameworkDetection.php => DetectionFramework.php} (91%) rename src/Appwrite/Utopia/Response/Model/{RuntimeDetection.php => DetectionRuntime.php} (90%) rename src/Appwrite/Utopia/Response/Model/{FrameworkProviderRepository.php => ProviderRepositoryFramework.php} (65%) rename src/Appwrite/Utopia/Response/Model/{RuntimeProviderRepository.php => ProviderRepositoryRuntime.php} (62%) create mode 100644 tests/resources/sites/astro-static/astro.config.mjs create mode 100644 tests/resources/sites/astro-static/package.json create mode 100644 tests/resources/sites/astro-static/src/pages/index.astro create mode 100644 tests/resources/sites/static-single-file/main.html diff --git a/app/config/frameworks.php b/app/config/frameworks.php index 332035a79c..8aee8b9e16 100644 --- a/app/config/frameworks.php +++ b/app/config/frameworks.php @@ -23,6 +23,8 @@ return [ 'name' => 'Next.js', 'buildRuntime' => 'node-22', 'runtimes' => getVersions($templateRuntimes['NODE']['versions'], 'node'), + 'bundleCommand' => 'sh /usr/local/server/helpers/next-js/bundle.sh', + 'envCommand' => 'source /usr/local/server/helpers/next-js/env.sh', 'adapters' => [ 'ssr' => [ 'key' => 'ssr', @@ -30,8 +32,6 @@ return [ 'installCommand' => 'npm install', 'outputDirectory' => './.next', 'startCommand' => 'sh helpers/next-js/server.sh', - 'bundleCommand' => 'sh /usr/local/server/helpers/next-js/bundle.sh', - 'envCommand' => 'source /usr/local/server/helpers/next-js/env.sh', ], 'static' => [ 'key' => 'static', @@ -39,8 +39,6 @@ return [ 'installCommand' => 'npm install', 'outputDirectory' => './out', 'startCommand' => 'sh helpers/server.sh', - 'bundleCommand' => '', - 'envCommand' => '', ] ] ], @@ -56,8 +54,6 @@ return [ 'installCommand' => 'npm install', 'outputDirectory' => './dist', 'startCommand' => 'sh helpers/server.sh', - 'bundleCommand' => '', - 'envCommand' => '', 'fallbackFile' => 'index.html' ] ] @@ -67,6 +63,8 @@ return [ 'name' => 'Nuxt', 'buildRuntime' => 'node-22', 'runtimes' => getVersions($templateRuntimes['NODE']['versions'], 'node'), + 'bundleCommand' => 'sh /usr/local/server/helpers/nuxt/bundle.sh', + 'envCommand' => 'source /usr/local/server/helpers/nuxt/env.sh', 'adapters' => [ 'ssr' => [ 'key' => 'ssr', @@ -74,8 +72,6 @@ return [ 'installCommand' => 'npm install', 'outputDirectory' => './.output', 'startCommand' => 'sh helpers/nuxt/server.sh', - 'bundleCommand' => 'sh /usr/local/server/helpers/nuxt/bundle.sh', - 'envCommand' => 'source /usr/local/server/helpers/nuxt/env.sh', ], 'static' => [ 'key' => 'static', @@ -83,8 +79,6 @@ return [ 'installCommand' => 'npm install', 'outputDirectory' => './dist', 'startCommand' => 'sh helpers/server.sh', - 'bundleCommand' => '', - 'envCommand' => '', ] ] ], @@ -100,8 +94,6 @@ return [ 'installCommand' => 'npm install', 'outputDirectory' => './dist', 'startCommand' => 'sh helpers/server.sh', - 'bundleCommand' => '', - 'envCommand' => '', 'fallbackFile' => 'index.html' ] ] @@ -111,6 +103,8 @@ return [ 'name' => 'SvelteKit', 'buildRuntime' => 'node-22', 'runtimes' => getVersions($templateRuntimes['NODE']['versions'], 'node'), + 'bundleCommand' => 'sh /usr/local/server/helpers/sveltekit/bundle.sh', + 'envCommand' => 'source /usr/local/server/helpers/sveltekit/env.sh', 'adapters' => [ 'ssr' => [ 'key' => 'ssr', @@ -118,8 +112,6 @@ return [ 'installCommand' => 'npm install', 'outputDirectory' => './build', 'startCommand' => 'sh helpers/sveltekit/server.sh', - 'bundleCommand' => 'sh /usr/local/server/helpers/sveltekit/bundle.sh', - 'envCommand' => 'source /usr/local/server/helpers/sveltekit/env.sh', ], 'static' => [ 'key' => 'static', @@ -127,8 +119,6 @@ return [ 'installCommand' => 'npm install', 'outputDirectory' => './build', 'startCommand' => 'sh helpers/server.sh', - 'bundleCommand' => '', - 'envCommand' => '', ] ] ], @@ -137,6 +127,8 @@ return [ 'name' => 'Astro', 'buildRuntime' => 'node-22', 'runtimes' => getVersions($templateRuntimes['NODE']['versions'], 'node'), + 'bundleCommand' => 'sh /usr/local/server/helpers/astro/bundle.sh', + 'envCommand' => 'source /usr/local/server/helpers/astro/env.sh', 'adapters' => [ 'ssr' => [ 'key' => 'ssr', @@ -144,8 +136,6 @@ return [ 'installCommand' => 'npm install', 'outputDirectory' => './dist', 'startCommand' => 'sh helpers/astro/server.sh', - 'bundleCommand' => 'sh /usr/local/server/helpers/astro/bundle.sh', - 'envCommand' => 'source /usr/local/server/helpers/astro/env.sh', ], 'static' => [ 'key' => 'static', @@ -153,8 +143,6 @@ return [ 'installCommand' => 'npm install', 'outputDirectory' => './dist', 'startCommand' => 'sh helpers/server.sh', - 'bundleCommand' => '', - 'envCommand' => '', ] ] ], @@ -163,6 +151,8 @@ return [ 'name' => 'Remix', 'buildRuntime' => 'node-22', 'runtimes' => getVersions($templateRuntimes['NODE']['versions'], 'node'), + 'bundleCommand' => 'sh /usr/local/server/helpers/remix/bundle.sh', + 'envCommand' => 'source /usr/local/server/helpers/remix/env.sh', 'adapters' => [ 'ssr' => [ 'key' => 'ssr', @@ -170,8 +160,6 @@ return [ 'installCommand' => 'npm install', 'outputDirectory' => './build', 'startCommand' => 'sh helpers/remix/server.sh', - 'bundleCommand' => 'sh /usr/local/server/helpers/remix/bundle.sh', - 'envCommand' => 'source /usr/local/server/helpers/remix/env.sh', ], 'static' => [ 'key' => 'static', @@ -179,8 +167,6 @@ return [ 'installCommand' => 'npm install', 'outputDirectory' => './build/client', 'startCommand' => 'sh helpers/server.sh', - 'bundleCommand' => '', - 'envCommand' => '', ] ] ], @@ -196,8 +182,6 @@ return [ 'installCommand' => '', 'outputDirectory' => './build/web', 'startCommand' => 'sh helpers/server.sh', - 'bundleCommand' => '', - 'envCommand' => '', ], ], ], @@ -213,8 +197,6 @@ return [ 'installCommand' => '', 'outputDirectory' => './', 'startCommand' => 'sh helpers/server.sh', - 'bundleCommand' => '', - 'envCommand' => '', ], ] ], diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 014a1233d5..c890bf5395 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -606,7 +606,7 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories/:pro App::post('/v1/vcs/github/installations/:installationId/detections') ->alias('/v1/vcs/github/installations/:installationId/providerRepositories/:providerRepositoryId/detection') - ->desc('Detect runtime and framework settings from source code') + ->desc('Create repository detection') ->groups(['api', 'vcs']) ->label('scope', 'vcs.write') ->label('sdk', new Method( @@ -617,11 +617,11 @@ App::post('/v1/vcs/github/installations/:installationId/detections') responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, - model: Response::MODEL_RUNTIME_DETECTION, + model: Response::MODEL_DETECTION_RUNTIME, ), new SDKResponse( code: Response::STATUS_CODE_OK, - model: Response::MODEL_FRAMEWORK_DETECTION, + model: Response::MODEL_DETECTION_FRAMEWORK, ) ] )) @@ -663,20 +663,20 @@ App::post('/v1/vcs/github/installations/:installationId/detections') ->addOption(new Yarn()) ->addOption(new PNPM()) ->addOption(new NPM()); - $detectedPackager = $detector->detect(); + $detection = $detector->detect(); - $packagerName = !\is_null($detectedPackager) ? $detectedPackager->getName() : 'npm'; + $packager = !\is_null($detection) ? $detection->getName() : 'npm'; if ($type === 'framework') { - $detection = new Document([ + $output = new Document([ 'framework' => '', 'installCommand' => '', 'buildCommand' => '', 'outputDirectory' => '', ]); - $frameworkDetector = new Framework($files, $packagerName); - $frameworkDetector + $detector = new Framework($files, $packager); + $detector ->addOption(new Flutter()) ->addOption(new Nuxt()) ->addOption(new Astro()) @@ -684,28 +684,27 @@ App::post('/v1/vcs/github/installations/:installationId/detections') ->addOption(new SvelteKit()) ->addOption(new NextJs()); - $detectedFramework = $frameworkDetector->detect(); + $framework = $detector->detect(); - if ($detectedFramework) { - $framework = $detectedFramework->getName(); - $detection->setAttribute('installCommand', $detectedFramework->getInstallCommand()); - $detection->setAttribute('buildCommand', $detectedFramework->getBuildCommand()); - $detection->setAttribute('outputDirectory', $detectedFramework->getOutputDirectory()); + if (!\is_null($framework)) { + $framework = $framework->getName(); + $output->setAttribute('installCommand', $framework->getInstallCommand()); + $output->setAttribute('buildCommand', $framework->getBuildCommand()); + $output->setAttribute('outputDirectory', $framework->getOutputDirectory()); } else { $framework = 'other'; - $detection->setAttribute('installCommand', ''); - $detection->setAttribute('buildCommand', ''); - $detection->setAttribute('outputDirectory', ''); + $output->setAttribute('installCommand', ''); + $output->setAttribute('buildCommand', ''); + $output->setAttribute('outputDirectory', ''); } $frameworks = Config::getParam('frameworks'); - $frameworkKey = 'other'; - if (\in_array($framework, array_keys($frameworks), true)) { - $frameworkKey = $framework; + if (!\in_array($framework, array_keys($frameworks), true)) { + $framework = 'other'; } - $detection->setAttribute('framework', $frameworkKey); + $output->setAttribute('framework', $framework); } else { - $detection = new Document([ + $output = new Document([ 'runtime' => '', 'commands' => '', 'entrypoint' => '', @@ -718,8 +717,8 @@ App::post('/v1/vcs/github/installations/:installationId/detections') ]; foreach ($strategies as $strategy) { - $runtimeDetector = new Runtime($strategy === Strategy::LANGUAGES ? $languages : $files, $strategy, $packagerName); - $runtimeDetector + $detector = new Runtime($strategy === Strategy::LANGUAGES ? $languages : $files, $strategy, $packager); + $detector ->addOption(new Node()) ->addOption(new Bun()) ->addOption(new Deno()) @@ -732,35 +731,35 @@ App::post('/v1/vcs/github/installations/:installationId/detections') ->addOption(new CPP()) ->addOption(new Dotnet()); - $detectedRuntime = $runtimeDetector->detect(); + $runtime = $detector->detect(); - if ($detectedRuntime) { - $detection->setAttribute('commands', $detectedRuntime->getCommands()); - $detection->setAttribute('entrypoint', $detectedRuntime->getEntrypoint()); - $runtime = $detectedRuntime->getName(); + if (!\is_null($runtime)) { + $output->setAttribute('commands', $runtime->getCommands()); + $output->setAttribute('entrypoint', $runtime->getEntrypoint()); + $runtime = $runtime->getName(); break; } } if (!empty($runtime)) { $runtimes = Config::getParam('runtimes'); - $versionedRuntime = ''; + $runtimeWithVersion = ''; foreach ($runtimes as $runtimeKey => $runtimeConfig) { if ($runtimeConfig['key'] === $runtime) { - $versionedRuntime = $runtimeKey; + $runtimeWithVersion = $runtimeKey; } } - if (empty($versionedRuntime)) { + if (empty($runtimeWithVersion)) { throw new Exception(Exception::FUNCTION_RUNTIME_NOT_DETECTED); } - $detection->setAttribute('runtime', $versionedRuntime); + $output->setAttribute('runtime', $runtimeWithVersion); } else { throw new Exception(Exception::FUNCTION_RUNTIME_NOT_DETECTED); } } - $response->dynamic($detection, $type === 'framework' ? Response::MODEL_FRAMEWORK_DETECTION : Response::MODEL_RUNTIME_DETECTION); + $response->dynamic($output, $type === 'framework' ? Response::MODEL_DETECTION_FRAMEWORK : Response::MODEL_DETECTION_RUNTIME); }); App::get('/v1/vcs/github/installations/:installationId/providerRepositories') @@ -775,11 +774,11 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') responses: [ new SDKResponse( code: Response::STATUS_CODE_OK, - model: Response::MODEL_RUNTIME_PROVIDER_REPOSITORY_LIST, + model: Response::MODEL_PROVIDER_REPOSITORY_RUNTIME_LIST, ), new SDKResponse( code: Response::STATUS_CODE_OK, - model: Response::MODEL_FRAMEWORK_PROVIDER_REPOSITORY_LIST, + model: Response::MODEL_PROVIDER_REPOSITORY_FRAMEWORK_LIST, ) ] )) @@ -829,12 +828,12 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') ->addOption(new Yarn()) ->addOption(new PNPM()) ->addOption(new NPM()); - $detectedPackager = $detector->detect(); + $detection = $detector->detect(); - $packagerName = !\is_null($detectedPackager) ? $detectedPackager->getName() : 'npm'; + $packager = !\is_null($detection) ? $detection->getName() : 'npm'; if ($type === 'framework') { - $frameworkDetector = new Framework($files, $packagerName); + $frameworkDetector = new Framework($files, $packager); $frameworkDetector ->addOption(new Flutter()) ->addOption(new Nuxt()) @@ -845,18 +844,17 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') $detectedFramework = $frameworkDetector->detect(); - if ($detectedFramework) { + if (!\is_null($detectedFramework)) { $framework = $detectedFramework->getName(); } else { $framework = 'other'; } $frameworks = Config::getParam('frameworks'); - $frameworkKey = ''; - if (\in_array($framework, array_keys($frameworks), true)) { - $frameworkKey = $framework; + if (!\in_array($framework, array_keys($frameworks), true)) { + $framework = 'other'; } - $repo['framework'] = !empty($frameworkKey) ? $frameworkKey : 'other'; + $repo['framework'] = $framework; } else { $languages = $github->listRepositoryLanguages($repo['organization'], $repo['name']); @@ -867,8 +865,8 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') ]; foreach ($strategies as $strategy) { - $runtimeDetector = new Runtime($strategy === Strategy::LANGUAGES ? $languages : $files, $strategy, $packagerName); - $runtimeDetector + $redector = new Runtime($strategy === Strategy::LANGUAGES ? $languages : $files, $strategy, $packager); + $redector ->addOption(new Node()) ->addOption(new Bun()) ->addOption(new Deno()) @@ -881,24 +879,24 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') ->addOption(new CPP()) ->addOption(new Dotnet()); - $detectedRuntime = $runtimeDetector->detect(); + $runtime = $redector->detect(); - if ($detectedRuntime) { - $runtime = $detectedRuntime->getName(); + if (!\is_null($runtime)) { + $runtime = $runtime->getName(); break; } } if (!empty($runtime)) { $runtimes = Config::getParam('runtimes'); - $versionedRuntime = ''; + $runtimeWithVersion = ''; foreach ($runtimes as $runtimeKey => $runtimeConfig) { if ($runtimeConfig['key'] === $runtime) { - $versionedRuntime = $runtimeKey; + $runtimeWithVersion = $runtimeKey; } } - $repo['runtime'] = $versionedRuntime ?? ''; + $repo['runtime'] = $runtimeWithVersion ?? ''; } } return $repo; @@ -912,7 +910,7 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') $response->dynamic(new Document([ $type === 'framework' ? 'frameworkProviderRepositories' : 'runtimeProviderRepositories' => $repos, 'total' => \count($repos), - ]), ($type === 'framework') ? Response::MODEL_FRAMEWORK_PROVIDER_REPOSITORY_LIST : Response::MODEL_RUNTIME_PROVIDER_REPOSITORY_LIST); + ]), ($type === 'framework') ? Response::MODEL_PROVIDER_REPOSITORY_FRAMEWORK_LIST : Response::MODEL_PROVIDER_REPOSITORY_RUNTIME_LIST); }); App::post('/v1/vcs/github/installations/:installationId/providerRepositories') diff --git a/composer.lock b/composer.lock index a7f0992f99..e3a8633120 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "3b6171de8c624cfbcd723f1cc76a9560", + "content-hash": "34ca1e0386c2325732532c6978cd93fb", "packages": [ { "name": "adhocore/jwt", @@ -3758,6 +3758,69 @@ }, "time": "2025-02-12T08:08:29+00:00" }, + { + "name": "utopia-php/detector", + "version": "dev-feat-pseudocode-draft2", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/detector.git", + "reference": "e843ff2c85ed55e2cbe7e29f11583903fe002774" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/detector/zipball/e843ff2c85ed55e2cbe7e29f11583903fe002774", + "reference": "e843ff2c85ed55e2cbe7e29f11583903fe002774", + "shasum": "" + }, + "require": { + "php": ">=8.0" + }, + "require-dev": { + "laravel/pint": "1.2.*", + "phpstan/phpstan": "1.8.*", + "phpunit/phpunit": "^9.4" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Detector\\": "src/" + } + }, + "autoload-dev": { + "psr-4": { + "Utopia\\Tests\\": "tests/Detector/" + } + }, + "scripts": { + "lint": [ + "./vendor/bin/pint --test --config pint.json" + ], + "format": [ + "./vendor/bin/pint --config pint.json" + ], + "check": [ + "./vendor/bin/phpstan analyse --level 8 -c phpstan.neon src tests" + ], + "test": [ + "./vendor/bin/phpunit --configuration phpunit.xml --debug" + ] + }, + "license": [ + "MIT" + ], + "description": "A simple library for fast and reliable environment identification.", + "keywords": [ + "detector", + "framework", + "php", + "utopia" + ], + "support": { + "source": "https://github.com/utopia-php/detector/tree/feat-pseudocode-draft2", + "issues": "https://github.com/utopia-php/detector/issues" + }, + "time": "2025-03-08T15:32:38+00:00" + }, { "name": "utopia-php/domains", "version": "0.5.0", @@ -8793,9 +8856,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/detector", + "version": "dev-feat-pseudocode-draft2", + "alias": "0.1.99", + "alias_normalized": "0.1.99.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/detector": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index 1cf0af5078..c50c12f206 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -976,7 +976,7 @@ services: # It's not possible to share mount file between 2 containers without host mount (copying is too slow) - /tmp:/tmp:rw environment: - - OPR_EXECUTOR_IMAGE_PULL=enabled + - OPR_EXECUTOR_IMAGE_PULL=disabled - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_COMPUTE_INACTIVE_THRESHOLD - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_COMPUTE_MAINTENANCE_INTERVAL - OPR_EXECUTOR_NETWORK=$_APP_COMPUTE_RUNTIMES_NETWORK diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index fb6688aec8..c5310378b9 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -610,7 +610,7 @@ class Builds extends Action memory: $memory, timeout: $timeout, remove: false, - entrypoint: $deployment->getAttribute('entrypoint', 'package.json'), // TODO: change this later so that sites don't need to have an entrypoint + entrypoint: $deployment->getAttribute('entrypoint', ''), destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", variables: $vars, command: $command, @@ -711,30 +711,32 @@ class Builds extends Action } if ($resource->getCollection() === 'sites' && isEmpty($resource->getAttribute('adapter'))) { - $listFilesCommand = "cd /usr/local/build/" . $resource->getAttribute('outputDirectory') . " && find . -name 'node_modules' -prune -o -type f -print"; - $commandExecutionResponse = $executor->executeCommand( + $listFilesCommand = "cd /usr/local/build && cd " . \escapeshellarg($resource->getAttribute('outputDirectory')) . " && find . -name 'node_modules' -prune -o -type f -print"; + $command = $executor->createCommand( deploymentId: $deployment->getId(), projectId: $project->getId(), command: $listFilesCommand, - timeout: 60 + timeout: 15 ); - $files = array_map('trim', array_filter(explode("\n", $commandExecutionResponse['output']))); + $files = \explode("\n", $command['output']); // Parse output + $files = \array_filter($files); // Remove empty + $files = \array_map(fn ($file) => \trim($file), $files); // Remove whitepsaces + $files = \array_map(fn ($file) => \str_starts_with($file, './') ? \substr($file, 2) : $file, $files); // Remove beginning ./ - $detector = new Rendering($files, $resource->getAttribute('framework')); + $detector = new Rendering($files, $resource->getAttribute('framework', '')); $detector ->addOption(new SSR()) ->addOption(new XStatic()); - $detectedRenderingStrategy = $detector->detect(); + $detection = $detector->detect(); - $renderingStrategy = $detectedRenderingStrategy->getName(); - $fallbackFile = $detectedRenderingStrategy->getFallbackFile() ?: ''; - - $resource->setAttribute('adapter', $renderingStrategy); - $resource->setAttribute('fallbackFile', $fallbackFile); + $resource->setAttribute('adapter', $detection->getName()); + $resource->setAttribute('fallbackFile', $detection->getFallbackFile() ?? ''); $resource = $dbForProject->updateDocument('sites', $resource->getId(), $resource); } + $executor->deleteRuntime($project->getId(), $deployment->getId(), '-build'); + /** Update the build document */ $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); $build->setAttribute('endTime', $endTime); @@ -1072,15 +1074,9 @@ class Builds extends Action $envCommand = ''; $bundleCommand = ''; - if (!is_null($framework)) { - $adapter = ($framework['adapters'] ?? [])[$resource->getAttribute('adapter', '')] ?? null; - if (!is_null($adapter) && isset($adapter['envCommand'])) { - $envCommand = $adapter['envCommand']; - } - if (!is_null($adapter) && isset($adapter['bundleCommand'])) { - $bundleCommand = $adapter['bundleCommand']; - } + $envCommand = $framework['envCommand'] ?? ''; + $bundleCommand = $framework['bundleCommand'] ?? ''; } $commands[] = $envCommand; diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php index cad4ac33a3..b3c0c9fb47 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Create.php @@ -91,7 +91,7 @@ class Create extends Base ->callback([$this, 'action']); } - public function action(string $siteId, string $name, string $framework, bool $enabled, int $timeout, string $installCommand, string $buildCommand, string $outputDirectory, string $buildRuntime, string $adapter, string $installationId, ?string $fallbackFile, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $specification, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Database $dbForPlatform) + public function action(string $siteId, string $name, string $framework, bool $enabled, int $timeout, string $installCommand, string $buildCommand, string $outputDirectory, string $buildRuntime, string $adapter, string $installationId, string $fallbackFile, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $specification, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Database $dbForPlatform) { if (!empty($adapter)) { $configFramework = Config::getParam('frameworks')[$framework] ?? []; diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php index c28fd58bc5..35e4fb07e3 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/Update.php @@ -98,7 +98,7 @@ class Update extends Base ->callback([$this, 'action']); } - public function action(string $siteId, string $name, string $framework, bool $enabled, int $timeout, string $installCommand, string $buildCommand, string $outputDirectory, string $buildRuntime, string $adapter, ?string $fallbackFile, string $installationId, ?string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) + public function action(string $siteId, string $name, string $framework, bool $enabled, int $timeout, string $installCommand, string $buildCommand, string $outputDirectory, string $buildRuntime, string $adapter, string $fallbackFile, string $installationId, ?string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) { if (!empty($adapter)) { $configFramework = Config::getParam('frameworks')[$framework] ?? []; diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 2863606106..384ef27f6d 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -38,6 +38,8 @@ use Appwrite\Utopia\Response\Model\Country; use Appwrite\Utopia\Response\Model\Currency; use Appwrite\Utopia\Response\Model\Database; use Appwrite\Utopia\Response\Model\Deployment; +use Appwrite\Utopia\Response\Model\DetectionFramework; +use Appwrite\Utopia\Response\Model\DetectionRuntime; use Appwrite\Utopia\Response\Model\Document as ModelDocument; use Appwrite\Utopia\Response\Model\Error; use Appwrite\Utopia\Response\Model\ErrorDev; @@ -45,8 +47,6 @@ use Appwrite\Utopia\Response\Model\Execution; use Appwrite\Utopia\Response\Model\File; use Appwrite\Utopia\Response\Model\Framework; use Appwrite\Utopia\Response\Model\FrameworkAdapter; -use Appwrite\Utopia\Response\Model\FrameworkDetection; -use Appwrite\Utopia\Response\Model\FrameworkProviderRepository; use Appwrite\Utopia\Response\Model\Func; use Appwrite\Utopia\Response\Model\Headers; use Appwrite\Utopia\Response\Model\HealthAntivirus; @@ -84,10 +84,10 @@ use Appwrite\Utopia\Response\Model\Preferences; use Appwrite\Utopia\Response\Model\Project; use Appwrite\Utopia\Response\Model\Provider; use Appwrite\Utopia\Response\Model\ProviderRepository; +use Appwrite\Utopia\Response\Model\ProviderRepositoryFramework; +use Appwrite\Utopia\Response\Model\ProviderRepositoryRuntime; use Appwrite\Utopia\Response\Model\Rule; use Appwrite\Utopia\Response\Model\Runtime; -use Appwrite\Utopia\Response\Model\RuntimeDetection; -use Appwrite\Utopia\Response\Model\RuntimeProviderRepository; use Appwrite\Utopia\Response\Model\Session; use Appwrite\Utopia\Response\Model\Site; use Appwrite\Utopia\Response\Model\Specification; @@ -250,14 +250,14 @@ class Response extends SwooleResponse public const MODEL_INSTALLATION_LIST = 'installationList'; public const MODEL_PROVIDER_REPOSITORY = 'providerRepository'; public const MODEL_PROVIDER_REPOSITORY_LIST = 'providerRepositoryList'; - public const MODEL_FRAMEWORK_PROVIDER_REPOSITORY = 'frameworkProviderRepository'; - public const MODEL_FRAMEWORK_PROVIDER_REPOSITORY_LIST = 'frameworkProviderRepositoryList'; - public const MODEL_RUNTIME_PROVIDER_REPOSITORY = 'runtimeProviderRepository'; - public const MODEL_RUNTIME_PROVIDER_REPOSITORY_LIST = 'runtimeProviderRepositoryList'; + public const MODEL_PROVIDER_REPOSITORY_FRAMEWORK = 'providerRepositoryFramework'; + public const MODEL_PROVIDER_REPOSITORY_FRAMEWORK_LIST = 'providerRepositoryFrameworkList'; + public const MODEL_PROVIDER_REPOSITORY_RUNTIME = 'providerRepositoryRuntime'; + public const MODEL_PROVIDER_REPOSITORY_RUNTIME_LIST = 'providerRepositoryRuntimeList'; public const MODEL_BRANCH = 'branch'; public const MODEL_BRANCH_LIST = 'branchList'; - public const MODEL_FRAMEWORK_DETECTION = 'frameworkDetection'; - public const MODEL_RUNTIME_DETECTION = 'runtimeDetection'; + public const MODEL_DETECTION_FRAMEWORK = 'detectionFramework'; + public const MODEL_DETECTION_RUNTIME = 'detectionRuntime'; public const MODEL_VCS_CONTENT = 'vcsContent'; public const MODEL_VCS_CONTENT_LIST = 'vcsContentList'; @@ -383,8 +383,8 @@ class Response extends SwooleResponse ->setModel(new BaseList('Functions List', self::MODEL_FUNCTION_LIST, 'functions', self::MODEL_FUNCTION)) ->setModel(new BaseList('Function Templates List', self::MODEL_TEMPLATE_FUNCTION_LIST, 'templates', self::MODEL_TEMPLATE_FUNCTION)) ->setModel(new BaseList('Installations List', self::MODEL_INSTALLATION_LIST, 'installations', self::MODEL_INSTALLATION)) - ->setModel(new BaseList('Framework Provider Repositories List', self::MODEL_FRAMEWORK_PROVIDER_REPOSITORY_LIST, 'frameworkProviderRepositories', self::MODEL_FRAMEWORK_PROVIDER_REPOSITORY)) - ->setModel(new BaseList('Runtime Provider Repositories List', self::MODEL_RUNTIME_PROVIDER_REPOSITORY_LIST, 'runtimeProviderRepositories', self::MODEL_RUNTIME_PROVIDER_REPOSITORY)) + ->setModel(new BaseList('Framework Provider Repositories List', self::MODEL_PROVIDER_REPOSITORY_FRAMEWORK_LIST, 'frameworkProviderRepositories', self::MODEL_PROVIDER_REPOSITORY_FRAMEWORK)) + ->setModel(new BaseList('Runtime Provider Repositories List', self::MODEL_PROVIDER_REPOSITORY_RUNTIME_LIST, 'runtimeProviderRepositories', self::MODEL_PROVIDER_REPOSITORY_RUNTIME)) ->setModel(new BaseList('Branches List', self::MODEL_BRANCH_LIST, 'branches', self::MODEL_BRANCH)) ->setModel(new BaseList('Frameworks List', self::MODEL_FRAMEWORK_LIST, 'frameworks', self::MODEL_FRAMEWORK)) ->setModel(new BaseList('Runtimes List', self::MODEL_RUNTIME_LIST, 'runtimes', self::MODEL_RUNTIME)) @@ -462,10 +462,10 @@ class Response extends SwooleResponse ->setModel(new TemplateVariable()) ->setModel(new Installation()) ->setModel(new ProviderRepository()) - ->setModel(new FrameworkProviderRepository()) - ->setModel(new RuntimeProviderRepository()) - ->setModel(new FrameworkDetection()) - ->setModel(new RuntimeDetection()) + ->setModel(new ProviderRepositoryFramework()) + ->setModel(new ProviderRepositoryRuntime()) + ->setModel(new DetectionFramework()) + ->setModel(new DetectionRuntime()) ->setModel(new VcsContent()) ->setModel(new Branch()) ->setModel(new Runtime()) diff --git a/src/Appwrite/Utopia/Response/Model/FrameworkDetection.php b/src/Appwrite/Utopia/Response/Model/DetectionFramework.php similarity index 91% rename from src/Appwrite/Utopia/Response/Model/FrameworkDetection.php rename to src/Appwrite/Utopia/Response/Model/DetectionFramework.php index d0c666ae91..9aeb885f76 100644 --- a/src/Appwrite/Utopia/Response/Model/FrameworkDetection.php +++ b/src/Appwrite/Utopia/Response/Model/DetectionFramework.php @@ -5,7 +5,7 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; -class FrameworkDetection extends Model +class DetectionFramework extends Model { public function __construct() { @@ -43,7 +43,7 @@ class FrameworkDetection extends Model */ public function getName(): string { - return 'FrameworkDetection'; + return 'DetectionFramework'; } /** @@ -53,6 +53,6 @@ class FrameworkDetection extends Model */ public function getType(): string { - return Response::MODEL_FRAMEWORK_DETECTION; + return Response::MODEL_DETECTION_FRAMEWORK; } } diff --git a/src/Appwrite/Utopia/Response/Model/RuntimeDetection.php b/src/Appwrite/Utopia/Response/Model/DetectionRuntime.php similarity index 90% rename from src/Appwrite/Utopia/Response/Model/RuntimeDetection.php rename to src/Appwrite/Utopia/Response/Model/DetectionRuntime.php index b79dccff9d..1c7c0be16b 100644 --- a/src/Appwrite/Utopia/Response/Model/RuntimeDetection.php +++ b/src/Appwrite/Utopia/Response/Model/DetectionRuntime.php @@ -5,7 +5,7 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; -class RuntimeDetection extends Model +class DetectionRuntime extends Model { public function __construct() { @@ -37,7 +37,7 @@ class RuntimeDetection extends Model */ public function getName(): string { - return 'RuntimeDetection'; + return 'DetectionRuntime'; } /** @@ -47,6 +47,6 @@ class RuntimeDetection extends Model */ public function getType(): string { - return Response::MODEL_RUNTIME_DETECTION; + return Response::MODEL_DETECTION_RUNTIME; } } diff --git a/src/Appwrite/Utopia/Response/Model/FrameworkProviderRepository.php b/src/Appwrite/Utopia/Response/Model/ProviderRepositoryFramework.php similarity index 65% rename from src/Appwrite/Utopia/Response/Model/FrameworkProviderRepository.php rename to src/Appwrite/Utopia/Response/Model/ProviderRepositoryFramework.php index 2ec75badb6..24556f4453 100644 --- a/src/Appwrite/Utopia/Response/Model/FrameworkProviderRepository.php +++ b/src/Appwrite/Utopia/Response/Model/ProviderRepositoryFramework.php @@ -4,7 +4,7 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; -class FrameworkProviderRepository extends ProviderRepository +class ProviderRepositoryFramework extends ProviderRepository { public function __construct() { @@ -12,7 +12,7 @@ class FrameworkProviderRepository extends ProviderRepository $this->addRule('framework', [ 'type' => self::TYPE_STRING, - 'description' => 'Auto-detected framework suggestion. Empty if getting response of getFramework().', + 'description' => 'Auto-detected framework. Empty if type is not "framework".', 'default' => '', 'example' => 'nextjs', ]); @@ -25,7 +25,7 @@ class FrameworkProviderRepository extends ProviderRepository */ public function getName(): string { - return 'FrameworkProviderRepository'; + return 'ProviderRepositoryFramework'; } /** @@ -35,6 +35,6 @@ class FrameworkProviderRepository extends ProviderRepository */ public function getType(): string { - return Response::MODEL_FRAMEWORK_PROVIDER_REPOSITORY; + return Response::MODEL_PROVIDER_REPOSITORY_FRAMEWORK; } } diff --git a/src/Appwrite/Utopia/Response/Model/RuntimeProviderRepository.php b/src/Appwrite/Utopia/Response/Model/ProviderRepositoryRuntime.php similarity index 62% rename from src/Appwrite/Utopia/Response/Model/RuntimeProviderRepository.php rename to src/Appwrite/Utopia/Response/Model/ProviderRepositoryRuntime.php index 5e0067d23f..ea20eeea47 100644 --- a/src/Appwrite/Utopia/Response/Model/RuntimeProviderRepository.php +++ b/src/Appwrite/Utopia/Response/Model/ProviderRepositoryRuntime.php @@ -4,7 +4,7 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; -class RuntimeProviderRepository extends ProviderRepository +class ProviderRepositoryRuntime extends ProviderRepository { public function __construct() { @@ -12,9 +12,9 @@ class RuntimeProviderRepository extends ProviderRepository $this->addRule('runtime', [ 'type' => self::TYPE_STRING, - 'description' => 'Auto-detected runtime suggestion. Empty if getting response of getRuntime().', + 'description' => 'Auto-detected runtime. Empty if type is not "runtime".', 'default' => '', - 'example' => 'node', + 'example' => 'node-22', ]); } @@ -25,7 +25,7 @@ class RuntimeProviderRepository extends ProviderRepository */ public function getName(): string { - return 'RuntimeProviderRepository'; + return 'ProviderRepositoryRuntime'; } /** @@ -35,6 +35,6 @@ class RuntimeProviderRepository extends ProviderRepository */ public function getType(): string { - return Response::MODEL_RUNTIME_PROVIDER_REPOSITORY; + return Response::MODEL_PROVIDER_REPOSITORY_RUNTIME; } } diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index a105503088..9702db43de 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -135,9 +135,9 @@ class Executor * @param string $projectId * @param string $deploymentId */ - public function deleteRuntime(string $projectId, string $deploymentId) + public function deleteRuntime(string $projectId, string $deploymentId, string $suffix = '') { - $runtimeId = "$projectId-$deploymentId"; + $runtimeId = "$projectId-$deploymentId" . $suffix; $route = "/runtimes/$runtimeId"; $response = $this->call(self::METHOD_DELETE, $route, [ @@ -249,7 +249,7 @@ class Executor return $response['body']; } - public function executeCommand( + public function createCommand( string $deploymentId, string $projectId, string $command, diff --git a/tests/e2e/Services/Sites/SitesCustomServerTest.php b/tests/e2e/Services/Sites/SitesCustomServerTest.php index 162ad9c091..fd5f9e7e66 100644 --- a/tests/e2e/Services/Sites/SitesCustomServerTest.php +++ b/tests/e2e/Services/Sites/SitesCustomServerTest.php @@ -362,40 +362,168 @@ class SitesCustomServerTest extends Scope $this->cleanupSite($siteId); } - public function testRenderingDetectionSSR(): void + public function testAdapterDetectionAstroSSR(): void { $siteId = $this->setupSite([ 'siteId' => ID::unique(), - 'name' => 'Astro site', + 'name' => 'Astro SSR site', 'framework' => 'astro', - 'buildRuntime' => 'ssr-22', + 'buildRuntime' => 'node-22', 'outputDirectory' => './dist', 'buildCommand' => 'npm run build', 'installCommand' => 'npm install', ]); - $this->assertNotEmpty($siteId); + $site = $this->getSite($siteId); + $this->assertEquals('200', $site['headers']['status-code']); + $this->assertArrayHasKey('adapter', $site['body']); + $this->assertEmpty($site['body']['adapter']); + $domain = $this->setupSiteDomain($siteId); + $this->assertNotEmpty($domain); $deploymentId = $this->setupDeployment($siteId, [ 'code' => $this->packageSite('astro'), 'activate' => 'true' ]); - $this->assertNotEmpty($deploymentId); - $domain = $this->getSiteDomain($siteId); + $site = $this->getSite($siteId); + $this->assertEquals('ssr', $site['body']['adapter']); + $proxyClient = new Client(); $proxyClient->setEndpoint('http://' . $domain); - $response = $proxyClient->call(Client::METHOD_GET, '/'); - $this->assertEquals(200, $response['headers']['status-code']); $this->cleanupSite($siteId); } + public function testAdapterDetectionAstroStatic(): void + { + $siteId = $this->setupSite([ + 'siteId' => ID::unique(), + 'name' => 'Astro static site', + 'framework' => 'astro', + 'buildRuntime' => 'node-22', + 'outputDirectory' => './dist', + 'buildCommand' => 'npm run build', + 'installCommand' => 'npm install', + ]); + $this->assertNotEmpty($siteId); + + $site = $this->getSite($siteId); + $this->assertEquals('200', $site['headers']['status-code']); + $this->assertArrayHasKey('adapter', $site['body']); + $this->assertEmpty($site['body']['adapter']); + + $domain = $this->setupSiteDomain($siteId); + $this->assertNotEmpty($domain); + + $deploymentId = $this->setupDeployment($siteId, [ + 'code' => $this->packageSite('astro-static'), + 'activate' => 'true' + ]); + $this->assertNotEmpty($deploymentId); + + $site = $this->getSite($siteId); + $this->assertEquals('200', $site['headers']['status-code']); + $this->assertEquals('static', $site['body']['adapter']); + + $proxyClient = new Client(); + $proxyClient->setEndpoint('http://' . $domain); + $response = $proxyClient->call(Client::METHOD_GET, '/'); + $this->assertEquals(200, $response['headers']['status-code']); + + $this->cleanupSite($siteId); + } + + public function testAdapterDetectionStatic(): void + { + $siteId = $this->setupSite([ + 'siteId' => ID::unique(), + 'name' => 'Static site', + 'framework' => 'other', + 'buildRuntime' => 'node-22', + 'outputDirectory' => '', + 'buildCommand' => '', + 'installCommand' => '', + ]); + $this->assertNotEmpty($siteId); + + $site = $this->getSite($siteId); + $this->assertEquals('200', $site['headers']['status-code']); + $this->assertArrayHasKey('adapter', $site['body']); + $this->assertEmpty($site['body']['adapter']); + + $domain = $this->setupSiteDomain($siteId); + $this->assertNotEmpty($domain); + + $deploymentId = $this->setupDeployment($siteId, [ + 'code' => $this->packageSite('static'), + 'activate' => 'true' + ]); + $this->assertNotEmpty($deploymentId); + + $site = $this->getSite($siteId); + $this->assertEquals('200', $site['headers']['status-code']); + $this->assertEquals('static', $site['body']['adapter']); + + $proxyClient = new Client(); + $proxyClient->setEndpoint('http://' . $domain); + $response = $proxyClient->call(Client::METHOD_GET, '/'); + $this->assertEquals(200, $response['headers']['status-code']); + + $this->cleanupSite($siteId); + } + + public function testAdapterDetectionStaticSPA(): void + { + $siteId = $this->setupSite([ + 'siteId' => ID::unique(), + 'name' => 'Static site', + 'framework' => 'other', + 'buildRuntime' => 'node-22', + 'outputDirectory' => '', + 'buildCommand' => '', + 'installCommand' => '', + ]); + $this->assertNotEmpty($siteId); + + $site = $this->getSite($siteId); + $this->assertEquals('200', $site['headers']['status-code']); + $this->assertArrayHasKey('adapter', $site['body']); + $this->assertArrayHasKey('fallbackFile', $site['body']); + $this->assertEmpty($site['body']['adapter']); + $this->assertEmpty($site['body']['fallbackFile']); + + $domain = $this->setupSiteDomain($siteId); + $this->assertNotEmpty($domain); + + $deploymentId = $this->setupDeployment($siteId, [ + 'code' => $this->packageSite('static-single-file'), + 'activate' => 'true' + ]); + $this->assertNotEmpty($deploymentId); + + $site = $this->getSite($siteId); + $this->assertEquals('200', $site['headers']['status-code']); + $this->assertEquals('static', $site['body']['adapter']); + $this->assertEquals('main.html', $site['body']['fallbackFile']); + + $proxyClient = new Client(); + $proxyClient->setEndpoint('http://' . $domain); + $response = $proxyClient->call(Client::METHOD_GET, '/'); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertStringContainsString('Main page', $response['body']); + $response = $proxyClient->call(Client::METHOD_GET, '/something'); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertStringContainsString('Main page', $response['body']); + + $this->cleanupSite($siteId); + } + public function testListSites(): void { /** diff --git a/tests/e2e/Services/VCS/VCSConsoleClientTest.php b/tests/e2e/Services/VCS/VCSConsoleClientTest.php index f60b91c27f..13c3ddb251 100644 --- a/tests/e2e/Services/VCS/VCSConsoleClientTest.php +++ b/tests/e2e/Services/VCS/VCSConsoleClientTest.php @@ -154,7 +154,7 @@ class VCSConsoleClientTest extends Scope * Test for FAILURE */ - $framework = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/providerRepositories/randomRepositoryId/frameworks', array_merge([ + $framework = $this->client->call(Client::METHOD_POST, '/vcs/github/installations/' . $installationId . '/detections', array_merge([ 'x-appwrite-project' => $this->getProject()['$id'], 'content-type' => 'application/json', ], $this->getHeaders()), [ @@ -165,8 +165,6 @@ class VCSConsoleClientTest extends Scope $this->assertEquals(404, $framework['headers']['status-code']); } - // TODO: Change search to provideScenarios - /** * @depends testGitHubAuthorize */ @@ -248,7 +246,6 @@ class VCSConsoleClientTest extends Scope * Test for SUCCESS */ - //runtimes $repositories = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories', array_merge([ 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ @@ -262,28 +259,39 @@ class VCSConsoleClientTest extends Scope $this->assertEquals($repositories['body']['runtimeProviderRepositories'][0]['provider'], 'github'); $this->assertEquals($repositories['body']['runtimeProviderRepositories'][0]['runtime'], 'node-22'); - $this->assertEquals($repositories['body']['runtimeProviderRepositories'][1]['name'], 'templates-for-functions'); - $this->assertEquals($repositories['body']['runtimeProviderRepositories'][1]['runtime'], ''); - - $this->assertEquals($repositories['body']['runtimeProviderRepositories'][2]['name'], 'function1.4'); - $this->assertEquals($repositories['body']['runtimeProviderRepositories'][2]['runtime'], 'node-22'); - - $this->assertEquals($repositories['body']['runtimeProviderRepositories'][3]['name'], 'appwrite'); - $this->assertEquals($repositories['body']['runtimeProviderRepositories'][3]['runtime'], 'php-8.3'); + $searchedRepositories = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'function1.4', + 'type' => 'runtime' + ]); + $this->assertEquals(200, $searchedRepositories['headers']['status-code']); + $this->assertEquals($searchedRepositories['body']['total'], 1); + $this->assertEquals($searchedRepositories['body']['runtimeProviderRepositories'][0]['name'], 'function1.4'); + $this->assertEquals($searchedRepositories['body']['runtimeProviderRepositories'][0]['runtime'], 'node-2'); $searchedRepositories = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories', array_merge([ 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ - 'search' => 'rub', + 'search' => 'appwrite', 'type' => 'runtime' ]); + $this->assertEquals(200, $searchedRepositories['headers']['status-code']); + $this->assertEquals($searchedRepositories['body']['total'], 1); + $this->assertEquals($searchedRepositories['body']['runtimeProviderRepositories'][0]['name'], 'appwrite'); + $this->assertEquals($searchedRepositories['body']['runtimeProviderRepositories'][0]['runtime'], 'php-8.3'); + $searchedRepositories = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'ruby-starter', + 'type' => 'runtime' + ]); $this->assertEquals(200, $searchedRepositories['headers']['status-code']); $this->assertEquals($searchedRepositories['body']['total'], 1); $this->assertEquals($searchedRepositories['body']['runtimeProviderRepositories'][0]['name'], 'ruby-starter'); $this->assertEquals($searchedRepositories['body']['runtimeProviderRepositories'][0]['runtime'], 'ruby-3.3'); - // frameworks $repositories = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories', array_merge([ 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ @@ -292,20 +300,23 @@ class VCSConsoleClientTest extends Scope $this->assertEquals(200, $repositories['headers']['status-code']); $this->assertEquals($repositories['body']['total'], 4); - $this->assertEquals($repositories['body']['frameworkProviderRepositories'][0]['name'], 'starter-for-svelte'); $this->assertEquals($repositories['body']['frameworkProviderRepositories'][0]['organization'], 'appwrite-test'); $this->assertEquals($repositories['body']['frameworkProviderRepositories'][0]['provider'], 'github'); $this->assertEquals($repositories['body']['frameworkProviderRepositories'][0]['framework'], 'sveltekit'); - $this->assertEquals($repositories['body']['frameworkProviderRepositories'][1]['name'], 'templates-for-functions'); - $this->assertEquals($repositories['body']['frameworkProviderRepositories'][1]['framework'], 'other'); + $searchedRepositories = $this->client->call(Client::METHOD_GET, '/vcs/github/installations/' . $installationId . '/providerRepositories', array_merge([ + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'appwrite', + 'type' => 'runtime' + ]); + $this->assertEquals(200, $searchedRepositories['headers']['status-code']); + $this->assertEquals($searchedRepositories['body']['total'], 1); + $this->assertEquals($searchedRepositories['body']['runtimeProviderRepositories'][0]['name'], 'appwrite'); + $this->assertEquals($searchedRepositories['body']['runtimeProviderRepositories'][0]['runtime'], 'other'); - $this->assertEquals($repositories['body']['frameworkProviderRepositories'][2]['name'], 'function1.4'); - $this->assertEquals($repositories['body']['frameworkProviderRepositories'][2]['framework'], 'other'); - - $this->assertEquals($repositories['body']['frameworkProviderRepositories'][3]['name'], 'appwrite'); - $this->assertEquals($repositories['body']['frameworkProviderRepositories'][3]['framework'], 'other'); + // TODO: If you are about to add another check, rewrite this to @provideScenarios /** * Test for FAILURE diff --git a/tests/resources/sites/astro-static/astro.config.mjs b/tests/resources/sites/astro-static/astro.config.mjs new file mode 100644 index 0000000000..20b0b8e2ba --- /dev/null +++ b/tests/resources/sites/astro-static/astro.config.mjs @@ -0,0 +1,4 @@ +import { defineConfig } from 'astro/config'; + +export default defineConfig({ +}); diff --git a/tests/resources/sites/astro-static/package.json b/tests/resources/sites/astro-static/package.json new file mode 100644 index 0000000000..147dcf1f07 --- /dev/null +++ b/tests/resources/sites/astro-static/package.json @@ -0,0 +1,14 @@ +{ + "name": "my-astro-app", + "type": "module", + "version": "0.0.1", + "scripts": { + "dev": "astro dev", + "build": "astro build", + "preview": "astro preview", + "astro": "astro" + }, + "dependencies": { + "astro": "^5.2.5" + } +} diff --git a/tests/resources/sites/astro-static/src/pages/index.astro b/tests/resources/sites/astro-static/src/pages/index.astro new file mode 100644 index 0000000000..0b4f43c3d6 --- /dev/null +++ b/tests/resources/sites/astro-static/src/pages/index.astro @@ -0,0 +1,10 @@ + + + + + + Astro static + + + + diff --git a/tests/resources/sites/static-single-file/main.html b/tests/resources/sites/static-single-file/main.html new file mode 100644 index 0000000000..2615f0c820 --- /dev/null +++ b/tests/resources/sites/static-single-file/main.html @@ -0,0 +1,10 @@ + + + + + + Main page + + + + \ No newline at end of file From 75f543c6b043dda07c321919c09108d402568289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sat, 8 Mar 2025 18:01:50 +0100 Subject: [PATCH 28/31] Upgrade lib version --- composer.json | 10 ++-------- composer.lock | 49 +++++++++++-------------------------------------- 2 files changed, 13 insertions(+), 46 deletions(-) diff --git a/composer.json b/composer.json index 55d50c0b6b..1623dbc4d7 100644 --- a/composer.json +++ b/composer.json @@ -53,7 +53,7 @@ "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", "utopia-php/database": "0.59.0", - "utopia-php/detector": "dev-feat-pseudocode-draft2 as 0.1.99", + "utopia-php/detector": "0.1.*", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", @@ -105,11 +105,5 @@ "php-http/discovery": true, "tbachert/spi": true } - }, - "repositories": [ - { - "type": "vcs", - "url": "https://github.com/utopia-php/detector" - } - ] + } } diff --git a/composer.lock b/composer.lock index e3a8633120..442734db0f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "34ca1e0386c2325732532c6978cd93fb", + "content-hash": "093587189a16826d640cab3e6ca5f251", "packages": [ { "name": "adhocore/jwt", @@ -3760,16 +3760,16 @@ }, { "name": "utopia-php/detector", - "version": "dev-feat-pseudocode-draft2", + "version": "0.1.0", "source": { "type": "git", "url": "https://github.com/utopia-php/detector.git", - "reference": "e843ff2c85ed55e2cbe7e29f11583903fe002774" + "reference": "ddeee9c3e702ae10b3eb53cafe5210a0c4896c94" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/detector/zipball/e843ff2c85ed55e2cbe7e29f11583903fe002774", - "reference": "e843ff2c85ed55e2cbe7e29f11583903fe002774", + "url": "https://api.github.com/repos/utopia-php/detector/zipball/ddeee9c3e702ae10b3eb53cafe5210a0c4896c94", + "reference": "ddeee9c3e702ae10b3eb53cafe5210a0c4896c94", "shasum": "" }, "require": { @@ -3786,25 +3786,7 @@ "Utopia\\Detector\\": "src/" } }, - "autoload-dev": { - "psr-4": { - "Utopia\\Tests\\": "tests/Detector/" - } - }, - "scripts": { - "lint": [ - "./vendor/bin/pint --test --config pint.json" - ], - "format": [ - "./vendor/bin/pint --config pint.json" - ], - "check": [ - "./vendor/bin/phpstan analyse --level 8 -c phpstan.neon src tests" - ], - "test": [ - "./vendor/bin/phpunit --configuration phpunit.xml --debug" - ] - }, + "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -3816,10 +3798,10 @@ "utopia" ], "support": { - "source": "https://github.com/utopia-php/detector/tree/feat-pseudocode-draft2", - "issues": "https://github.com/utopia-php/detector/issues" + "issues": "https://github.com/utopia-php/detector/issues", + "source": "https://github.com/utopia-php/detector/tree/0.1.0" }, - "time": "2025-03-08T15:32:38+00:00" + "time": "2025-03-08T16:04:33+00:00" }, { "name": "utopia-php/domains", @@ -8856,18 +8838,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/detector", - "version": "dev-feat-pseudocode-draft2", - "alias": "0.1.99", - "alias_normalized": "0.1.99.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/detector": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 83dd6a989713e7dd2d4a1b5d798cb5e73cbef05d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sat, 8 Mar 2025 18:39:13 +0100 Subject: [PATCH 29/31] FIx tests --- app/config/site-templates.php | 12 +++--- .../Modules/Functions/Workers/Builds.php | 4 +- .../Services/Sites/SitesCustomServerTest.php | 40 +++++++++---------- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/app/config/site-templates.php b/app/config/site-templates.php index 9e0b36053e..52d2817c3d 100644 --- a/app/config/site-templates.php +++ b/app/config/site-templates.php @@ -27,7 +27,7 @@ const TEMPLATE_FRAMEWORKS = [ 'outputDirectory' => './build', 'buildRuntime' => 'node-22', 'adapter' => 'ssr', - 'fallbackFile' => null, + 'fallbackFile' => '', ], 'NEXTJS' => [ 'key' => 'nextjs', @@ -37,7 +37,7 @@ const TEMPLATE_FRAMEWORKS = [ 'outputDirectory' => './.next', 'buildRuntime' => 'node-22', 'adapter' => 'ssr', - 'fallbackFile' => null, + 'fallbackFile' => '', ], 'NUXT' => [ 'key' => 'nuxt', @@ -47,7 +47,7 @@ const TEMPLATE_FRAMEWORKS = [ 'outputDirectory' => './.output', 'buildRuntime' => 'node-22', 'adapter' => 'ssr', - 'fallbackFile' => null, + 'fallbackFile' => '', ], 'REMIX' => [ 'key' => 'remix', @@ -57,7 +57,7 @@ const TEMPLATE_FRAMEWORKS = [ 'outputDirectory' => './build', 'buildRuntime' => 'node-22', 'adapter' => 'ssr', - 'fallbackFile' => null, + 'fallbackFile' => '', ], 'ASTRO' => [ 'key' => 'astro', @@ -67,7 +67,7 @@ const TEMPLATE_FRAMEWORKS = [ 'outputDirectory' => './dist', 'buildRuntime' => 'node-22', 'adapter' => 'ssr', - 'fallbackFile' => null, + 'fallbackFile' => '', ], 'FLUTTER' => [ 'key' => 'flutter', @@ -77,7 +77,7 @@ const TEMPLATE_FRAMEWORKS = [ 'outputDirectory' => './build/web', 'buildRuntime' => 'flutter-3.24', 'adapter' => 'static', - 'fallbackFile' => null, + 'fallbackFile' => '', ], 'OTHER' => [ 'key' => 'other', diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index c5310378b9..e970b6213d 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -39,8 +39,6 @@ use Utopia\Storage\Device\Local; use Utopia\System\System; use Utopia\VCS\Adapter\Git\GitHub; -use function PHPUnit\Framework\isEmpty; - class Builds extends Action { public static function getName(): string @@ -710,7 +708,7 @@ class Builds extends Action throw new \Exception('Build size should be less than ' . number_format($buildSizeLimit / 1048576, 2) . ' MBs.'); } - if ($resource->getCollection() === 'sites' && isEmpty($resource->getAttribute('adapter'))) { + if ($resource->getCollection() === 'sites' && empty($resource->getAttribute('adapter'))) { $listFilesCommand = "cd /usr/local/build && cd " . \escapeshellarg($resource->getAttribute('outputDirectory')) . " && find . -name 'node_modules' -prune -o -type f -print"; $command = $executor->createCommand( deploymentId: $deployment->getId(), diff --git a/tests/e2e/Services/Sites/SitesCustomServerTest.php b/tests/e2e/Services/Sites/SitesCustomServerTest.php index fd5f9e7e66..12721e3dc8 100644 --- a/tests/e2e/Services/Sites/SitesCustomServerTest.php +++ b/tests/e2e/Services/Sites/SitesCustomServerTest.php @@ -64,7 +64,7 @@ class SitesCustomServerTest extends Scope */ $site = $this->createSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -112,7 +112,7 @@ class SitesCustomServerTest extends Scope 'framework' => 'other', 'buildRuntime' => 'node-22', 'outputDirectory' => './', - 'fallbackFile' => null, + 'fallbackFile' => '', ]); $this->assertNotEmpty($siteId); @@ -178,7 +178,7 @@ class SitesCustomServerTest extends Scope { $site = $this->createSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -531,7 +531,7 @@ class SitesCustomServerTest extends Scope */ $siteId = $this->setupSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -611,7 +611,7 @@ class SitesCustomServerTest extends Scope */ $siteId2 = $this->setupSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site 2', 'outputDirectory' => './', @@ -667,7 +667,7 @@ class SitesCustomServerTest extends Scope { $siteId = $this->setupSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -700,7 +700,7 @@ class SitesCustomServerTest extends Scope { $site = $this->createSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -717,7 +717,7 @@ class SitesCustomServerTest extends Scope $site = $this->updateSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site Updated', 'outputDirectory' => './', @@ -818,7 +818,7 @@ class SitesCustomServerTest extends Scope { $siteId = $this->setupSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -879,7 +879,7 @@ class SitesCustomServerTest extends Scope { $siteId = $this->setupSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -931,7 +931,7 @@ class SitesCustomServerTest extends Scope { $siteId = $this->setupSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -976,7 +976,7 @@ class SitesCustomServerTest extends Scope { $siteId = $this->setupSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -1155,7 +1155,7 @@ class SitesCustomServerTest extends Scope { $siteId = $this->setupSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -1206,7 +1206,7 @@ class SitesCustomServerTest extends Scope { $siteId = $this->setupSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -1223,7 +1223,7 @@ class SitesCustomServerTest extends Scope // Change the function specs $site = $this->updateSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -1240,7 +1240,7 @@ class SitesCustomServerTest extends Scope // Change the specs to 1vcpu 512mb $site = $this->updateSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -1260,7 +1260,7 @@ class SitesCustomServerTest extends Scope $site = $this->updateSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -1280,7 +1280,7 @@ class SitesCustomServerTest extends Scope { $siteId = $this->setupSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -1324,7 +1324,7 @@ class SitesCustomServerTest extends Scope { $siteId = $this->setupSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'outputDirectory' => './', @@ -1830,7 +1830,7 @@ class SitesCustomServerTest extends Scope { $siteId = $this->setupSite([ 'buildRuntime' => 'node-22', - 'fallbackFile' => null, + 'fallbackFile' => '', 'framework' => 'other', 'name' => 'Test Site', 'adapter' => 'static', From b69c40345fb9738cc995535a0fc4bb8922152e82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sat, 8 Mar 2025 18:45:38 +0100 Subject: [PATCH 30/31] Improve docs --- app/controllers/general.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index f6352bcd29..4e5e41b16c 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -158,7 +158,17 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw $protocol = $request->getProtocol(); - // Ensure preview authorization + /** + Ensure preview authorization + - Authorization is skippable for tests, and build screenshot + - If cookie is not sent by client -> not authorized + - If JWT in cookie is invalid or expired -> not authorized + - If user is blocked or removed -> not authorized + - If user's session is removed or expired -> not authorized + - If user is not member of team of this deployment -> not authorized + - If not authorized, redirect to Console redirect UI + - If authorized, continue as if auth was not required + */ $requirePreview = \is_null($apiKey) || !$apiKey->isPreviewAuthDisabled(); if ($isPreview && $requirePreview) { $cookie = $request->getCookie(Auth::$cookieNamePreview, ''); From 8a4954d4e88130be29d71b2f6dde16db6cf1960c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sat, 8 Mar 2025 20:44:54 +0100 Subject: [PATCH 31/31] PR review changes --- .../{function-templates.php => templates/function.php} | 0 app/config/{site-templates.php => templates/site.php} | 0 app/controllers/api/vcs.php | 6 +++--- app/init.php | 4 ++-- docker-compose.yml | 2 +- src/Appwrite/Platform/Modules/Functions/Workers/Builds.php | 1 + src/Appwrite/Platform/Tasks/Screenshot.php | 2 +- 7 files changed, 8 insertions(+), 7 deletions(-) rename app/config/{function-templates.php => templates/function.php} (100%) rename app/config/{site-templates.php => templates/site.php} (100%) diff --git a/app/config/function-templates.php b/app/config/templates/function.php similarity index 100% rename from app/config/function-templates.php rename to app/config/templates/function.php diff --git a/app/config/site-templates.php b/app/config/templates/site.php similarity index 100% rename from app/config/site-templates.php rename to app/config/templates/site.php diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index c890bf5395..7c84738592 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -865,8 +865,8 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') ]; foreach ($strategies as $strategy) { - $redector = new Runtime($strategy === Strategy::LANGUAGES ? $languages : $files, $strategy, $packager); - $redector + $detector = new Runtime($strategy === Strategy::LANGUAGES ? $languages : $files, $strategy, $packager); + $detector ->addOption(new Node()) ->addOption(new Bun()) ->addOption(new Deno()) @@ -879,7 +879,7 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories') ->addOption(new CPP()) ->addOption(new Dotnet()); - $runtime = $redector->detect(); + $runtime = $detector->detect(); if (!\is_null($runtime)) { $runtime = $runtime->getName(); diff --git a/app/init.php b/app/init.php index 6b57035ce9..ae3d96cd97 100644 --- a/app/init.php +++ b/app/init.php @@ -390,8 +390,8 @@ Config::load('storage-mimes', __DIR__ . '/config/storage/mimes.php'); Config::load('storage-inputs', __DIR__ . '/config/storage/inputs.php'); Config::load('storage-outputs', __DIR__ . '/config/storage/outputs.php'); Config::load('specifications', __DIR__ . '/config/specifications.php'); -Config::load('function-templates', __DIR__ . '/config/function-templates.php'); -Config::load('site-templates', __DIR__ . '/config/site-templates.php'); +Config::load('function-templates', __DIR__ . '/config/templates/function.php'); +Config::load('site-templates', __DIR__ . '/config/templates/site.php'); /** * New DB Filters diff --git a/docker-compose.yml b/docker-compose.yml index c50c12f206..1cf0af5078 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -976,7 +976,7 @@ services: # It's not possible to share mount file between 2 containers without host mount (copying is too slow) - /tmp:/tmp:rw environment: - - OPR_EXECUTOR_IMAGE_PULL=disabled + - OPR_EXECUTOR_IMAGE_PULL=enabled - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_COMPUTE_INACTIVE_THRESHOLD - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_COMPUTE_MAINTENANCE_INTERVAL - OPR_EXECUTOR_NETWORK=$_APP_COMPUTE_RUNTIMES_NETWORK diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index e970b6213d..0d632f6dcd 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -709,6 +709,7 @@ class Builds extends Action } if ($resource->getCollection() === 'sites' && empty($resource->getAttribute('adapter'))) { + // TODO: Refactor with structured command in future, using utopia library (CLI) $listFilesCommand = "cd /usr/local/build && cd " . \escapeshellarg($resource->getAttribute('outputDirectory')) . " && find . -name 'node_modules' -prune -o -type f -print"; $command = $executor->createCommand( deploymentId: $deployment->getId(), diff --git a/src/Appwrite/Platform/Tasks/Screenshot.php b/src/Appwrite/Platform/Tasks/Screenshot.php index 43cf434408..fb0cce3d38 100644 --- a/src/Appwrite/Platform/Tasks/Screenshot.php +++ b/src/Appwrite/Platform/Tasks/Screenshot.php @@ -34,7 +34,7 @@ class Screenshot extends Action $template = \array_shift($allowedTemplates); if (empty($template)) { - throw new \Exception("Template {$templateId} not found. Find correct ID in app/config/site-templates.php"); + throw new \Exception("Template {$templateId} not found. Find correct ID in app/config/templates/site.php"); } Console::info("Found: " . $template['name']);