From ec71ff228d20c0af3292c1754d172e973888dcfd Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 25 Mar 2026 15:25:56 +1300 Subject: [PATCH] fix: add utopia-php/async, query, database deps and address PR review comments --- composer.json | 7 + composer.lock | 190 +++++++++++++++++- src/Appwrite/Platform/Workers/Deletes.php | 7 +- .../Platform/Workers/StatsResources.php | 129 ++++++------ 4 files changed, 262 insertions(+), 71 deletions(-) diff --git a/composer.json b/composer.json index 65838a1615..3827f3c28e 100644 --- a/composer.json +++ b/composer.json @@ -60,6 +60,7 @@ "utopia-php/compression": "0.1.*", "utopia-php/config": "1.*", "utopia-php/console": "0.1.*", + "utopia-php/async": "@dev", "utopia-php/database": "5.*", "utopia-php/agents": "1.*", "utopia-php/detector": "0.2.*", @@ -111,6 +112,12 @@ "provide": { "ext-phpiredis": "*" }, + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/utopia-php/async.git" + } + ], "config": { "platform": { }, diff --git a/composer.lock b/composer.lock index 7a30e2f265..c21345b41e 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f9225f2b580de0ccb796b2fb8c881384", + "content-hash": "98a9cffeea945bf19942a259a64fdb7b", "packages": [ { "name": "adhocore/jwt", @@ -1634,6 +1634,71 @@ }, "time": "2026-01-21T04:14:03+00:00" }, + { + "name": "opis/closure", + "version": "4.5.0", + "source": { + "type": "git", + "url": "https://github.com/opis/closure.git", + "reference": "b97e42b95bb72d87507f5e2d137ceb239aea8d6b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/opis/closure/zipball/b97e42b95bb72d87507f5e2d137ceb239aea8d6b", + "reference": "b97e42b95bb72d87507f5e2d137ceb239aea8d6b", + "shasum": "" + }, + "require": { + "php": "^8.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.x-dev" + } + }, + "autoload": { + "files": [ + "src/functions.php" + ], + "psr-4": { + "Opis\\Closure\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Marius Sarca", + "email": "marius.sarca@gmail.com" + }, + { + "name": "Sorin Sarca", + "email": "sarca_sorin@hotmail.com" + } + ], + "description": "A library that can be used to serialize closures (anonymous functions) and arbitrary data.", + "homepage": "https://opis.io/closure", + "keywords": [ + "anonymous classes", + "anonymous functions", + "closure", + "function", + "serializable", + "serialization", + "serialize" + ], + "support": { + "issues": "https://github.com/opis/closure/issues", + "source": "https://github.com/opis/closure/tree/4.5.0" + }, + "time": "2026-03-05T13:32:42+00:00" + }, { "name": "paragonie/constant_time_encoding", "version": "v3.1.3", @@ -3500,6 +3565,125 @@ }, "time": "2026-02-09T12:46:39+00:00" }, + { + "name": "utopia-php/async", + "version": "dev-main", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/async.git", + "reference": "7a0c6957b41731a5c999382ad26a0b2fdbd19812" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/async/zipball/7a0c6957b41731a5c999382ad26a0b2fdbd19812", + "reference": "7a0c6957b41731a5c999382ad26a0b2fdbd19812", + "shasum": "" + }, + "require": { + "opis/closure": "4.*", + "php": ">=8.1" + }, + "require-dev": { + "amphp/amp": "3.*", + "amphp/parallel": "2.*", + "amphp/process": "^2.0", + "laravel/pint": "1.*", + "phpstan/phpstan": "2.*", + "phpunit/phpunit": "11.5.45", + "react/child-process": "0.*", + "react/event-loop": "1.*", + "swoole/ide-helper": "*" + }, + "suggest": { + "amphp/amp": "Required for Amp promise adapter", + "amphp/parallel": "Required for Amp parallel adapter", + "ext-ev": "Required for ReactPHP event loop (recommended for best performance)", + "ext-parallel": "Required for parallel adapter (requires PHP ZTS build)", + "ext-sockets": "Required for Swoole Process adapter", + "ext-swoole": "Required for Swoole Thread and Process adapters (recommended for best performance)", + "react/child-process": "Required for ReactPHP parallel adapter", + "react/event-loop": "Required for ReactPHP promise and parallel adapters" + }, + "default-branch": true, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Async\\": "src/" + } + }, + "autoload-dev": { + "psr-4": { + "Utopia\\Tests\\": "tests/" + } + }, + "scripts": { + "test-unit": [ + "vendor/bin/phpunit tests/Unit --exclude-group no-swoole" + ], + "test-promise-sync": [ + "vendor/bin/phpunit tests/E2e/Promise/SyncTest.php" + ], + "test-promise-swoole": [ + "vendor/bin/phpunit tests/E2e/Promise/Swoole" + ], + "test-promise-amp": [ + "vendor/bin/phpunit tests/E2e/Promise/Amp" + ], + "test-promise-react": [ + "vendor/bin/phpunit tests/E2e/Promise/React" + ], + "test-parallel-sync": [ + "vendor/bin/phpunit tests/E2e/Parallel/Sync" + ], + "test-parallel-swoole-thread": [ + "vendor/bin/phpunit tests/E2e/Parallel/Swoole/ThreadTest.php" + ], + "test-parallel-swoole-process": [ + "vendor/bin/phpunit tests/E2e/Parallel/Swoole/ProcessTest.php" + ], + "test-parallel-amp": [ + "vendor/bin/phpunit tests/E2e/Parallel/Amp" + ], + "test-parallel-react": [ + "vendor/bin/phpunit tests/E2e/Parallel/React" + ], + "test-parallel-ext": [ + "php -n -d extension=parallel.so -d extension=sockets.so vendor/bin/phpunit tests/E2e/Parallel/Parallel" + ], + "test-e2e": [ + "vendor/bin/phpunit tests/E2e --exclude-group ext-parallel" + ], + "test": [ + "@test-unit", + "@test-e2e", + "@test-parallel-ext" + ], + "lint": [ + "vendor/bin/pint" + ], + "format": [ + "php -d memory_limit=4G vendor/bin/pint" + ], + "check": [ + "vendor/bin/phpstan analyse src tests --level=max --memory-limit=4G" + ] + }, + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Appwrite Team", + "email": "team@appwrite.io" + } + ], + "description": "High-performance concurrent + parallel library with Promise and Parallel execution support for PHP.", + "support": { + "source": "https://github.com/utopia-php/async/tree/main", + "issues": "https://github.com/utopia-php/async/issues" + }, + "time": "2026-01-09T06:16:09+00:00" + }, { "name": "utopia-php/audit", "version": "2.2.1", @@ -8435,7 +8619,9 @@ ], "aliases": [], "minimum-stability": "dev", - "stability-flags": {}, + "stability-flags": { + "utopia-php/async": 20 + }, "prefer-stable": true, "prefer-lowest": false, "platform": { diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index caee5c3321..dada68c17e 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -1141,8 +1141,11 @@ class Deletes extends Action $functionId = $document->getId(); $functionInternalId = $document->getSequence(); - // Delete function resources concurrently + // Request executor to delete all deployment containers before deleting deployments Console::info("Deleting resources for function " . $functionId); + $this->deleteRuntimes($getProjectDB, $document, $project, $executor); + + // Delete function resources concurrently $deploymentInternalIds = []; Promise::map([ // Delete rules @@ -1193,8 +1196,6 @@ class Deletes extends Action Query::orderAsc() ], $dbForPlatform); }), - // Request executor to delete all deployment containers - fn () => $this->deleteRuntimes($getProjectDB, $document, $project, $executor), ])->await(); } diff --git a/src/Appwrite/Platform/Workers/StatsResources.php b/src/Appwrite/Platform/Workers/StatsResources.php index d9ae20f762..bf39214a02 100644 --- a/src/Appwrite/Platform/Workers/StatsResources.php +++ b/src/Appwrite/Platform/Workers/StatsResources.php @@ -95,7 +95,7 @@ class StatsResources extends Action $region = $project->getAttribute('region'); $last30Days = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days'))->format('Y-m-d 00:00:00'); - $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours'))->format('Y-m-d h:m:00'); + $last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours'))->format('Y-m-d H:i:00'); $last7Days = (new \DateTime())->sub(\DateInterval::createFromDateString('7 days'))->format('Y-m-d 00:00:00'); $results = Promise::map([ @@ -220,7 +220,7 @@ class StatsResources extends Action 'storage' => fn () => $dbForProject->sum('bucket_' . $bucket->getSequence(), 'sizeActual'), ])->await(); } catch (Throwable $th) { - call_user_func_array($this->logError, [$th, "StatsResources", "count_for_bucket_{$bucket->getSequence()}"]); + call_user_func_array($this->logError, [$th, "StatsResources", "bucket_{$bucket->getSequence()}"]); return; } @@ -347,10 +347,15 @@ class StatsResources extends Action $this->foreachDocument($dbForProject, 'database_' . $database->getSequence(), [], function ($collection) use ($dbForDatabases, $database, $region, &$databaseStorage, &$databaseDocuments, $databaseIdCollectionIdDocumentsMetric, $databaseIdCollectionIdStorageMetric) { $collectionName = 'database_' . $database->getSequence() . '_collection_' . $collection->getSequence(); - $collectionResults = Promise::map([ - 'documents' => fn () => $dbForDatabases->count($collectionName), - 'storage' => fn () => $dbForDatabases->getSizeOfCollection($collectionName), - ])->await(); + try { + $collectionResults = Promise::map([ + 'documents' => fn () => $dbForDatabases->count($collectionName), + 'storage' => fn () => $dbForDatabases->getSizeOfCollection($collectionName), + ])->await(); + } catch (Throwable $th) { + call_user_func_array($this->logError, [$th, "StatsResources", "collection_{$database->getSequence()}_{$collection->getSequence()}"]); + return; + } $metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getSequence(), $collection->getSequence()], $databaseIdCollectionIdDocumentsMetric); $this->createStatsDocuments($region, $metric, $collectionResults['documents']); @@ -406,40 +411,36 @@ class StatsResources extends Action $this->createStatsDocuments($region, str_replace("{resourceType}", RESOURCE_TYPE_FUNCTIONS, METRIC_RESOURCE_TYPE_DEPLOYMENTS), $results['deployments']); $this->createStatsDocuments($region, str_replace("{resourceType}", RESOURCE_TYPE_FUNCTIONS, METRIC_RESOURCE_TYPE_BUILDS), $results['deployments']); - // Per-function deployment stats via groupBy (replaces N+1 loop) - $groupedDeployments = $dbForProject->find('deployments', [ - Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), - Query::groupBy(['resourceInternalId']), - Query::count('*', 'deploymentsCount'), - Query::sum('sourceSize', 'deploymentsStorage'), - Query::sum('buildSize', 'buildsStorage'), - ]); + $runtimes = []; + $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $region, &$runtimes) { + $functionResults = Promise::map([ + 'deploymentsStorage' => fn () => $dbForProject->sum('deployments', 'sourceSize', [ + Query::equal('resourceInternalId', [$function->getSequence()]), + Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), + ]), + 'deployments' => fn () => $dbForProject->count('deployments', [ + Query::equal('resourceInternalId', [$function->getSequence()]), + Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), + ]), + 'buildsStorage' => fn () => $dbForProject->sum('deployments', 'buildSize', [ + Query::equal('resourceInternalId', [$function->getSequence()]), + Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]), + ]), + ])->await(); - foreach ($groupedDeployments as $row) { - $resourceInternalId = $row->getAttribute('resourceInternalId'); - $deploymentsCount = $row->getAttribute('deploymentsCount', 0); - $deploymentsStorage = $row->getAttribute('deploymentsStorage', 0); - $buildsStorage = $row->getAttribute('buildsStorage', 0); + $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $function->getSequence()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE), $functionResults['deploymentsStorage']); + $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $function->getSequence()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS), $functionResults['deployments']); + $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $function->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS), $functionResults['deployments']); + $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $function->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $functionResults['buildsStorage']); - $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE), $deploymentsStorage); - $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS), $deploymentsCount); - $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_BUILDS), $deploymentsCount); - $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $buildsStorage); - } - - // Count runtimes via groupBy (replaces N+1 loop) - $runtimeGroups = $dbForProject->find('functions', [ - Query::select(['runtime']), - Query::groupBy(['runtime']), - Query::count('*', 'count'), - Query::isNotNull('runtime'), - ]); - - foreach ($runtimeGroups as $row) { - $runtime = $row->getAttribute('runtime'); + $runtime = $function->getAttribute('runtime'); if (!empty($runtime)) { - $this->createStatsDocuments($region, str_replace('{runtime}', $runtime, METRIC_FUNCTIONS_RUNTIME), $row->getAttribute('count', 0)); + $runtimes[$runtime] = ($runtimes[$runtime] ?? 0) + 1; } + }); + + foreach ($runtimes as $runtime => $count) { + $this->createStatsDocuments($region, str_replace('{runtime}', $runtime, METRIC_FUNCTIONS_RUNTIME), $count); } } @@ -462,40 +463,36 @@ class StatsResources extends Action $this->createStatsDocuments($region, str_replace("{resourceType}", RESOURCE_TYPE_SITES, METRIC_RESOURCE_TYPE_DEPLOYMENTS), $results['deployments']); $this->createStatsDocuments($region, str_replace("{resourceType}", RESOURCE_TYPE_SITES, METRIC_RESOURCE_TYPE_BUILDS), $results['deployments']); - // Per-site deployment stats via groupBy (replaces N+1 loop) - $groupedDeployments = $dbForProject->find('deployments', [ - Query::equal('resourceType', [RESOURCE_TYPE_SITES]), - Query::groupBy(['resourceInternalId']), - Query::count('*', 'deploymentsCount'), - Query::sum('sourceSize', 'deploymentsStorage'), - Query::sum('buildSize', 'buildsStorage'), - ]); + $frameworks = []; + $this->foreachDocument($dbForProject, 'sites', [], function (Document $site) use ($dbForProject, $region, &$frameworks) { + $siteResults = Promise::map([ + 'deploymentsStorage' => fn () => $dbForProject->sum('deployments', 'sourceSize', [ + Query::equal('resourceInternalId', [$site->getSequence()]), + Query::equal('resourceType', [RESOURCE_TYPE_SITES]), + ]), + 'deployments' => fn () => $dbForProject->count('deployments', [ + Query::equal('resourceInternalId', [$site->getSequence()]), + Query::equal('resourceType', [RESOURCE_TYPE_SITES]), + ]), + 'buildsStorage' => fn () => $dbForProject->sum('deployments', 'buildSize', [ + Query::equal('resourceInternalId', [$site->getSequence()]), + Query::equal('resourceType', [RESOURCE_TYPE_SITES]), + ]), + ])->await(); - foreach ($groupedDeployments as $row) { - $resourceInternalId = $row->getAttribute('resourceInternalId'); - $deploymentsCount = $row->getAttribute('deploymentsCount', 0); - $deploymentsStorage = $row->getAttribute('deploymentsStorage', 0); - $buildsStorage = $row->getAttribute('buildsStorage', 0); + $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $site->getSequence()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE), $siteResults['deploymentsStorage']); + $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $site->getSequence()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS), $siteResults['deployments']); + $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $site->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS), $siteResults['deployments']); + $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $site->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $siteResults['buildsStorage']); - $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE), $deploymentsStorage); - $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS), $deploymentsCount); - $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_BUILDS), $deploymentsCount); - $this->createStatsDocuments($region, str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $buildsStorage); - } - - // Count frameworks via groupBy (replaces N+1 loop) - $frameworkGroups = $dbForProject->find('sites', [ - Query::select(['framework']), - Query::groupBy(['framework']), - Query::count('*', 'count'), - Query::isNotNull('framework'), - ]); - - foreach ($frameworkGroups as $row) { - $framework = $row->getAttribute('framework'); + $framework = $site->getAttribute('framework'); if (!empty($framework)) { - $this->createStatsDocuments($region, str_replace('{framework}', $framework, METRIC_SITES_FRAMEWORK), $row->getAttribute('count', 0)); + $frameworks[$framework] = ($frameworks[$framework] ?? 0) + 1; } + }); + + foreach ($frameworks as $framework => $count) { + $this->createStatsDocuments($region, str_replace('{framework}', $framework, METRIC_SITES_FRAMEWORK), $count); } }