From 8df22db3eb26b0c1a817d8b6c81afca2333d9c01 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 23 Dec 2025 17:33:34 +0530 Subject: [PATCH 01/13] added swoole based pools for realtime --- app/init/registers.php | 39 +++++++++++++++++++++++++-------------- app/realtime.php | 10 +++++----- composer.json | 8 +++++++- composer.lock | 34 ++++++++++++++++++++++------------ 4 files changed, 59 insertions(+), 32 deletions(-) diff --git a/app/init/registers.php b/app/init/registers.php index be2009449e..fc2111b15c 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -23,6 +23,8 @@ use Utopia\Logger\Adapter\LogOwl; use Utopia\Logger\Adapter\Raygun; use Utopia\Logger\Adapter\Sentry; use Utopia\Logger\Logger; +use Utopia\Pools\Adapter\Stack as Stack; +use Utopia\Pools\Adapter\Swoole as SwoolePool; use Utopia\Pools\Group; use Utopia\Pools\Pool; use Utopia\Queue; @@ -143,7 +145,14 @@ $register->set('realtimeLogger', function () { return new Logger($adapter); }); -$register->set('pools', function () { +/** + * Build a pool Group with shared config. + * + * @param string $configPrefix Config param prefix (e.g. 'pools', 'coroutinepools') + * @param callable(): \Utopia\Pools\Adapter $adapterFactory Factory returning the Pool adapter (Stack or Swoole) + * @param int|null $syncTimeout Optional synchronization timeout to apply on each pool (null to skip) + */ +$buildPoolGroup = function (string $configPrefix, callable $adapterFactory, ?int $syncTimeout = null): Group { $group = new Group(); $fallbackForDB = 'db_main=' . AppwriteURL::unparse([ @@ -236,7 +245,6 @@ $register->set('pools', function () { $dsn = $dsn[1] ?? ''; $config[] = $name; if (empty($dsn)) { - //throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}"); continue; } @@ -252,13 +260,6 @@ $register->set('pools', function () { throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); } - /** - * Get Resource - * - * Creation could be reused across connection types like database, cache, queue, etc. - * - * Resource assignment to an adapter will happen below. - */ $resource = match ($dsnScheme) { 'mysql', 'mariadb' => function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { @@ -285,8 +286,8 @@ $register->set('pools', function () { default => throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Invalid scheme'), }; - $pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) { - // Get Adapter + $poolAdapter = $adapterFactory(); + $pool = new Pool($poolAdapter, $name, $poolSize, function () use ($type, $resource, $dsn) { switch ($type) { case 'database': $adapter = match ($dsn->getScheme()) { @@ -294,7 +295,6 @@ $register->set('pools', function () { 'mysql' => new MySQL($resource()), default => null }; - $adapter->setDatabase($dsn->getPath()); return $adapter; case 'pubsub': @@ -318,14 +318,25 @@ $register->set('pools', function () { } }); + if ($syncTimeout !== null) { + $pool->setSynchronizationTimeout($syncTimeout); + } + $group->add($pool); } - Config::setParam('pools-' . $key, $config); + Config::setParam($configPrefix . '-' . $key, $config); } return $group; -}); +}; + +$register->set('pools', fn () => $buildPoolGroup('pools', fn () => new Stack(), null)); + +/** + * Separate pool group for async/realtime contexts, using Swoole adapter and 10s sync timeout. + */ +$register->set('coroutinepools', fn () => $buildPoolGroup('coroutinepools', fn () => new SwoolePool(), 10)); $register->set('db', function () { // This is usually for our workers or CLI commands scope diff --git a/app/realtime.php b/app/realtime.php index fab0ce7561..d25d952eff 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -62,7 +62,7 @@ if (!function_exists('getConsoleDB')) { global $register; /** @var Group $pools */ - $pools = $register->get('pools'); + $pools = $register->get('coroutinepools'); $adapter = new DatabasePool($pools->get('console')); $database = new Database($adapter, getCache()); @@ -92,7 +92,7 @@ if (!function_exists('getProjectDB')) { global $register; /** @var Group $pools */ - $pools = $register->get('pools'); + $pools = $register->get('coroutinepools'); if ($project->isEmpty() || $project->getId() === 'console') { return getConsoleDB(); @@ -144,7 +144,7 @@ if (!function_exists('getCache')) { global $register; - $pools = $register->get('pools'); /** @var Group $pools */ + $pools = $register->get('coroutinepools'); /** @var Group $pools */ $list = Config::getParam('pools-cache', []); $adapters = []; @@ -445,7 +445,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, } $start = time(); - $pubsub = new PubSubPool($register->get('pools')->get('pubsub')); + $pubsub = new PubSubPool($register->get('coroutinepools')->get('pubsub')); if ($pubsub->ping(true)) { $attempts = 0; @@ -519,7 +519,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, Console::info("Connection open (user: {$connection})"); - App::setResource('pools', fn () => $register->get('pools')); + App::setResource('pools', fn () => $register->get('coroutinepools')); App::setResource('request', fn () => $request); App::setResource('response', fn () => $response); diff --git a/composer.json b/composer.json index d32b739311..c71ad4aad6 100644 --- a/composer.json +++ b/composer.json @@ -29,6 +29,12 @@ "Appwrite\\Tests\\": "tests/extensions" } }, + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/utopia-php/pools" + } + ], "require": { "php": ">=8.3.0", "ext-curl": "*", @@ -67,7 +73,7 @@ "utopia-php/migration": "1.3.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", - "utopia-php/pools": "0.8.*", + "utopia-php/pools": "dev-dat-966 as 0.8.3", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.11.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index 4ffc7e7db4..ca24859965 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "7c9cb03eb5267f1e7a3ffc037ae22b6a", + "content-hash": "8ae1ada02b140a48fd3ce9ae74c95cad", "packages": [ { "name": "adhocore/jwt", @@ -4730,26 +4730,27 @@ }, { "name": "utopia-php/pools", - "version": "0.8.2", + "version": "dev-dat-966", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "05c67aba42eb68ac65489cc1e7fc5db83db2dd4d" + "reference": "c32a92afd4f1b4ad524a79fa0b04fc253317cc2d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/05c67aba42eb68ac65489cc1e7fc5db83db2dd4d", - "reference": "05c67aba42eb68ac65489cc1e7fc5db83db2dd4d", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/c32a92afd4f1b4ad524a79fa0b04fc253317cc2d", + "reference": "c32a92afd4f1b4ad524a79fa0b04fc253317cc2d", "shasum": "" }, "require": { - "php": ">=8.3", - "utopia-php/telemetry": "0.1.*" + "php": ">=8.4", + "utopia-php/telemetry": "*" }, "require-dev": { "laravel/pint": "1.*", "phpstan/phpstan": "1.*", - "phpunit/phpunit": "11.*" + "phpunit/phpunit": "11.*", + "swoole/ide-helper": "5.1.2" }, "type": "library", "autoload": { @@ -4776,9 +4777,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.8.2" + "source": "https://github.com/utopia-php/pools/tree/dat-966" }, - "time": "2025-04-17T02:04:54+00:00" + "time": "2025-12-22T13:50:08+00:00" }, { "name": "utopia-php/preloader", @@ -8941,9 +8942,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/pools", + "version": "dev-dat-966", + "alias": "0.8.3", + "alias_normalized": "0.8.3.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/pools": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From 7e4d40454992b0d44ab9c0ad98ac8c087d81b3f6 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 23 Dec 2025 17:38:52 +0530 Subject: [PATCH 02/13] updated composer json --- composer.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/composer.json b/composer.json index c71ad4aad6..4663bd3522 100644 --- a/composer.json +++ b/composer.json @@ -29,12 +29,6 @@ "Appwrite\\Tests\\": "tests/extensions" } }, - "repositories": [ - { - "type": "vcs", - "url": "https://github.com/utopia-php/pools" - } - ], "require": { "php": ">=8.3.0", "ext-curl": "*", From c03cd258ceb542515829998b080764a5d451be0a Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 23 Dec 2025 18:30:53 +0530 Subject: [PATCH 03/13] changed the pool size distribution --- app/init/registers.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init/registers.php b/app/init/registers.php index fc2111b15c..f6b3234e39 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -231,7 +231,7 @@ $buildPoolGroup = function (string $configPrefix, callable $adapterFactory, ?int throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500); } - $poolSize = (int)($instanceConnections / $workerCount); + $poolSize = (int)(($instanceConnections / $workerCount)/2); foreach ($connections as $key => $connection) { $type = $connection['type'] ?? ''; From 62e881b7e94f999e4f2973118e00822d6f9446dc Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 23 Dec 2025 20:34:59 +0530 Subject: [PATCH 04/13] reverted the use retry mechanism of the pools --- composer.json | 2 +- composer.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/composer.json b/composer.json index 4663bd3522..aa183e4865 100644 --- a/composer.json +++ b/composer.json @@ -67,7 +67,7 @@ "utopia-php/migration": "1.3.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", - "utopia-php/pools": "dev-dat-966 as 0.8.3", + "utopia-php/pools": "dev-dat-966#a70164f as 0.8.3", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.11.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index ca24859965..1c6e50dcb6 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8ae1ada02b140a48fd3ce9ae74c95cad", + "content-hash": "59fcb531753c6e05c69624cd93d6360d", "packages": [ { "name": "adhocore/jwt", @@ -4734,12 +4734,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "c32a92afd4f1b4ad524a79fa0b04fc253317cc2d" + "reference": "a70164f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/c32a92afd4f1b4ad524a79fa0b04fc253317cc2d", - "reference": "c32a92afd4f1b4ad524a79fa0b04fc253317cc2d", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/a70164f", + "reference": "a70164f", "shasum": "" }, "require": { From f7668e041151296302595513ab581703ab9ea29f Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 24 Dec 2025 13:41:31 +0530 Subject: [PATCH 05/13] * added concurrent traffic load tests for realtime * added assert eventual case for project console * linting --- app/init/registers.php | 2 +- .../Projects/ProjectsConsoleClientTest.php | 2 +- .../Realtime/RealtimeCustomClientTest.php | 150 ++++++++++++++++++ 3 files changed, 152 insertions(+), 2 deletions(-) diff --git a/app/init/registers.php b/app/init/registers.php index f6b3234e39..c5bba06df8 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -231,7 +231,7 @@ $buildPoolGroup = function (string $configPrefix, callable $adapterFactory, ?int throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500); } - $poolSize = (int)(($instanceConnections / $workerCount)/2); + $poolSize = (int)(($instanceConnections / $workerCount) / 2); foreach ($connections as $key => $connection) { $type = $connection['type'] ?? ''; diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 769d3a4c85..6c77303fd8 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -1867,7 +1867,7 @@ class ProjectsConsoleClientTest extends Scope $this->assertEquals(1, count($sessions)); $this->assertEquals($sessionId2, $sessions[0]['$id']); - }); + }, 30000); /** * Reset Limit diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php index c6a1686864..847737b86c 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php @@ -4,6 +4,7 @@ namespace Tests\E2E\Services\Realtime; use CURLFile; use Exception; +use Swoole\Coroutine; use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; @@ -3122,4 +3123,153 @@ class RealtimeCustomClientTest extends Scope $client->close(); } + + /** + * Simulate concurrent realtime traffic using Swoole coroutines. + * Opens multiple websocket clients concurrently, then performs create/update/delete ops. + */ + public function testConcurrentRealtimeTrafficCoroutines() + { + if (!class_exists(\Swoole\Coroutine::class)) { + $this->markTestSkipped('Swoole Coroutine not available in this environment.'); + } + + $user = $this->getUser(); + $session = $user['session'] ?? ''; + $projectId = $this->getProject()['$id']; + + Coroutine\run(function () use ($session, $projectId) { + $headers = [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_' . $projectId . '=' . $session + ]; + + $clientCount = 5; + $clients = []; + for ($i = 0; $i < $clientCount; $i++) { + $clients[] = $this->getWebsocket(['documents', 'collections'], $headers); + } + + foreach ($clients as $client) { + $response = json_decode($client->receive(), true); + $this->assertEquals('connected', $response['type']); + } + + // Setup DB/collection/attribute + $database = $this->client->call(Client::METHOD_POST, '/databases', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'databaseId' => ID::unique(), + 'name' => 'Concurrent DB', + ]); + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'Concurrent Collection', + 'permissions' => [ + Permission::create(Role::user($this->getUser()['$id'])), + ], + 'documentSecurity' => true, + ]); + $collectionId = $collection['body']['$id']; + + $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collectionId}/attributes/string", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'name', + 'size' => 64, + 'required' => true, + ]); + + Coroutine::sleep(1); + + $creates = [ + ['name' => 'Doc A'], + ['name' => 'Doc B'], + ['name' => 'Doc C'], + ['name' => 'Doc D'], + ['name' => 'Doc E'], + ['name' => 'Doc F'], + ]; + + $expectedEvents = count($creates); + + // Per-client receipts + $receivedEvents = array_fill(0, $clientCount, []); + + // Launch receiver coroutines (one per client) + foreach ($clients as $idx => $client) { + Coroutine::create(function () use ($client, &$receivedEvents, $expectedEvents, $idx) { + $local = []; + for ($i = 0; $i < $expectedEvents; $i++) { + $event = json_decode($client->receive(), true); + $local[] = $event; + } + $receivedEvents[$idx] = $local; + }); + } + + // Create docs + foreach ($creates as $payload) { + $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collectionId}/documents", array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'documentId' => ID::unique(), + 'data' => $payload, + 'permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + } + + // Wait for receivers to collect; timeout ~10s + $deadline = microtime(true) + 10; + while (microtime(true) < $deadline) { + $done = true; + foreach ($receivedEvents as $events) { + if (count($events) < $expectedEvents) { + $done = false; + break; + } + } + if ($done) { + break; + } + Coroutine::sleep(0.1); + } + + $expectedNames = array_column($creates, 'name'); + + for ($c = 0; $c < $clientCount; $c++) { + $events = $receivedEvents[$c]; + $this->assertCount($expectedEvents, $events, 'Unexpected event count on client '.$c); + $seen = []; + foreach ($events as $event) { + $this->assertEquals('event', $event['type']); + $this->assertArrayHasKey('payload', $event['data']); + $seen[] = $event['data']['payload']['name'] ?? ''; + } + foreach ($expectedNames as $name) { + $this->assertContains($name, $seen); + } + } + + foreach ($clients as $client) { + $client->close(); + } + }); + } } From 2baa33351dc1e285ff5ee74e70761fc9c1199697 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 24 Dec 2025 14:38:43 +0530 Subject: [PATCH 06/13] fix tests --- tests/e2e/Services/Projects/ProjectsConsoleClientTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 6c77303fd8..d24434845f 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -1867,7 +1867,7 @@ class ProjectsConsoleClientTest extends Scope $this->assertEquals(1, count($sessions)); $this->assertEquals($sessionId2, $sessions[0]['$id']); - }, 30000); + }, 30000,300); /** * Reset Limit From f36dd2748c4454ccc5f9a6560880491ac22b16f0 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 24 Dec 2025 16:22:02 +0530 Subject: [PATCH 07/13] fixed tests --- tests/e2e/Services/Projects/ProjectsConsoleClientTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index d24434845f..ab112d4edb 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -1867,7 +1867,7 @@ class ProjectsConsoleClientTest extends Scope $this->assertEquals(1, count($sessions)); $this->assertEquals($sessionId2, $sessions[0]['$id']); - }, 30000,300); + }, 120_000, 300); /** * Reset Limit From 00e00ff166dcc2da1765b9e53f9f84e0125ce3eb Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 24 Dec 2025 16:55:13 +0530 Subject: [PATCH 08/13] temp-check: force purging cache --- app/controllers/shared/api.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 83b56f626a..4f1e723c46 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -777,6 +777,8 @@ App::shutdown() return; } + // Purge cache to ensure we get fresh session count + $dbForProject->purgeCachedDocument('users', $userId); $user = $dbForProject->getDocument('users', $userId); if ($user->isEmpty()) { return; From 11f23d99fb81ec4dd0d619d96ff42d9abb7a65dc Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 24 Dec 2025 17:09:01 +0530 Subject: [PATCH 09/13] Revert "temp-check: force purging cache" This reverts commit 00e00ff166dcc2da1765b9e53f9f84e0125ce3eb. --- app/controllers/shared/api.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 4f1e723c46..83b56f626a 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -777,8 +777,6 @@ App::shutdown() return; } - // Purge cache to ensure we get fresh session count - $dbForProject->purgeCachedDocument('users', $userId); $user = $dbForProject->getDocument('users', $userId); if ($user->isEmpty()) { return; From 2e82f0c2ece473cc96cf76c324a7e11bfa59a66d Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 30 Dec 2025 13:03:59 +0530 Subject: [PATCH 10/13] fix: update coroutine pool handling and configuration --- .env | 3 ++- app/init/registers.php | 41 +++++++++++++++++------------------------ app/realtime.php | 10 +++++----- docker-compose.yml | 1 + 4 files changed, 25 insertions(+), 30 deletions(-) diff --git a/.env b/.env index e849e83801..ff989ec9e1 100644 --- a/.env +++ b/.env @@ -125,4 +125,5 @@ _APP_WEBHOOK_MAX_FAILED_ATTEMPTS=10 _APP_PROJECT_REGIONS=default _APP_FUNCTIONS_CREATION_ABUSE_LIMIT=5000 _APP_STATS_USAGE_DUAL_WRITING_DBS=database_db_main -_APP_TRUSTED_HEADERS=x-forwarded-for \ No newline at end of file +_APP_TRUSTED_HEADERS=x-forwarded-for +COROUTINE_POOLS=disabled \ No newline at end of file diff --git a/app/init/registers.php b/app/init/registers.php index c5bba06df8..ab88d5fbbd 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -23,7 +23,7 @@ use Utopia\Logger\Adapter\LogOwl; use Utopia\Logger\Adapter\Raygun; use Utopia\Logger\Adapter\Sentry; use Utopia\Logger\Logger; -use Utopia\Pools\Adapter\Stack as Stack; +use Utopia\Pools\Adapter\Stack as StackPool; use Utopia\Pools\Adapter\Swoole as SwoolePool; use Utopia\Pools\Group; use Utopia\Pools\Pool; @@ -145,14 +145,7 @@ $register->set('realtimeLogger', function () { return new Logger($adapter); }); -/** - * Build a pool Group with shared config. - * - * @param string $configPrefix Config param prefix (e.g. 'pools', 'coroutinepools') - * @param callable(): \Utopia\Pools\Adapter $adapterFactory Factory returning the Pool adapter (Stack or Swoole) - * @param int|null $syncTimeout Optional synchronization timeout to apply on each pool (null to skip) - */ -$buildPoolGroup = function (string $configPrefix, callable $adapterFactory, ?int $syncTimeout = null): Group { +$register->set('pools', function () { $group = new Group(); $fallbackForDB = 'db_main=' . AppwriteURL::unparse([ @@ -231,7 +224,7 @@ $buildPoolGroup = function (string $configPrefix, callable $adapterFactory, ?int throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500); } - $poolSize = (int)(($instanceConnections / $workerCount) / 2); + $poolSize = (int)($instanceConnections / $workerCount); foreach ($connections as $key => $connection) { $type = $connection['type'] ?? ''; @@ -245,6 +238,7 @@ $buildPoolGroup = function (string $configPrefix, callable $adapterFactory, ?int $dsn = $dsn[1] ?? ''; $config[] = $name; if (empty($dsn)) { + //throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}"); continue; } @@ -260,6 +254,13 @@ $buildPoolGroup = function (string $configPrefix, callable $adapterFactory, ?int throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); } + /** + * Get Resource + * + * Creation could be reused across connection types like database, cache, queue, etc. + * + * Resource assignment to an adapter will happen below. + */ $resource = match ($dsnScheme) { 'mysql', 'mariadb' => function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { @@ -286,8 +287,10 @@ $buildPoolGroup = function (string $configPrefix, callable $adapterFactory, ?int default => throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Invalid scheme'), }; - $poolAdapter = $adapterFactory(); + $poolAdapter = System::getEnv('COROUTINE_POOLS', 'disabled') === 'enabled' ? new SwoolePool() : new StackPool(); + $pool = new Pool($poolAdapter, $name, $poolSize, function () use ($type, $resource, $dsn) { + // Get Adapter switch ($type) { case 'database': $adapter = match ($dsn->getScheme()) { @@ -295,6 +298,7 @@ $buildPoolGroup = function (string $configPrefix, callable $adapterFactory, ?int 'mysql' => new MySQL($resource()), default => null }; + $adapter->setDatabase($dsn->getPath()); return $adapter; case 'pubsub': @@ -318,25 +322,14 @@ $buildPoolGroup = function (string $configPrefix, callable $adapterFactory, ?int } }); - if ($syncTimeout !== null) { - $pool->setSynchronizationTimeout($syncTimeout); - } - $group->add($pool); } - Config::setParam($configPrefix . '-' . $key, $config); + Config::setParam('pools-' . $key, $config); } return $group; -}; - -$register->set('pools', fn () => $buildPoolGroup('pools', fn () => new Stack(), null)); - -/** - * Separate pool group for async/realtime contexts, using Swoole adapter and 10s sync timeout. - */ -$register->set('coroutinepools', fn () => $buildPoolGroup('coroutinepools', fn () => new SwoolePool(), 10)); +}); $register->set('db', function () { // This is usually for our workers or CLI commands scope diff --git a/app/realtime.php b/app/realtime.php index d25d952eff..fab0ce7561 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -62,7 +62,7 @@ if (!function_exists('getConsoleDB')) { global $register; /** @var Group $pools */ - $pools = $register->get('coroutinepools'); + $pools = $register->get('pools'); $adapter = new DatabasePool($pools->get('console')); $database = new Database($adapter, getCache()); @@ -92,7 +92,7 @@ if (!function_exists('getProjectDB')) { global $register; /** @var Group $pools */ - $pools = $register->get('coroutinepools'); + $pools = $register->get('pools'); if ($project->isEmpty() || $project->getId() === 'console') { return getConsoleDB(); @@ -144,7 +144,7 @@ if (!function_exists('getCache')) { global $register; - $pools = $register->get('coroutinepools'); /** @var Group $pools */ + $pools = $register->get('pools'); /** @var Group $pools */ $list = Config::getParam('pools-cache', []); $adapters = []; @@ -445,7 +445,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, } $start = time(); - $pubsub = new PubSubPool($register->get('coroutinepools')->get('pubsub')); + $pubsub = new PubSubPool($register->get('pools')->get('pubsub')); if ($pubsub->ping(true)) { $attempts = 0; @@ -519,7 +519,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, Console::info("Connection open (user: {$connection})"); - App::setResource('pools', fn () => $register->get('coroutinepools')); + App::setResource('pools', fn () => $register->get('pools')); App::setResource('request', fn () => $request); App::setResource('response', fn () => $response); diff --git a/docker-compose.yml b/docker-compose.yml index 3b935b84fb..ddfbcf421b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -298,6 +298,7 @@ services: - _APP_LOGGING_CONFIG - _APP_LOGGING_CONFIG_REALTIME - _APP_DATABASE_SHARED_TABLES + - COROUTINE_POOLS=enabled appwrite-worker-audits: entrypoint: worker-audits From a7b729b9a7d1e1ddcc379f94a7b952bf71bc5de7 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 16 Jan 2026 15:33:18 +0530 Subject: [PATCH 11/13] updated deps --- composer.json | 2 +- composer.lock | 87 +++++++++++++++++++++++---------------------------- 2 files changed, 41 insertions(+), 48 deletions(-) diff --git a/composer.json b/composer.json index c9dbd7930e..976a246a1a 100644 --- a/composer.json +++ b/composer.json @@ -67,7 +67,7 @@ "utopia-php/migration": "1.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", - "utopia-php/pools": "dev-dat-966#a70164f as 0.8.3", + "utopia-php/pools": "1.*", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.15.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index f754da4edf..55a0437fe0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "33da844fdf5648d1d1a027dfb6ae42bc", + "content-hash": "970c5cdbbd34f2be34b466ece05edbdf", "packages": [ { "name": "adhocore/jwt", @@ -1365,16 +1365,16 @@ }, { "name": "open-telemetry/exporter-otlp", - "version": "1.3.3", + "version": "1.3.4", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/exporter-otlp.git", - "reference": "07b02bc71838463f6edcc78d3485c04b48fb263d" + "reference": "62e680d587beb42e5247aa6ecd89ad1ca406e8ca" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/07b02bc71838463f6edcc78d3485c04b48fb263d", - "reference": "07b02bc71838463f6edcc78d3485c04b48fb263d", + "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/62e680d587beb42e5247aa6ecd89ad1ca406e8ca", + "reference": "62e680d587beb42e5247aa6ecd89ad1ca406e8ca", "shasum": "" }, "require": { @@ -1425,7 +1425,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-11-13T08:04:37+00:00" + "time": "2026-01-15T09:31:34+00:00" }, { "name": "open-telemetry/gen-otlp-protobuf", @@ -3657,16 +3657,16 @@ }, { "name": "utopia-php/cache", - "version": "0.13.2", + "version": "0.13.3", "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", - "reference": "5768498c9f451482f0bf3eede4d6452ddcd4a0f6" + "reference": "355707ab2c0090435059216165db86976b68a126" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cache/zipball/5768498c9f451482f0bf3eede4d6452ddcd4a0f6", - "reference": "5768498c9f451482f0bf3eede4d6452ddcd4a0f6", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/355707ab2c0090435059216165db86976b68a126", + "reference": "355707ab2c0090435059216165db86976b68a126", "shasum": "" }, "require": { @@ -3674,7 +3674,7 @@ "ext-memcached": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/pools": "0.8.*", + "utopia-php/pools": "1.*", "utopia-php/telemetry": "*" }, "require-dev": { @@ -3703,9 +3703,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/0.13.2" + "source": "https://github.com/utopia-php/cache/tree/0.13.3" }, - "time": "2025-12-17T08:55:43+00:00" + "time": "2026-01-16T07:54:34+00:00" }, { "name": "utopia-php/cli", @@ -3899,16 +3899,16 @@ }, { "name": "utopia-php/database", - "version": "4.5.2", + "version": "4.5.3", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "8e6a033d4da09a2f2ac1f79fd85fcfa2da018d23" + "reference": "78f7c97e12872b206c4ee6bc8cdc342654b7568c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/8e6a033d4da09a2f2ac1f79fd85fcfa2da018d23", - "reference": "8e6a033d4da09a2f2ac1f79fd85fcfa2da018d23", + "url": "https://api.github.com/repos/utopia-php/database/zipball/78f7c97e12872b206c4ee6bc8cdc342654b7568c", + "reference": "78f7c97e12872b206c4ee6bc8cdc342654b7568c", "shasum": "" }, "require": { @@ -3919,7 +3919,7 @@ "utopia-php/cache": "0.13.*", "utopia-php/framework": "0.33.*", "utopia-php/mongo": "0.11.*", - "utopia-php/pools": "0.8.*" + "utopia-php/pools": "1.*" }, "require-dev": { "fakerphp/faker": "1.23.*", @@ -3951,9 +3951,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/4.5.2" + "source": "https://github.com/utopia-php/database/tree/4.5.3" }, - "time": "2026-01-15T04:23:30+00:00" + "time": "2026-01-16T08:45:47+00:00" }, { "name": "utopia-php/detector", @@ -4516,16 +4516,16 @@ }, { "name": "utopia-php/migration", - "version": "1.4.3", + "version": "1.4.4", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "52ca4234d8229b68e27e052248734a08784d9d3d" + "reference": "3fe751902012d09d323420cd3523be1ed855e868" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/52ca4234d8229b68e27e052248734a08784d9d3d", - "reference": "52ca4234d8229b68e27e052248734a08784d9d3d", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/3fe751902012d09d323420cd3523be1ed855e868", + "reference": "3fe751902012d09d323420cd3523be1ed855e868", "shasum": "" }, "require": { @@ -4565,9 +4565,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/1.4.3" + "source": "https://github.com/utopia-php/migration/tree/1.4.4" }, - "time": "2026-01-13T09:51:08+00:00" + "time": "2026-01-16T10:00:07+00:00" }, { "name": "utopia-php/mongo", @@ -4733,16 +4733,16 @@ }, { "name": "utopia-php/pools", - "version": "0.8.3", + "version": "1.0.0", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "ad7d6ba946376e81c603204285ce9a674b6502b8" + "reference": "74ba7dc985c2f629df8cf08ed95507955e3bcf86" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/ad7d6ba946376e81c603204285ce9a674b6502b8", - "reference": "ad7d6ba946376e81c603204285ce9a674b6502b8", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/74ba7dc985c2f629df8cf08ed95507955e3bcf86", + "reference": "74ba7dc985c2f629df8cf08ed95507955e3bcf86", "shasum": "" }, "require": { @@ -4780,9 +4780,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.8.3" + "source": "https://github.com/utopia-php/pools/tree/1.0.0" }, - "time": "2025-12-17T09:35:18+00:00" + "time": "2026-01-15T12:34:17+00:00" }, { "name": "utopia-php/preloader", @@ -4839,16 +4839,16 @@ }, { "name": "utopia-php/queue", - "version": "0.15.0", + "version": "0.15.1", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "6abb268ba7ec00dea4e5201b007776ea1bce9242" + "reference": "e551606385990ec7901d222017c4cfc2749a518c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/6abb268ba7ec00dea4e5201b007776ea1bce9242", - "reference": "6abb268ba7ec00dea4e5201b007776ea1bce9242", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/e551606385990ec7901d222017c4cfc2749a518c", + "reference": "e551606385990ec7901d222017c4cfc2749a518c", "shasum": "" }, "require": { @@ -4857,7 +4857,7 @@ "utopia-php/console": "0.0.*", "utopia-php/fetch": "0.5.*", "utopia-php/framework": "0.33.*", - "utopia-php/pools": "0.8.*", + "utopia-php/pools": "1.*", "utopia-php/telemetry": "*" }, "require-dev": { @@ -4899,9 +4899,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.15.0" + "source": "https://github.com/utopia-php/queue/tree/0.15.1" }, - "time": "2026-01-06T12:41:51+00:00" + "time": "2026-01-16T07:54:54+00:00" }, { "name": "utopia-php/registry", @@ -8987,14 +8987,7 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/pools", - "version": "dev-dat-966", - "alias": "0.8.3", - "alias_normalized": "0.8.3.0" - } - ], + "aliases": [], "minimum-stability": "stable", "stability-flags": {}, "prefer-stable": false, From fc04e17a69a1b663edbf3a4b82cf0c47134f27f0 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 16 Jan 2026 17:37:56 +0530 Subject: [PATCH 12/13] updated env --- .env | 2 +- app/init/registers.php | 2 +- docker-compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 51960bbaf1..a5f1c0a752 100644 --- a/.env +++ b/.env @@ -127,4 +127,4 @@ _APP_PROJECT_REGIONS=default _APP_FUNCTIONS_CREATION_ABUSE_LIMIT=5000 _APP_STATS_USAGE_DUAL_WRITING_DBS=database_db_main _APP_TRUSTED_HEADERS=x-forwarded-for -COROUTINE_POOLS=disabled \ No newline at end of file +_APP_POOL_ADAPTER=stack \ No newline at end of file diff --git a/app/init/registers.php b/app/init/registers.php index d15cce4d6b..8c596aae8e 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -287,7 +287,7 @@ $register->set('pools', function () { default => throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Invalid scheme'), }; - $poolAdapter = System::getEnv('COROUTINE_POOLS', 'disabled') === 'enabled' ? new SwoolePool() : new StackPool(); + $poolAdapter = System::getEnv('_APP_POOL_ADAPTER', default: 'stack') === 'swoole' ? new SwoolePool() : new StackPool(); $pool = new Pool($poolAdapter, $name, $poolSize, function () use ($type, $resource, $dsn) { // Get Adapter diff --git a/docker-compose.yml b/docker-compose.yml index cb3d7a6281..c2891d844e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -298,7 +298,7 @@ services: - _APP_LOGGING_CONFIG - _APP_LOGGING_CONFIG_REALTIME - _APP_DATABASE_SHARED_TABLES - - COROUTINE_POOLS=enabled + - _APP_POOL_ADAPTER=swoole appwrite-worker-audits: entrypoint: worker-audits From d095f25a946b5865a084a20f386dcf7ed75ea11c Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 19 Jan 2026 11:04:24 +0530 Subject: [PATCH 13/13] updated composer --- composer.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/composer.lock b/composer.lock index 55a0437fe0..e1712a4df4 100644 --- a/composer.lock +++ b/composer.lock @@ -1492,16 +1492,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.10.0", + "version": "1.11.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "3dfc3d1ad729ec7eb25f1b9a4ae39fe779affa99" + "reference": "d91f21addcdb42da9a451c002777f8318432461a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/3dfc3d1ad729ec7eb25f1b9a4ae39fe779affa99", - "reference": "3dfc3d1ad729ec7eb25f1b9a4ae39fe779affa99", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/d91f21addcdb42da9a451c002777f8318432461a", + "reference": "d91f21addcdb42da9a451c002777f8318432461a", "shasum": "" }, "require": { @@ -1585,7 +1585,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-11-25T10:59:15+00:00" + "time": "2026-01-15T11:21:03+00:00" }, { "name": "open-telemetry/sem-conv", @@ -3899,16 +3899,16 @@ }, { "name": "utopia-php/database", - "version": "4.5.3", + "version": "4.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "78f7c97e12872b206c4ee6bc8cdc342654b7568c" + "reference": "b5c16caf4f6b12fa2c04d5a48f6e5785c99da8df" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/78f7c97e12872b206c4ee6bc8cdc342654b7568c", - "reference": "78f7c97e12872b206c4ee6bc8cdc342654b7568c", + "url": "https://api.github.com/repos/utopia-php/database/zipball/b5c16caf4f6b12fa2c04d5a48f6e5785c99da8df", + "reference": "b5c16caf4f6b12fa2c04d5a48f6e5785c99da8df", "shasum": "" }, "require": { @@ -3951,9 +3951,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/4.5.3" + "source": "https://github.com/utopia-php/database/tree/4.6.0" }, - "time": "2026-01-16T08:45:47+00:00" + "time": "2026-01-16T12:35:16+00:00" }, { "name": "utopia-php/detector", @@ -9013,5 +9013,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.9.0" }