From 2191a332ef5cfe93c82e0921a31f211c04cfb83b Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 21:23:31 +0100 Subject: [PATCH 1/8] Add coroutine concurrency to queue workers Upgrade utopia-php/queue to 0.19.0 and utopia-php/platform to 1.0.0-rc4, enabling workers to consume multiple jobs concurrently via Swoole coroutines within a single worker process. - Workers pass a maxCoroutines option to the queue adapter, configured via the new _APP_WORKER_MAX_COROUTINES env var (default 1) - Per-message DI resources now live in a coroutine-scoped context container (Server::context()) instead of the shared container - Consumer Redis connections are wrapped in the Locking decorator so concurrent coroutines can safely share one connection - Add a functions worker load test: 30 async 1s executions drain in 31.1s at 1 coroutine, 10.4s at 5, 7.3s at 10 Co-Authored-By: Claude Opus 4.8 (1M context) --- .env | 2 + app/init/registers.php | 8 +- app/worker.php | 10 +- composer.json | 4 +- composer.lock | 28 ++-- docker-compose.yml | 2 + .../Services/Functions/FunctionsLoadTest.php | 126 ++++++++++++++++++ tests/resources/functions/sleep/index.js | 4 + 8 files changed, 163 insertions(+), 21 deletions(-) create mode 100644 tests/e2e/Services/Functions/FunctionsLoadTest.php create mode 100644 tests/resources/functions/sleep/index.js diff --git a/.env b/.env index aa7438f6bb..61de326ee5 100644 --- a/.env +++ b/.env @@ -150,3 +150,5 @@ _APP_POOL_ADAPTER=stack _APP_WORKER_SCREENSHOTS_ROUTER=http://appwrite _TESTS_OAUTH2_GITHUB_CLIENT_ID= _TESTS_OAUTH2_GITHUB_CLIENT_SECRET= + +_APP_WORKER_MAX_COROUTINES=3 diff --git a/app/init/registers.php b/app/init/registers.php index 9ea19eee24..881aadd249 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -364,11 +364,17 @@ $register->set('pools', function () { default => null }; case 'publisher': - case 'consumer': return match ($dsn->getScheme()) { 'redis' => new Queue\Broker\Redis(new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort())), default => null }; + case 'consumer': + // Consumers are shared by multiple coroutines within a worker + // process, so serialize commands on the single connection. + return match ($dsn->getScheme()) { + 'redis' => new Queue\Broker\Redis(new Queue\Connection\Locking(new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()))), + default => null + }; case 'cache': $adapter = match ($dsn->getScheme()) { 'redis' => new RedisCache($resource()), diff --git a/app/worker.php b/app/worker.php index 169b8e9770..e65655c2de 100644 --- a/app/worker.php +++ b/app/worker.php @@ -86,14 +86,16 @@ $pools = $container->get('pools'); $adapter = new Swoole( $pools->get('consumer')->pop()->getResource(), System::getEnv('_APP_WORKERS_NUM', 1), - $queueName + $queueName, + maxCoroutines: (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1), + resources: $container, ); -$worker = new Server($adapter, $container); +$worker = new Server($adapter); try { $worker->init()->action(function () use ($worker, $registerWorkerMessageResources, $queueName) { - $registerWorkerMessageResources($worker->getContainer()); + $registerWorkerMessageResources($worker->context()); Span::init("worker.{$queueName}"); }); @@ -103,7 +105,7 @@ try { $container->set('bus', function ($register) use ($worker) { return $register->get('bus')->setResolver( - fn (string $name) => $worker->getContainer()->get($name) + fn (string $name) => $worker->context()->get($name) ); }, ['register']); diff --git a/composer.json b/composer.json index 1e3ce344b8..b78637ccd9 100644 --- a/composer.json +++ b/composer.json @@ -76,11 +76,11 @@ "utopia-php/logger": "0.8.*", "utopia-php/messaging": "1.0.*", "utopia-php/migration": "1.*", - "utopia-php/platform": "1.0.0-rc3", + "utopia-php/platform": "1.0.0-rc4", "utopia-php/pools": "1.*", "utopia-php/span": "1.1.*", "utopia-php/preloader": "0.2.*", - "utopia-php/queue": "0.18.*", + "utopia-php/queue": "0.19.*", "utopia-php/servers": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/storage": "2.*", diff --git a/composer.lock b/composer.lock index 11711529ba..b456b34ec7 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "551924e300a646e6f433ea0a0f3d6c9f", + "content-hash": "0c23c30bba86ad84377d4d1abce0992a", "packages": [ { "name": "adhocore/jwt", @@ -4773,16 +4773,16 @@ }, { "name": "utopia-php/platform", - "version": "1.0.0-rc3", + "version": "1.0.0-rc4", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "674fd3cea6d1f8db660b01c5be89e8333dfb6de9" + "reference": "455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/platform/zipball/674fd3cea6d1f8db660b01c5be89e8333dfb6de9", - "reference": "674fd3cea6d1f8db660b01c5be89e8333dfb6de9", + "url": "https://api.github.com/repos/utopia-php/platform/zipball/455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0", + "reference": "455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0", "shasum": "" }, "require": { @@ -4791,7 +4791,7 @@ "php": ">=8.3", "utopia-php/cli": "0.23.*", "utopia-php/http": "^2.0@RC", - "utopia-php/queue": "0.18.*", + "utopia-php/queue": "^0.19", "utopia-php/servers": "0.4.*" }, "require-dev": { @@ -4818,9 +4818,9 @@ ], "support": { "issues": "https://github.com/utopia-php/platform/issues", - "source": "https://github.com/utopia-php/platform/tree/1.0.0-rc3" + "source": "https://github.com/utopia-php/platform/tree/1.0.0-rc4" }, - "time": "2026-05-25T11:48:18+00:00" + "time": "2026-06-02T20:19:26+00:00" }, { "name": "utopia-php/pools", @@ -4976,16 +4976,16 @@ }, { "name": "utopia-php/queue", - "version": "0.18.6", + "version": "0.19.0", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "954cd3788b37697b83fd2757adcd93422f21a846" + "reference": "cd4248af69006b2568c3486a675f27cb21191859" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/954cd3788b37697b83fd2757adcd93422f21a846", - "reference": "954cd3788b37697b83fd2757adcd93422f21a846", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/cd4248af69006b2568c3486a675f27cb21191859", + "reference": "cd4248af69006b2568c3486a675f27cb21191859", "shasum": "" }, "require": { @@ -5037,9 +5037,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.18.6" + "source": "https://github.com/utopia-php/queue/tree/0.19.0" }, - "time": "2026-06-01T19:47:57+00:00" + "time": "2026-06-02T20:12:18+00:00" }, { "name": "utopia-php/registry", diff --git a/docker-compose.yml b/docker-compose.yml index ab51ad32c0..dca86166e0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -350,6 +350,7 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_WORKER_MAX_COROUTINES - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 - _APP_REDIS_HOST @@ -737,6 +738,7 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_WORKER_MAX_COROUTINES - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN diff --git a/tests/e2e/Services/Functions/FunctionsLoadTest.php b/tests/e2e/Services/Functions/FunctionsLoadTest.php new file mode 100644 index 0000000000..81d861fc6b --- /dev/null +++ b/tests/e2e/Services/Functions/FunctionsLoadTest.php @@ -0,0 +1,126 @@ +setupFunction([ + 'functionId' => ID::unique(), + 'name' => 'Load Test Sleep', + 'runtime' => 'node-22', + 'entrypoint' => 'index.js', + // Must exceed worst-case queue drain time: executions waiting in the + // queue longer than the function timeout are reported as failed. + 'timeout' => 300, + ]); + + $this->setupDeployment($functionId, [ + 'code' => $this->packageFunction('sleep'), + 'activate' => true, + ]); + + // Warm up the runtime so cold start does not skew the measurement. + $execution = $this->createExecution($functionId, ['async' => false]); + $this->assertEquals(201, $execution['headers']['status-code']); + $this->assertEquals('completed', $execution['body']['status']); + + // Fire N async executions. + $start = \microtime(true); + + for ($i = 0; $i < self::EXECUTIONS; $i++) { + $execution = $this->createExecution($functionId, ['async' => true]); + $this->assertEquals(202, $execution['headers']['status-code']); + } + + $enqueued = \microtime(true) - $start; + + // Poll until all executions reach a terminal state. + // The warmup execution is already completed, so expect N + 1. + $expected = self::EXECUTIONS + 1; + $completed = 0; + $failed = 0; + $drained = null; + + while (\microtime(true) - $start < self::DRAIN_TIMEOUT_SECONDS) { + $completed = $this->countExecutions($functionId, 'completed'); + $failed = $this->countExecutions($functionId, 'failed'); + + if ($completed + $failed >= $expected) { + $drained = \microtime(true) - $start; + break; + } + + \usleep(250_000); + } + + $this->assertNotNull($drained, "Queue did not drain in " . self::DRAIN_TIMEOUT_SECONDS . "s ({$completed} completed, {$failed} failed)"); + + if ($failed > 0) { + $samples = $this->listExecutions($functionId, [ + 'queries' => [ + Query::equal('status', ['failed'])->toString(), + Query::limit(3)->toString(), + ], + ]); + + $details = \array_map(fn ($e) => [ + 'responseStatusCode' => $e['responseStatusCode'] ?? null, + 'errors' => $e['errors'] ?? null, + 'logs' => $e['logs'] ?? null, + 'duration' => $e['duration'] ?? null, + ], $samples['body']['executions'] ?? []); + + $this->fail("{$failed} executions failed. Samples: " . \json_encode($details, JSON_PRETTY_PRINT)); + } + $this->assertEquals($expected, $completed); + + $throughput = self::EXECUTIONS / $drained; + + \fwrite(STDERR, \sprintf( + "\n[load] executions=%d enqueue=%.2fs drain=%.2fs throughput=%.2f exec/s\n", + self::EXECUTIONS, + $enqueued, + $drained, + $throughput, + )); + + $this->cleanupFunction($functionId); + } + + private function countExecutions(string $functionId, string $status): int + { + $executions = $this->listExecutions($functionId, [ + 'queries' => [ + Query::equal('status', [$status])->toString(), + Query::limit(1)->toString(), + ], + ]); + + $this->assertEquals(200, $executions['headers']['status-code']); + + return $executions['body']['total']; + } +} diff --git a/tests/resources/functions/sleep/index.js b/tests/resources/functions/sleep/index.js new file mode 100644 index 0000000000..416bb512ec --- /dev/null +++ b/tests/resources/functions/sleep/index.js @@ -0,0 +1,4 @@ +module.exports = async (context) => { + await new Promise((resolve) => setTimeout(resolve, 1000)); + return context.res.send('OK'); +}; From 2e49713194fbf53b48be29ac475c360ee795d814 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 21:31:17 +0100 Subject: [PATCH 2/8] Set explicit per-worker process and coroutine counts in compose Workers run a single process (_APP_WORKERS_NUM=1) with concurrency provided by coroutines instead. I/O-bound workers get 8 coroutines; CPU-heavy, stateful, or ordering-sensitive workers (databases, builds, migrations, stats-resources, stats-usage) stay serial at 1. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env | 3 ++- docker-compose.yml | 31 ++++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 61de326ee5..57deb8cc0a 100644 --- a/.env +++ b/.env @@ -151,4 +151,5 @@ _APP_WORKER_SCREENSHOTS_ROUTER=http://appwrite _TESTS_OAUTH2_GITHUB_CLIENT_ID= _TESTS_OAUTH2_GITHUB_CLIENT_SECRET= -_APP_WORKER_MAX_COROUTINES=3 +_APP_WORKERS_NUM=1 +_APP_WORKER_MAX_COROUTINES=1 diff --git a/docker-compose.yml b/docker-compose.yml index dca86166e0..981835b866 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -349,8 +349,9 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - - _APP_WORKER_MAX_COROUTINES - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 - _APP_REDIS_HOST @@ -384,6 +385,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -425,6 +428,8 @@ services: environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -485,6 +490,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -505,7 +512,6 @@ services: - _APP_DB_USER_VECTORSDB - _APP_DB_PASS_VECTORSDB - _APP_LOGGING_CONFIG - - _APP_WORKERS_NUM - _APP_QUEUE_NAME - _APP_DATABASE_SHARED_TABLES @@ -528,6 +534,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -606,6 +614,8 @@ services: - _APP_OPTIONS_FORCE_HTTPS # Basic - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_LOGGING_CONFIG @@ -666,6 +676,8 @@ services: - ./src:/usr/src/code/src environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -706,6 +718,8 @@ services: - ${_APP_DB_HOST:-mongodb} environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -737,8 +751,9 @@ services: - redis environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - - _APP_WORKER_MAX_COROUTINES - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN @@ -784,6 +799,8 @@ services: # - smtp environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -825,6 +842,8 @@ services: - redis environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -885,6 +904,8 @@ services: - ${_APP_DB_HOST:-mongodb} environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1046,6 +1067,8 @@ services: - ${_APP_DB_HOST:-mongodb} environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1080,6 +1103,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 From e3afc66bac38fa64ceeb40ac2f07e19bd0ee7fc8 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 21:33:16 +0100 Subject: [PATCH 3/8] Run all workers at 8 coroutines except databases Only the databases worker is ordering-sensitive (schema operations must apply in sequence); every other worker handles independent jobs. Co-Authored-By: Claude Opus 4.8 (1M context) --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 981835b866..5bada8bb6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -535,7 +535,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -905,7 +905,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1068,7 +1068,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1104,7 +1104,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 From ead860457ca6eb47975ed3af0df6097ee16c85f2 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 21:35:08 +0100 Subject: [PATCH 4/8] Keep stats workers serial: shared accumulator state is not coroutine-safe StatsUsage batches metrics across messages on shared instance state ($stats/$keys/$projects/$statDocuments); concurrent consume coroutines race the flush in commitToDB, losing aggregates on reset and double-applying upsertDocumentsWithIncrease on overlapping flushes. StatsResources resets its shared $documents buffer at the start of each job, so a second coroutine starting mid-count wipes the first one's batch and the projects' metrics interleave into one write. Both stay at 1 coroutine until accumulation moves to per-message state. Co-Authored-By: Claude Opus 4.8 (1M context) --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5bada8bb6b..2a175751a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1068,7 +1068,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=8 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1104,7 +1104,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=8 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 From 7889a1448f27858830a4d513d263a4378782c6c3 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 21:38:26 +0100 Subject: [PATCH 5/8] Serialize workers with shared accumulator state; fix Messaging local device cache Coroutine-safety audit of all worker actions (instance state mutated during action() is shared across consume coroutines): - audits: batches logs across messages ($logs/$lastTriggeredTime) with Commit/NoCommit ack semantics tied to the shared batch; concurrent flushes lose or duplicate audit entries - webhooks: $errors buffer is reset per job and drives the retry/disable path; concurrent jobs wipe or steal each other's errors - migrations: project DB callables are rebound on the instance per message; a concurrent message swaps database handles mid-migration Messaging cached a project-scoped Local device unkeyed, reusing the first project's storage path for every subsequent message; create it per call instead. Deletes, mails, certificates, executions, functions, builds and screenshots keep all per-message state in locals and stay at 8. Co-Authored-By: Claude Opus 4.8 (1M context) --- docker-compose.yml | 6 +++--- src/Appwrite/Platform/Workers/Messaging.php | 10 +++------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2a175751a4..a065408197 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -350,7 +350,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=8 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -386,7 +386,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=8 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -905,7 +905,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=8 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 759d78f81e..82548b83a7 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -53,8 +53,6 @@ use function Swoole\Coroutine\batch; class Messaging extends Action { - private ?Local $localDevice = null; - private ?SMSAdapter $adapter = null; public static function getName(): string @@ -716,11 +714,9 @@ class Messaging extends Action private function getLocalDevice($project): Local { - if ($this->localDevice === null) { - $this->localDevice = new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); - } - - return $this->localDevice; + // Not cached: the path is project-scoped and the worker handles + // messages from many projects (and coroutines run them concurrently). + return new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); } private function createInternalSMSAdapter(): ?SMSAdapter From b53268a4ddaf2c237d69e3e36aebab0fae24303f Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 21:40:48 +0100 Subject: [PATCH 6/8] Remove functions load test harness The throughput numbers live in the PR; the harness itself is not CI material. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Services/Functions/FunctionsLoadTest.php | 126 ------------------ tests/resources/functions/sleep/index.js | 4 - 2 files changed, 130 deletions(-) delete mode 100644 tests/e2e/Services/Functions/FunctionsLoadTest.php delete mode 100644 tests/resources/functions/sleep/index.js diff --git a/tests/e2e/Services/Functions/FunctionsLoadTest.php b/tests/e2e/Services/Functions/FunctionsLoadTest.php deleted file mode 100644 index 81d861fc6b..0000000000 --- a/tests/e2e/Services/Functions/FunctionsLoadTest.php +++ /dev/null @@ -1,126 +0,0 @@ -setupFunction([ - 'functionId' => ID::unique(), - 'name' => 'Load Test Sleep', - 'runtime' => 'node-22', - 'entrypoint' => 'index.js', - // Must exceed worst-case queue drain time: executions waiting in the - // queue longer than the function timeout are reported as failed. - 'timeout' => 300, - ]); - - $this->setupDeployment($functionId, [ - 'code' => $this->packageFunction('sleep'), - 'activate' => true, - ]); - - // Warm up the runtime so cold start does not skew the measurement. - $execution = $this->createExecution($functionId, ['async' => false]); - $this->assertEquals(201, $execution['headers']['status-code']); - $this->assertEquals('completed', $execution['body']['status']); - - // Fire N async executions. - $start = \microtime(true); - - for ($i = 0; $i < self::EXECUTIONS; $i++) { - $execution = $this->createExecution($functionId, ['async' => true]); - $this->assertEquals(202, $execution['headers']['status-code']); - } - - $enqueued = \microtime(true) - $start; - - // Poll until all executions reach a terminal state. - // The warmup execution is already completed, so expect N + 1. - $expected = self::EXECUTIONS + 1; - $completed = 0; - $failed = 0; - $drained = null; - - while (\microtime(true) - $start < self::DRAIN_TIMEOUT_SECONDS) { - $completed = $this->countExecutions($functionId, 'completed'); - $failed = $this->countExecutions($functionId, 'failed'); - - if ($completed + $failed >= $expected) { - $drained = \microtime(true) - $start; - break; - } - - \usleep(250_000); - } - - $this->assertNotNull($drained, "Queue did not drain in " . self::DRAIN_TIMEOUT_SECONDS . "s ({$completed} completed, {$failed} failed)"); - - if ($failed > 0) { - $samples = $this->listExecutions($functionId, [ - 'queries' => [ - Query::equal('status', ['failed'])->toString(), - Query::limit(3)->toString(), - ], - ]); - - $details = \array_map(fn ($e) => [ - 'responseStatusCode' => $e['responseStatusCode'] ?? null, - 'errors' => $e['errors'] ?? null, - 'logs' => $e['logs'] ?? null, - 'duration' => $e['duration'] ?? null, - ], $samples['body']['executions'] ?? []); - - $this->fail("{$failed} executions failed. Samples: " . \json_encode($details, JSON_PRETTY_PRINT)); - } - $this->assertEquals($expected, $completed); - - $throughput = self::EXECUTIONS / $drained; - - \fwrite(STDERR, \sprintf( - "\n[load] executions=%d enqueue=%.2fs drain=%.2fs throughput=%.2f exec/s\n", - self::EXECUTIONS, - $enqueued, - $drained, - $throughput, - )); - - $this->cleanupFunction($functionId); - } - - private function countExecutions(string $functionId, string $status): int - { - $executions = $this->listExecutions($functionId, [ - 'queries' => [ - Query::equal('status', [$status])->toString(), - Query::limit(1)->toString(), - ], - ]); - - $this->assertEquals(200, $executions['headers']['status-code']); - - return $executions['body']['total']; - } -} diff --git a/tests/resources/functions/sleep/index.js b/tests/resources/functions/sleep/index.js deleted file mode 100644 index 416bb512ec..0000000000 --- a/tests/resources/functions/sleep/index.js +++ /dev/null @@ -1,4 +0,0 @@ -module.exports = async (context) => { - await new Promise((resolve) => setTimeout(resolve, 1000)); - return context.res.send('OK'); -}; From 4baaa7d93fd544b8fc82faa08372c8f745c366da Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 22:03:10 +0100 Subject: [PATCH 7/8] Fix coroutine starvation: scale pools to coroutine count, serialize SMTP workers Two CI failure causes: - Connection pools are sized for HTTP-server worker counts and came out at 1 per worker process; 8 consume coroutines exhausted them (Pool 'cache_redis_main' is empty). Floor the pool size at _APP_WORKER_MAX_COROUTINES. - The mails and messaging workers disable the Swoole TCP hook at runtime to work around the TLS handshake crash (swoole-src#4909, #7914). The flag is process-global, so after the first job every blocking Redis pop freezes the coroutine scheduler and deliveries arrive minutes late. Both workers stay at 1 coroutine, matching pre-coroutine behavior. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/init/registers.php | 4 ++++ docker-compose.yml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/init/registers.php b/app/init/registers.php index 881aadd249..2f5f39ceb0 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -254,6 +254,10 @@ $register->set('pools', function () { $workerCount = intval(System::getEnv('_APP_CPU_NUM', swoole_cpu_num())) * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); $poolSize = max(1, (int)($instanceConnections / $workerCount)); + // Queue workers consume jobs concurrently with coroutines; each in-flight + // job may hold a connection, so pools must cover the coroutine count. + $poolSize = max($poolSize, (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1)); + foreach ($connections as $key => $connection) { $type = $connection['type']; $multiple = $connection['multiple']; diff --git a/docker-compose.yml b/docker-compose.yml index a065408197..90bea1dc5d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -800,7 +800,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=8 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -843,7 +843,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=8 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 From 78769b7f13e6286b7d8cc04dfbab2e2ad0e2b840 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 22:37:37 +0100 Subject: [PATCH 8/8] Lease one broker connection per consume coroutine instead of sharing one The Locking decorator serialized every Redis command on the single consumer connection shared by all consume coroutines. On an idle queue each waiting coroutine holds the mutex for a full 2s blocking pop, so a finished job's completion ops (remove job key, increment stats) starve behind the poppers: a 10ms cache purge took 22-24s to ack, and Storage's testFilePreviewCache observed stale previews because purges landed tens of seconds after enqueue. Pass a Broker\Pool as the adapter consumer instead: each consume coroutine leases its own connection from the consumer pool for the lifetime of its loop, eliminating the shared-connection mutex entirely. The Locking wrapper in the pool factory is reverted, and consumer pools get +2 headroom above the coroutine count for queue-size telemetry reads. Purge ack latency drops to 4-58ms. Known tradeoff: Broker\Pool::close() is a no-op, so SIGTERM no longer breaks the blocking pop early; workers rely on the process kill that already follows. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/init/registers.php | 18 ++++++++---------- app/worker.php | 8 +++++++- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/app/init/registers.php b/app/init/registers.php index 2f5f39ceb0..50080f0538 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -344,9 +344,13 @@ $register->set('pools', function () { // PubSub workers hold one long-lived subscribed connection and also need // spare capacity for publishes from the same process. - $connectionPoolSize = $type === 'pubsub' - ? max(2, $poolSize) - : $poolSize; + // Consumer pools lease one connection per consume coroutine for the + // lifetime of the worker, plus headroom for queue-size reads. + $connectionPoolSize = match ($type) { + 'pubsub' => max(2, $poolSize), + 'consumer' => $poolSize + 2, + default => $poolSize, + }; $pool = new Pool($poolAdapter, $name, $connectionPoolSize, function () use ($type, $resource, $dsn) { // Get Adapter @@ -368,17 +372,11 @@ $register->set('pools', function () { default => null }; case 'publisher': + case 'consumer': return match ($dsn->getScheme()) { 'redis' => new Queue\Broker\Redis(new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort())), default => null }; - case 'consumer': - // Consumers are shared by multiple coroutines within a worker - // process, so serialize commands on the single connection. - return match ($dsn->getScheme()) { - 'redis' => new Queue\Broker\Redis(new Queue\Connection\Locking(new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()))), - default => null - }; case 'cache': $adapter = match ($dsn->getScheme()) { 'redis' => new RedisCache($resource()), diff --git a/app/worker.php b/app/worker.php index e65655c2de..063d4746ac 100644 --- a/app/worker.php +++ b/app/worker.php @@ -83,8 +83,14 @@ if (\str_starts_with($workerName, 'databases')) { /** @var \Utopia\Pools\Group $pools */ $pools = $container->get('pools'); +// Each consume coroutine leases its own broker connection from the pool for +// the lifetime of its consume loop, so concurrent coroutines never share a +// connection. The publisher side backs the queue-depth telemetry gauge. $adapter = new Swoole( - $pools->get('consumer')->pop()->getResource(), + new BrokerPool( + publisher: $pools->get('consumer'), + consumer: $pools->get('consumer'), + ), System::getEnv('_APP_WORKERS_NUM', 1), $queueName, maxCoroutines: (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1),