From 78769b7f13e6286b7d8cc04dfbab2e2ad0e2b840 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 22:37:37 +0100 Subject: [PATCH] Lease one broker connection per consume coroutine instead of sharing one The Locking decorator serialized every Redis command on the single consumer connection shared by all consume coroutines. On an idle queue each waiting coroutine holds the mutex for a full 2s blocking pop, so a finished job's completion ops (remove job key, increment stats) starve behind the poppers: a 10ms cache purge took 22-24s to ack, and Storage's testFilePreviewCache observed stale previews because purges landed tens of seconds after enqueue. Pass a Broker\Pool as the adapter consumer instead: each consume coroutine leases its own connection from the consumer pool for the lifetime of its loop, eliminating the shared-connection mutex entirely. The Locking wrapper in the pool factory is reverted, and consumer pools get +2 headroom above the coroutine count for queue-size telemetry reads. Purge ack latency drops to 4-58ms. Known tradeoff: Broker\Pool::close() is a no-op, so SIGTERM no longer breaks the blocking pop early; workers rely on the process kill that already follows. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/init/registers.php | 18 ++++++++---------- app/worker.php | 8 +++++++- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/app/init/registers.php b/app/init/registers.php index 2f5f39ceb0..50080f0538 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -344,9 +344,13 @@ $register->set('pools', function () { // PubSub workers hold one long-lived subscribed connection and also need // spare capacity for publishes from the same process. - $connectionPoolSize = $type === 'pubsub' - ? max(2, $poolSize) - : $poolSize; + // Consumer pools lease one connection per consume coroutine for the + // lifetime of the worker, plus headroom for queue-size reads. + $connectionPoolSize = match ($type) { + 'pubsub' => max(2, $poolSize), + 'consumer' => $poolSize + 2, + default => $poolSize, + }; $pool = new Pool($poolAdapter, $name, $connectionPoolSize, function () use ($type, $resource, $dsn) { // Get Adapter @@ -368,17 +372,11 @@ $register->set('pools', function () { default => null }; case 'publisher': + case 'consumer': return match ($dsn->getScheme()) { 'redis' => new Queue\Broker\Redis(new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort())), default => null }; - case 'consumer': - // Consumers are shared by multiple coroutines within a worker - // process, so serialize commands on the single connection. - return match ($dsn->getScheme()) { - 'redis' => new Queue\Broker\Redis(new Queue\Connection\Locking(new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()))), - default => null - }; case 'cache': $adapter = match ($dsn->getScheme()) { 'redis' => new RedisCache($resource()), diff --git a/app/worker.php b/app/worker.php index e65655c2de..063d4746ac 100644 --- a/app/worker.php +++ b/app/worker.php @@ -83,8 +83,14 @@ if (\str_starts_with($workerName, 'databases')) { /** @var \Utopia\Pools\Group $pools */ $pools = $container->get('pools'); +// Each consume coroutine leases its own broker connection from the pool for +// the lifetime of its consume loop, so concurrent coroutines never share a +// connection. The publisher side backs the queue-depth telemetry gauge. $adapter = new Swoole( - $pools->get('consumer')->pop()->getResource(), + new BrokerPool( + publisher: $pools->get('consumer'), + consumer: $pools->get('consumer'), + ), System::getEnv('_APP_WORKERS_NUM', 1), $queueName, maxCoroutines: (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1),