Merge pull request #12468 from appwrite/feat/coroutine-workers

Add coroutine concurrency to queue workers
This commit is contained in:
Luke B. Silver authored and GitHub committed 2026-06-02 22:58:49 +01:00
commit 61e189fb4e
7 files changed
+74 -32

No files matched your search

+3
View File
@@ -150,3 +150,6 @@ _APP_POOL_ADAPTER=stack
_APP_WORKER_SCREENSHOTS_ROUTER=http://appwrite
_TESTS_OAUTH2_GITHUB_CLIENT_ID=
_TESTS_OAUTH2_GITHUB_CLIENT_SECRET=
_APP_WORKERS_NUM=1
_APP_WORKER_MAX_COROUTINES=1
+11 -3
View File
@@ -254,6 +254,10 @@ $register->set('pools', function () {
$workerCount = intval(System::getEnv('_APP_CPU_NUM', swoole_cpu_num())) * intval(System::getEnv('_APP_WORKER_PER_CORE', 6));
$poolSize = max(1, (int)($instanceConnections / $workerCount));
// Queue workers consume jobs concurrently with coroutines; each in-flight
// job may hold a connection, so pools must cover the coroutine count.
$poolSize = max($poolSize, (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1));
foreach ($connections as $key => $connection) {
$type = $connection['type'];
$multiple = $connection['multiple'];
@@ -340,9 +344,13 @@ $register->set('pools', function () {
// PubSub workers hold one long-lived subscribed connection and also need
// spare capacity for publishes from the same process.
$connectionPoolSize = $type === 'pubsub'
? max(2, $poolSize)
: $poolSize;
// Consumer pools lease one connection per consume coroutine for the
// lifetime of the worker, plus headroom for queue-size reads.
$connectionPoolSize = match ($type) {
'pubsub' => max(2, $poolSize),
'consumer' => $poolSize + 2,
default => $poolSize,
};
$pool = new Pool($poolAdapter, $name, $connectionPoolSize, function () use ($type, $resource, $dsn) {
// Get Adapter
+13 -5
View File
@@ -83,17 +83,25 @@ if (\str_starts_with($workerName, 'databases')) {
/** @var \Utopia\Pools\Group $pools */
$pools = $container->get('pools');
// Each consume coroutine leases its own broker connection from the pool for
// the lifetime of its consume loop, so concurrent coroutines never share a
// connection. The publisher side backs the queue-depth telemetry gauge.
$adapter = new Swoole(
$pools->get('consumer')->pop()->getResource(),
new BrokerPool(
publisher: $pools->get('consumer'),
consumer: $pools->get('consumer'),
),
System::getEnv('_APP_WORKERS_NUM', 1),
$queueName
$queueName,
maxCoroutines: (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1),
resources: $container,
);
$worker = new Server($adapter, $container);
$worker = new Server($adapter);
try {
$worker->init()->action(function () use ($worker, $registerWorkerMessageResources, $queueName) {
$registerWorkerMessageResources($worker->getContainer());
$registerWorkerMessageResources($worker->context());
Span::init("worker.{$queueName}");
});
@@ -103,7 +111,7 @@ try {
$container->set('bus', function ($register) use ($worker) {
return $register->get('bus')->setResolver(
fn (string $name) => $worker->getContainer()->get($name)
fn (string $name) => $worker->context()->get($name)
);
}, ['register']);
+2 -2
View File
@@ -76,11 +76,11 @@
"utopia-php/logger": "0.8.*",
"utopia-php/messaging": "1.0.*",
"utopia-php/migration": "1.*",
"utopia-php/platform": "1.0.0-rc3",
"utopia-php/platform": "1.0.0-rc4",
"utopia-php/pools": "1.*",
"utopia-php/span": "1.1.*",
"utopia-php/preloader": "0.2.*",
"utopia-php/queue": "0.18.*",
"utopia-php/queue": "0.19.*",
"utopia-php/servers": "0.4.*",
"utopia-php/registry": "0.5.*",
"utopia-php/storage": "2.*",
Generated
+14 -14
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "551924e300a646e6f433ea0a0f3d6c9f",
"content-hash": "0c23c30bba86ad84377d4d1abce0992a",
"packages": [
{
"name": "adhocore/jwt",
@@ -4773,16 +4773,16 @@
},
{
"name": "utopia-php/platform",
"version": "1.0.0-rc3",
"version": "1.0.0-rc4",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/platform.git",
"reference": "674fd3cea6d1f8db660b01c5be89e8333dfb6de9"
"reference": "455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/platform/zipball/674fd3cea6d1f8db660b01c5be89e8333dfb6de9",
"reference": "674fd3cea6d1f8db660b01c5be89e8333dfb6de9",
"url": "https://api.github.com/repos/utopia-php/platform/zipball/455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0",
"reference": "455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0",
"shasum": ""
},
"require": {
@@ -4791,7 +4791,7 @@
"php": ">=8.3",
"utopia-php/cli": "0.23.*",
"utopia-php/http": "^2.0@RC",
"utopia-php/queue": "0.18.*",
"utopia-php/queue": "^0.19",
"utopia-php/servers": "0.4.*"
},
"require-dev": {
@@ -4818,9 +4818,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/platform/issues",
"source": "https://github.com/utopia-php/platform/tree/1.0.0-rc3"
"source": "https://github.com/utopia-php/platform/tree/1.0.0-rc4"
},
"time": "2026-05-25T11:48:18+00:00"
"time": "2026-06-02T20:19:26+00:00"
},
{
"name": "utopia-php/pools",
@@ -4976,16 +4976,16 @@
},
{
"name": "utopia-php/queue",
"version": "0.18.6",
"version": "0.19.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/queue.git",
"reference": "954cd3788b37697b83fd2757adcd93422f21a846"
"reference": "cd4248af69006b2568c3486a675f27cb21191859"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/queue/zipball/954cd3788b37697b83fd2757adcd93422f21a846",
"reference": "954cd3788b37697b83fd2757adcd93422f21a846",
"url": "https://api.github.com/repos/utopia-php/queue/zipball/cd4248af69006b2568c3486a675f27cb21191859",
"reference": "cd4248af69006b2568c3486a675f27cb21191859",
"shasum": ""
},
"require": {
@@ -5037,9 +5037,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/queue/issues",
"source": "https://github.com/utopia-php/queue/tree/0.18.6"
"source": "https://github.com/utopia-php/queue/tree/0.19.0"
},
"time": "2026-06-01T19:47:57+00:00"
"time": "2026-06-02T20:12:18+00:00"
},
{
"name": "utopia-php/registry",
+28 -1
View File
@@ -349,6 +349,8 @@ services:
- appwrite-embedding
environment:
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_WORKER_MAX_COROUTINES=1
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
@@ -383,6 +385,8 @@ services:
- appwrite-embedding
environment:
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_WORKER_MAX_COROUTINES=1
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
@@ -424,6 +428,8 @@ services:
environment:
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_WORKER_MAX_COROUTINES=8
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
@@ -484,6 +490,8 @@ services:
- appwrite-embedding
environment:
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_WORKER_MAX_COROUTINES=1
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
@@ -504,7 +512,6 @@ services:
- _APP_DB_USER_VECTORSDB
- _APP_DB_PASS_VECTORSDB
- _APP_LOGGING_CONFIG
- _APP_WORKERS_NUM
- _APP_QUEUE_NAME
- _APP_DATABASE_SHARED_TABLES
@@ -527,6 +534,8 @@ services:
- appwrite-embedding
environment:
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_WORKER_MAX_COROUTINES=8
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
@@ -605,6 +614,8 @@ services:
- _APP_OPTIONS_FORCE_HTTPS
# Basic
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_WORKER_MAX_COROUTINES=8
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_LOGGING_CONFIG
@@ -665,6 +676,8 @@ services:
- ./src:/usr/src/code/src
environment:
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_WORKER_MAX_COROUTINES=8
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
@@ -705,6 +718,8 @@ services:
- ${_APP_DB_HOST:-mongodb}
environment:
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_WORKER_MAX_COROUTINES=8
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
@@ -736,6 +751,8 @@ services:
- redis
environment:
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_WORKER_MAX_COROUTINES=8
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
@@ -782,6 +799,8 @@ services:
# - smtp
environment:
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_WORKER_MAX_COROUTINES=1
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
@@ -823,6 +842,8 @@ services:
- redis
environment:
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_WORKER_MAX_COROUTINES=1
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
@@ -883,6 +904,8 @@ services:
- ${_APP_DB_HOST:-mongodb}
environment:
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_WORKER_MAX_COROUTINES=1
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
@@ -1044,6 +1067,8 @@ services:
- ${_APP_DB_HOST:-mongodb}
environment:
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_WORKER_MAX_COROUTINES=1
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
@@ -1078,6 +1103,8 @@ services:
- appwrite-embedding
environment:
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_WORKER_MAX_COROUTINES=1
- _APP_WORKER_PER_CORE
- _APP_POOL_ADAPTER
- _APP_OPENSSL_KEY_V1
+3 -7
View File
@@ -53,8 +53,6 @@ use function Swoole\Coroutine\batch;
class Messaging extends Action
{
private ?Local $localDevice = null;
private ?SMSAdapter $adapter = null;
public static function getName(): string
@@ -716,11 +714,9 @@ class Messaging extends Action
private function getLocalDevice($project): Local
{
if ($this->localDevice === null) {
$this->localDevice = new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId());
}
return $this->localDevice;
// Not cached: the path is project-scoped and the worker handles
// messages from many projects (and coroutines run them concurrently).
return new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId());
}
private function createInternalSMSAdapter(): ?SMSAdapter