mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Merge pull request #12468 from appwrite/feat/coroutine-workers
Add coroutine concurrency to queue workers
This commit is contained in:
7 files changed
+74
-32
No files matched your search
@@ -150,3 +150,6 @@ _APP_POOL_ADAPTER=stack
|
||||
_APP_WORKER_SCREENSHOTS_ROUTER=http://appwrite
|
||||
_TESTS_OAUTH2_GITHUB_CLIENT_ID=
|
||||
_TESTS_OAUTH2_GITHUB_CLIENT_SECRET=
|
||||
|
||||
_APP_WORKERS_NUM=1
|
||||
_APP_WORKER_MAX_COROUTINES=1
|
||||
+11
-3
@@ -254,6 +254,10 @@ $register->set('pools', function () {
|
||||
$workerCount = intval(System::getEnv('_APP_CPU_NUM', swoole_cpu_num())) * intval(System::getEnv('_APP_WORKER_PER_CORE', 6));
|
||||
$poolSize = max(1, (int)($instanceConnections / $workerCount));
|
||||
|
||||
// Queue workers consume jobs concurrently with coroutines; each in-flight
|
||||
// job may hold a connection, so pools must cover the coroutine count.
|
||||
$poolSize = max($poolSize, (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1));
|
||||
|
||||
foreach ($connections as $key => $connection) {
|
||||
$type = $connection['type'];
|
||||
$multiple = $connection['multiple'];
|
||||
@@ -340,9 +344,13 @@ $register->set('pools', function () {
|
||||
|
||||
// PubSub workers hold one long-lived subscribed connection and also need
|
||||
// spare capacity for publishes from the same process.
|
||||
$connectionPoolSize = $type === 'pubsub'
|
||||
? max(2, $poolSize)
|
||||
: $poolSize;
|
||||
// Consumer pools lease one connection per consume coroutine for the
|
||||
// lifetime of the worker, plus headroom for queue-size reads.
|
||||
$connectionPoolSize = match ($type) {
|
||||
'pubsub' => max(2, $poolSize),
|
||||
'consumer' => $poolSize + 2,
|
||||
default => $poolSize,
|
||||
};
|
||||
|
||||
$pool = new Pool($poolAdapter, $name, $connectionPoolSize, function () use ($type, $resource, $dsn) {
|
||||
// Get Adapter
|
||||
|
||||
+13
-5
@@ -83,17 +83,25 @@ if (\str_starts_with($workerName, 'databases')) {
|
||||
/** @var \Utopia\Pools\Group $pools */
|
||||
$pools = $container->get('pools');
|
||||
|
||||
// Each consume coroutine leases its own broker connection from the pool for
|
||||
// the lifetime of its consume loop, so concurrent coroutines never share a
|
||||
// connection. The publisher side backs the queue-depth telemetry gauge.
|
||||
$adapter = new Swoole(
|
||||
$pools->get('consumer')->pop()->getResource(),
|
||||
new BrokerPool(
|
||||
publisher: $pools->get('consumer'),
|
||||
consumer: $pools->get('consumer'),
|
||||
),
|
||||
System::getEnv('_APP_WORKERS_NUM', 1),
|
||||
$queueName
|
||||
$queueName,
|
||||
maxCoroutines: (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1),
|
||||
resources: $container,
|
||||
);
|
||||
|
||||
$worker = new Server($adapter, $container);
|
||||
$worker = new Server($adapter);
|
||||
|
||||
try {
|
||||
$worker->init()->action(function () use ($worker, $registerWorkerMessageResources, $queueName) {
|
||||
$registerWorkerMessageResources($worker->getContainer());
|
||||
$registerWorkerMessageResources($worker->context());
|
||||
Span::init("worker.{$queueName}");
|
||||
});
|
||||
|
||||
@@ -103,7 +111,7 @@ try {
|
||||
|
||||
$container->set('bus', function ($register) use ($worker) {
|
||||
return $register->get('bus')->setResolver(
|
||||
fn (string $name) => $worker->getContainer()->get($name)
|
||||
fn (string $name) => $worker->context()->get($name)
|
||||
);
|
||||
}, ['register']);
|
||||
|
||||
|
||||
+2
-2
@@ -76,11 +76,11 @@
|
||||
"utopia-php/logger": "0.8.*",
|
||||
"utopia-php/messaging": "1.0.*",
|
||||
"utopia-php/migration": "1.*",
|
||||
"utopia-php/platform": "1.0.0-rc3",
|
||||
"utopia-php/platform": "1.0.0-rc4",
|
||||
"utopia-php/pools": "1.*",
|
||||
"utopia-php/span": "1.1.*",
|
||||
"utopia-php/preloader": "0.2.*",
|
||||
"utopia-php/queue": "0.18.*",
|
||||
"utopia-php/queue": "0.19.*",
|
||||
"utopia-php/servers": "0.4.*",
|
||||
"utopia-php/registry": "0.5.*",
|
||||
"utopia-php/storage": "2.*",
|
||||
|
||||
Generated
+14
-14
@@ -4,7 +4,7 @@
|
||||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "551924e300a646e6f433ea0a0f3d6c9f",
|
||||
"content-hash": "0c23c30bba86ad84377d4d1abce0992a",
|
||||
"packages": [
|
||||
{
|
||||
"name": "adhocore/jwt",
|
||||
@@ -4773,16 +4773,16 @@
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/platform",
|
||||
"version": "1.0.0-rc3",
|
||||
"version": "1.0.0-rc4",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/platform.git",
|
||||
"reference": "674fd3cea6d1f8db660b01c5be89e8333dfb6de9"
|
||||
"reference": "455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/platform/zipball/674fd3cea6d1f8db660b01c5be89e8333dfb6de9",
|
||||
"reference": "674fd3cea6d1f8db660b01c5be89e8333dfb6de9",
|
||||
"url": "https://api.github.com/repos/utopia-php/platform/zipball/455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0",
|
||||
"reference": "455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
@@ -4791,7 +4791,7 @@
|
||||
"php": ">=8.3",
|
||||
"utopia-php/cli": "0.23.*",
|
||||
"utopia-php/http": "^2.0@RC",
|
||||
"utopia-php/queue": "0.18.*",
|
||||
"utopia-php/queue": "^0.19",
|
||||
"utopia-php/servers": "0.4.*"
|
||||
},
|
||||
"require-dev": {
|
||||
@@ -4818,9 +4818,9 @@
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/utopia-php/platform/issues",
|
||||
"source": "https://github.com/utopia-php/platform/tree/1.0.0-rc3"
|
||||
"source": "https://github.com/utopia-php/platform/tree/1.0.0-rc4"
|
||||
},
|
||||
"time": "2026-05-25T11:48:18+00:00"
|
||||
"time": "2026-06-02T20:19:26+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/pools",
|
||||
@@ -4976,16 +4976,16 @@
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/queue",
|
||||
"version": "0.18.6",
|
||||
"version": "0.19.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/queue.git",
|
||||
"reference": "954cd3788b37697b83fd2757adcd93422f21a846"
|
||||
"reference": "cd4248af69006b2568c3486a675f27cb21191859"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/queue/zipball/954cd3788b37697b83fd2757adcd93422f21a846",
|
||||
"reference": "954cd3788b37697b83fd2757adcd93422f21a846",
|
||||
"url": "https://api.github.com/repos/utopia-php/queue/zipball/cd4248af69006b2568c3486a675f27cb21191859",
|
||||
"reference": "cd4248af69006b2568c3486a675f27cb21191859",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
@@ -5037,9 +5037,9 @@
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/utopia-php/queue/issues",
|
||||
"source": "https://github.com/utopia-php/queue/tree/0.18.6"
|
||||
"source": "https://github.com/utopia-php/queue/tree/0.19.0"
|
||||
},
|
||||
"time": "2026-06-01T19:47:57+00:00"
|
||||
"time": "2026-06-02T20:12:18+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/registry",
|
||||
|
||||
+28
-1
@@ -349,6 +349,8 @@ services:
|
||||
- appwrite-embedding
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=1
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
@@ -383,6 +385,8 @@ services:
|
||||
- appwrite-embedding
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=1
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
@@ -424,6 +428,8 @@ services:
|
||||
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=8
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
@@ -484,6 +490,8 @@ services:
|
||||
- appwrite-embedding
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=1
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
@@ -504,7 +512,6 @@ services:
|
||||
- _APP_DB_USER_VECTORSDB
|
||||
- _APP_DB_PASS_VECTORSDB
|
||||
- _APP_LOGGING_CONFIG
|
||||
- _APP_WORKERS_NUM
|
||||
- _APP_QUEUE_NAME
|
||||
- _APP_DATABASE_SHARED_TABLES
|
||||
|
||||
@@ -527,6 +534,8 @@ services:
|
||||
- appwrite-embedding
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=8
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
@@ -605,6 +614,8 @@ services:
|
||||
- _APP_OPTIONS_FORCE_HTTPS
|
||||
# Basic
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=8
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_LOGGING_CONFIG
|
||||
@@ -665,6 +676,8 @@ services:
|
||||
- ./src:/usr/src/code/src
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=8
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
@@ -705,6 +718,8 @@ services:
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=8
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
@@ -736,6 +751,8 @@ services:
|
||||
- redis
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=8
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
@@ -782,6 +799,8 @@ services:
|
||||
# - smtp
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=1
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
@@ -823,6 +842,8 @@ services:
|
||||
- redis
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=1
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
@@ -883,6 +904,8 @@ services:
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=1
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
@@ -1044,6 +1067,8 @@ services:
|
||||
- ${_APP_DB_HOST:-mongodb}
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=1
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
@@ -1078,6 +1103,8 @@ services:
|
||||
- appwrite-embedding
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKERS_NUM=1
|
||||
- _APP_WORKER_MAX_COROUTINES=1
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_POOL_ADAPTER
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
|
||||
@@ -53,8 +53,6 @@ use function Swoole\Coroutine\batch;
|
||||
|
||||
class Messaging extends Action
|
||||
{
|
||||
private ?Local $localDevice = null;
|
||||
|
||||
private ?SMSAdapter $adapter = null;
|
||||
|
||||
public static function getName(): string
|
||||
@@ -716,11 +714,9 @@ class Messaging extends Action
|
||||
|
||||
private function getLocalDevice($project): Local
|
||||
{
|
||||
if ($this->localDevice === null) {
|
||||
$this->localDevice = new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId());
|
||||
}
|
||||
|
||||
return $this->localDevice;
|
||||
// Not cached: the path is project-scoped and the worker handles
|
||||
// messages from many projects (and coroutines run them concurrently).
|
||||
return new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId());
|
||||
}
|
||||
|
||||
private function createInternalSMSAdapter(): ?SMSAdapter
|
||||
|
||||
Reference in new issue
Block a user