Merge pull request #13762 from appwrite/feat/queue-3

chore(queue): move to utopia-php/queue 3.0
This commit is contained in:
Levi van Noort authored and GitHub committed 2026-09-18 16:29:41 +02:00
commit f61009be8c
4 files changed
+93 -18

No files matched your search

+2 -2
View File
@@ -104,10 +104,10 @@
"utopia-php/messaging": "^2.1",
"utopia-php/migration": "^2.0.0",
"utopia-php/mqtt": "0.0.*",
"utopia-php/platform": "1.0.0-rc20",
"utopia-php/platform": "1.0.0-rc21",
"utopia-php/pools": "2.*",
"utopia-php/span": "^4.2",
"utopia-php/queue": "^2.2.4",
"utopia-php/queue": "^3.0.0",
"utopia-php/servers": "0.4.*",
"utopia-php/registry": "0.5.*",
"utopia-php/storage": "^4.0",
Generated
+15 -14
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "8a4455dd9986b24d5ab9788f1d97a2bf",
"content-hash": "b6de094fa4baf857915a252387762572",
"packages": [
{
"name": "adhocore/jwt",
@@ -4248,16 +4248,16 @@
},
{
"name": "utopia-php/platform",
"version": "1.0.0-rc20",
"version": "1.0.0-rc21",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/platform.git",
"reference": "746d403d417bb8ae42e5f8d94e1bbe0e75e4c87d"
"reference": "b5bb2f661efa35ef06a6e86e97eeaea20ead66f8"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/platform/zipball/746d403d417bb8ae42e5f8d94e1bbe0e75e4c87d",
"reference": "746d403d417bb8ae42e5f8d94e1bbe0e75e4c87d",
"url": "https://api.github.com/repos/utopia-php/platform/zipball/b5bb2f661efa35ef06a6e86e97eeaea20ead66f8",
"reference": "b5bb2f661efa35ef06a6e86e97eeaea20ead66f8",
"shasum": ""
},
"require": {
@@ -4266,7 +4266,7 @@
"php": ">=8.5",
"utopia-php/cli": "^0.24",
"utopia-php/http": "^2.0@RC",
"utopia-php/queue": "0.23.* || 0.24.* || ^1.0.0 || ^2.0.0",
"utopia-php/queue": "0.23.* || 0.24.* || ^1.0.0 || ^2.0.0 || ^3.0.0",
"utopia-php/servers": "^0.4"
},
"type": "library",
@@ -4289,9 +4289,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/platform/issues",
"source": "https://github.com/utopia-php/platform/tree/1.0.0-rc20"
"source": "https://github.com/utopia-php/platform/tree/1.0.0-rc21"
},
"time": "2026-09-04T17:01:57+00:00"
"time": "2026-09-18T13:07:00+00:00"
},
{
"name": "utopia-php/pools",
@@ -4400,16 +4400,16 @@
},
{
"name": "utopia-php/queue",
"version": "2.2.4",
"version": "3.0.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/queue.git",
"reference": "c6994fe68c58d48f303693dad2762d1f80d5bf97"
"reference": "ee2c461a8244173bcfba7ad5e299d5e11c406b02"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/queue/zipball/c6994fe68c58d48f303693dad2762d1f80d5bf97",
"reference": "c6994fe68c58d48f303693dad2762d1f80d5bf97",
"url": "https://api.github.com/repos/utopia-php/queue/zipball/ee2c461a8244173bcfba7ad5e299d5e11c406b02",
"reference": "ee2c461a8244173bcfba7ad5e299d5e11c406b02",
"shasum": ""
},
"require": {
@@ -4428,6 +4428,7 @@
"workerman/workerman": "^4.0"
},
"suggest": {
"ext-igbinary": "Required for the Igbinary codec (Codec\\Igbinary).",
"ext-pcntl": "Fallback for Adapter\\KubernetesJob graceful shutdown when Swoole is unavailable.",
"ext-swoole": "Needed to support Swoole.",
"utopia-php/nats": "Needed to support the NATS JetStream broker (Broker\\Nats).",
@@ -4460,9 +4461,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/queue/issues",
"source": "https://github.com/utopia-php/queue/tree/2.2.4"
"source": "https://github.com/utopia-php/queue/tree/3.0.0"
},
"time": "2026-09-11T17:50:20+00:00"
"time": "2026-09-18T11:36:46+00:00"
},
{
"name": "utopia-php/registry",
+32 -2
View File
@@ -63,6 +63,32 @@ final class WorkerConcurrencyTest extends TestCase
$this->assertSame(3, $maxActive['v1-functions'], 'sibling queues keep their own higher caps');
}
/**
* A batched sibling must not cost the databases queue its serialisation.
*
* Batching changes how many messages one receive claims, not how many run at
* once -- the cap still decides that. The way it could go wrong is at the
* boundary between queues: a batch claimed for one queue must not occupy the
* slots another one is serialised by, and must not lose or duplicate the
* messages it claimed. So this is the combined-mode case with the parallel
* sibling batching, asserted on what came out rather than on how it was
* fetched.
*/
public function testABatchedSiblingLeavesTheDatabasesQueueSerial(): void
{
[$processed, $maxActive] = $this->runQueues(
queues: [
['name' => 'database_db_main', 'messages' => 6, 'maxCoroutines' => 1],
['name' => 'v1-functions', 'messages' => 9, 'maxCoroutines' => 3, 'batch' => 3],
],
);
$this->assertSame(6, $processed['database_db_main']);
$this->assertSame(9, $processed['v1-functions'], 'a batch must deliver every message it claimed, exactly once');
$this->assertSame(1, $maxActive['database_db_main'], 'databases stays serial beside a batching sibling');
$this->assertSame(3, $maxActive['v1-functions'], 'a batch is claimed three at a time, not run three-deep past the cap');
}
public function testMessageWithoutFreeDatabasesSlotStaysInBroker(): void
{
$connection = new InMemoryConnection();
@@ -106,7 +132,7 @@ final class WorkerConcurrencyTest extends TestCase
}
/**
* @param list<array{name: string, messages: int, maxCoroutines: int}> $queues
* @param list<array{name: string, messages: int, maxCoroutines: int, batch?: int}> $queues
* @return array{0: array<string, int>, 1: array<string, int>} [processedByQueue, maxActiveByQueue]
*/
private function runQueues(array $queues): array
@@ -135,7 +161,11 @@ final class WorkerConcurrencyTest extends TestCase
for ($i = 0; $i < $spec['messages']; $i++) {
$broker->publish($queue, ['n' => $i]);
}
$specs[] = ['queue' => $queue, 'maxCoroutines' => $spec['maxCoroutines']];
$specs[] = [
'queue' => $queue,
'maxCoroutines' => $spec['maxCoroutines'],
'batch' => $spec['batch'] ?? 1,
];
}
$adapter = new Swoole($broker, 1, self::NAMESPACE);
@@ -88,6 +88,45 @@ final class InMemoryConnection implements Connection
return \is_string($value) ? $value : false;
}
/**
* Pop up to $count payloads from the tail, in pop order.
*
* Fewer than asked — including none — is the ordinary answer on a list that
* holds fewer, matching what LMPOP does for the real connection: it waits
* only for the first message and then takes whatever company it finds.
*
* $timeout is unused, as it is in every other pop here. Nothing in this fake
* blocks; an empty pop yields instead, which is what keeps a busy receive
* loop from starving the handlers it shares a thread with.
*
* @return list<string>
*/
public function rightPopMany(string $queue, int $count, int $timeout): array
{
// The first one goes through pop(), so an empty queue costs exactly what
// it costs rightPop().
$first = $this->pop($queue, fromTail: true);
if (!\is_string($first)) {
return [];
}
$batch = [$first];
// The rest is only what is already on the list. Going back through pop()
// would yield once more on the message that is not there, turning a
// short batch into a sleep the real connection never takes.
while (\count($batch) < $count && !empty($this->lists[$queue])) {
$value = array_pop($this->lists[$queue]);
if (!\is_string($value)) {
break;
}
$batch[] = $value;
}
return $batch;
}
public function rightPopLeftPush(string $queue, string $destination, int $timeout): string|false
{
$value = $this->rightPop($queue, $timeout);
@@ -185,6 +224,11 @@ final class InMemoryConnection implements Connection
return $this->counters[$key] = ($this->counters[$key] ?? 0) + 1;
}
public function incrementBy(string $key, int $by): int
{
return $this->counters[$key] = ($this->counters[$key] ?? 0) + $by;
}
public function decrement(string $key): int
{
return $this->counters[$key] = ($this->counters[$key] ?? 0) - 1;