From 6cc05c74fe6a59d5f61ce41c5913664cd3b8c933 Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Fri, 18 Sep 2026 14:58:35 +0200 Subject: [PATCH 1/4] chore(queue): move to utopia-php/queue 3.0 The major adds rightPopMany() and incrementBy() to Connection, which is the whole of its breaking surface here: InMemoryConnection is the only implementation in this repository, and WorkerConcurrencyTest drives it. Consumer is unchanged, so FakeConsumer in InitJobsTest needs nothing. 3.0 brings PermanentFailure (a handler can end a message's life rather than spend its whole redelivery budget on work that cannot succeed), a pluggable envelope codec, and batched receive -- none of which this commit takes up. It moves the floor so applications built on server-ce can. The lock is not updated yet: utopia-php/platform 1.0.0-rc20 caps queue at ^2.0.0, so this needs rc21 (utopia-php/monorepo#301) first. --- composer.json | 2 +- tests/extensions/Queue/InMemoryConnection.php | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 2397a2cf06..89c128d45f 100644 --- a/composer.json +++ b/composer.json @@ -107,7 +107,7 @@ "utopia-php/platform": "1.0.0-rc20", "utopia-php/pools": "2.*", "utopia-php/span": "^4.2", - "utopia-php/queue": "^2.2.4", + "utopia-php/queue": "^3.0.0", "utopia-php/servers": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/storage": "^4.0", diff --git a/tests/extensions/Queue/InMemoryConnection.php b/tests/extensions/Queue/InMemoryConnection.php index bb76a899e5..006a5d07ab 100644 --- a/tests/extensions/Queue/InMemoryConnection.php +++ b/tests/extensions/Queue/InMemoryConnection.php @@ -88,6 +88,29 @@ final class InMemoryConnection implements Connection return \is_string($value) ? $value : false; } + /** + * Pop up to $count payloads from the tail, in pop order. + * + * Fewer than asked — including none — is the ordinary answer on a list that + * holds fewer, matching what LMPOP does for the real connection. + * + * @return list + */ + public function rightPopMany(string $queue, int $count, int $timeout): array + { + $batch = []; + for ($taken = 0; $taken < $count; $taken++) { + $value = $this->pop($queue, fromTail: true); + if (!\is_string($value)) { + break; + } + + $batch[] = $value; + } + + return $batch; + } + public function rightPopLeftPush(string $queue, string $destination, int $timeout): string|false { $value = $this->rightPop($queue, $timeout); @@ -185,6 +208,11 @@ final class InMemoryConnection implements Connection return $this->counters[$key] = ($this->counters[$key] ?? 0) + 1; } + public function incrementBy(string $key, int $by): int + { + return $this->counters[$key] = ($this->counters[$key] ?? 0) + $by; + } + public function decrement(string $key): int { return $this->counters[$key] = ($this->counters[$key] ?? 0) - 1; From 657b1af2ef8d272d5391b6e39d189645f1eb25d7 Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:11:13 +0200 Subject: [PATCH 2/4] chore(queue): lock queue 3.0.0 and platform 1.0.0-rc21 platform rc20 caps queue at ^2.0.0, so the major could not resolve through it. rc21 widens that and passes a job spec's batch size through to Server::job(), which is what makes batched receive reachable from an application at all. --- composer.json | 2 +- composer.lock | 29 +++++++++++++++-------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/composer.json b/composer.json index 89c128d45f..28053cdb1a 100644 --- a/composer.json +++ b/composer.json @@ -104,7 +104,7 @@ "utopia-php/messaging": "^2.1", "utopia-php/migration": "^2.0.0", "utopia-php/mqtt": "0.0.*", - "utopia-php/platform": "1.0.0-rc20", + "utopia-php/platform": "1.0.0-rc21", "utopia-php/pools": "2.*", "utopia-php/span": "^4.2", "utopia-php/queue": "^3.0.0", diff --git a/composer.lock b/composer.lock index 31388535d7..486bd945e4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8a4455dd9986b24d5ab9788f1d97a2bf", + "content-hash": "b6de094fa4baf857915a252387762572", "packages": [ { "name": "adhocore/jwt", @@ -4248,16 +4248,16 @@ }, { "name": "utopia-php/platform", - "version": "1.0.0-rc20", + "version": "1.0.0-rc21", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "746d403d417bb8ae42e5f8d94e1bbe0e75e4c87d" + "reference": "b5bb2f661efa35ef06a6e86e97eeaea20ead66f8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/platform/zipball/746d403d417bb8ae42e5f8d94e1bbe0e75e4c87d", - "reference": "746d403d417bb8ae42e5f8d94e1bbe0e75e4c87d", + "url": "https://api.github.com/repos/utopia-php/platform/zipball/b5bb2f661efa35ef06a6e86e97eeaea20ead66f8", + "reference": "b5bb2f661efa35ef06a6e86e97eeaea20ead66f8", "shasum": "" }, "require": { @@ -4266,7 +4266,7 @@ "php": ">=8.5", "utopia-php/cli": "^0.24", "utopia-php/http": "^2.0@RC", - "utopia-php/queue": "0.23.* || 0.24.* || ^1.0.0 || ^2.0.0", + "utopia-php/queue": "0.23.* || 0.24.* || ^1.0.0 || ^2.0.0 || ^3.0.0", "utopia-php/servers": "^0.4" }, "type": "library", @@ -4289,9 +4289,9 @@ ], "support": { "issues": "https://github.com/utopia-php/platform/issues", - "source": "https://github.com/utopia-php/platform/tree/1.0.0-rc20" + "source": "https://github.com/utopia-php/platform/tree/1.0.0-rc21" }, - "time": "2026-09-04T17:01:57+00:00" + "time": "2026-09-18T13:07:00+00:00" }, { "name": "utopia-php/pools", @@ -4400,16 +4400,16 @@ }, { "name": "utopia-php/queue", - "version": "2.2.4", + "version": "3.0.0", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "c6994fe68c58d48f303693dad2762d1f80d5bf97" + "reference": "ee2c461a8244173bcfba7ad5e299d5e11c406b02" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/c6994fe68c58d48f303693dad2762d1f80d5bf97", - "reference": "c6994fe68c58d48f303693dad2762d1f80d5bf97", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/ee2c461a8244173bcfba7ad5e299d5e11c406b02", + "reference": "ee2c461a8244173bcfba7ad5e299d5e11c406b02", "shasum": "" }, "require": { @@ -4428,6 +4428,7 @@ "workerman/workerman": "^4.0" }, "suggest": { + "ext-igbinary": "Required for the Igbinary codec (Codec\\Igbinary).", "ext-pcntl": "Fallback for Adapter\\KubernetesJob graceful shutdown when Swoole is unavailable.", "ext-swoole": "Needed to support Swoole.", "utopia-php/nats": "Needed to support the NATS JetStream broker (Broker\\Nats).", @@ -4460,9 +4461,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/2.2.4" + "source": "https://github.com/utopia-php/queue/tree/3.0.0" }, - "time": "2026-09-11T17:50:20+00:00" + "time": "2026-09-18T11:36:46+00:00" }, { "name": "utopia-php/registry", From a0fd9a00ace123bbbe5c8d5abbf5da7a865ba9c9 Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:23:18 +0200 Subject: [PATCH 3/4] test(queue): prove the in-memory fake serves a batched receive Review flagged rightPopMany() for ignoring $timeout. It does, as every pop in this fake does -- nothing here blocks, an empty pop yields, and that is what keeps a busy receive loop from starving the handlers it shares a thread with. Making one method block would be a new behaviour, not a fix. The real defect underneath was quieter: each message after the first went back through pop(), so a batch shorter than asked ended on a yield the real connection never takes. LMPOP waits for the first message only and takes the company it finds. The rest now comes straight off the list. A batch the fake cannot serve looks exactly like an idle queue, so the failure it would cause is silence. The test claims a three-message backlog in one receive and asserts the order and the drained queue -- seen red against a rightPopMany() that returns nothing. --- tests/e2e/General/WorkerConcurrencyTest.php | 32 +++++++++++++++++++ tests/extensions/Queue/InMemoryConnection.php | 24 +++++++++++--- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/tests/e2e/General/WorkerConcurrencyTest.php b/tests/e2e/General/WorkerConcurrencyTest.php index 5b742b9796..aa2a67f6a8 100644 --- a/tests/e2e/General/WorkerConcurrencyTest.php +++ b/tests/e2e/General/WorkerConcurrencyTest.php @@ -105,6 +105,38 @@ final class WorkerConcurrencyTest extends TestCase ); } + /** + * The fake has to serve a batched receive, not just a single pop. + * + * queue 3.0 gave Broker\Redis a receiveBatch() that takes the message it + * waited for plus whatever is already behind it in one round trip, over + * Connection::rightPopMany(). A fake that answered that with an empty batch + * would leave every batched wiring untestable here, and would do it quietly: + * an empty receive is what an idle queue looks like. + */ + public function testTheFakeServesABatchedReceive(): void + { + $connection = new InMemoryConnection(); + $broker = new Redis($connection, $connection); + $queue = new Queue('v1-functions', self::NAMESPACE); + + $broker->publish($queue, ['n' => 0]); + $broker->publish($queue, ['n' => 1]); + $broker->publish($queue, ['n' => 2]); + + // Asking for more than the list holds: a short batch is the ordinary + // answer, not an error, because only the first message is waited for. + $claimed = $broker->receiveBatch($queue, 1, 5); + + $this->assertCount(3, $claimed); + $this->assertSame( + [0, 1, 2], + \array_map(static fn ($message): int => $message->getPayload()['n'], $claimed), + 'a batch keeps the order the messages were published in', + ); + $this->assertSame(0, $broker->getQueueSize($queue), 'one receive claimed the whole backlog'); + } + /** * @param list $queues * @return array{0: array, 1: array} [processedByQueue, maxActiveByQueue] diff --git a/tests/extensions/Queue/InMemoryConnection.php b/tests/extensions/Queue/InMemoryConnection.php index 006a5d07ab..08d079c700 100644 --- a/tests/extensions/Queue/InMemoryConnection.php +++ b/tests/extensions/Queue/InMemoryConnection.php @@ -92,15 +92,31 @@ final class InMemoryConnection implements Connection * Pop up to $count payloads from the tail, in pop order. * * Fewer than asked — including none — is the ordinary answer on a list that - * holds fewer, matching what LMPOP does for the real connection. + * holds fewer, matching what LMPOP does for the real connection: it waits + * only for the first message and then takes whatever company it finds. + * + * $timeout is unused, as it is in every other pop here. Nothing in this fake + * blocks; an empty pop yields instead, which is what keeps a busy receive + * loop from starving the handlers it shares a thread with. * * @return list */ public function rightPopMany(string $queue, int $count, int $timeout): array { - $batch = []; - for ($taken = 0; $taken < $count; $taken++) { - $value = $this->pop($queue, fromTail: true); + // The first one goes through pop(), so an empty queue costs exactly what + // it costs rightPop(). + $first = $this->pop($queue, fromTail: true); + if (!\is_string($first)) { + return []; + } + + $batch = [$first]; + + // The rest is only what is already on the list. Going back through pop() + // would yield once more on the message that is not there, turning a + // short batch into a sleep the real connection never takes. + while (\count($batch) < $count && !empty($this->lists[$queue])) { + $value = array_pop($this->lists[$queue]); if (!\is_string($value)) { break; } From 2a0f45ec4bc2986440a636bf6a4e9c8f4c2cf99f Mon Sep 17 00:00:00 2001 From: Levi van Noort <73097785+levivannoort@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:27:59 +0200 Subject: [PATCH 4/4] test(queue): assert the batched sibling at the worker boundary The previous test's subject was the fake itself -- it drove receiveBatch() directly and asserted the broker's return. Review called that implementation-coupled and it is: a test whose subject is a fixture proves the fixture, not the worker. What matters about batching here is the boundary between queues. Batching changes how many messages one receive claims, not how many run at once, so the way it could go wrong is by occupying slots another queue is serialised by, or by losing or duplicating what it claimed. The combined -mode case now runs its parallel sibling batched and asserts what came out: every message once, databases still serial, the cap still the cap. Worth knowing what this cannot see: a batch size that is ignored outright behaves identically at this boundary, because the whole point of a batch is fewer round trips and nothing else. Claiming is not observable from out here. --- tests/e2e/General/WorkerConcurrencyTest.php | 66 ++++++++++----------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/tests/e2e/General/WorkerConcurrencyTest.php b/tests/e2e/General/WorkerConcurrencyTest.php index aa2a67f6a8..20ad5e1226 100644 --- a/tests/e2e/General/WorkerConcurrencyTest.php +++ b/tests/e2e/General/WorkerConcurrencyTest.php @@ -63,6 +63,32 @@ final class WorkerConcurrencyTest extends TestCase $this->assertSame(3, $maxActive['v1-functions'], 'sibling queues keep their own higher caps'); } + /** + * A batched sibling must not cost the databases queue its serialisation. + * + * Batching changes how many messages one receive claims, not how many run at + * once -- the cap still decides that. The way it could go wrong is at the + * boundary between queues: a batch claimed for one queue must not occupy the + * slots another one is serialised by, and must not lose or duplicate the + * messages it claimed. So this is the combined-mode case with the parallel + * sibling batching, asserted on what came out rather than on how it was + * fetched. + */ + public function testABatchedSiblingLeavesTheDatabasesQueueSerial(): void + { + [$processed, $maxActive] = $this->runQueues( + queues: [ + ['name' => 'database_db_main', 'messages' => 6, 'maxCoroutines' => 1], + ['name' => 'v1-functions', 'messages' => 9, 'maxCoroutines' => 3, 'batch' => 3], + ], + ); + + $this->assertSame(6, $processed['database_db_main']); + $this->assertSame(9, $processed['v1-functions'], 'a batch must deliver every message it claimed, exactly once'); + $this->assertSame(1, $maxActive['database_db_main'], 'databases stays serial beside a batching sibling'); + $this->assertSame(3, $maxActive['v1-functions'], 'a batch is claimed three at a time, not run three-deep past the cap'); + } + public function testMessageWithoutFreeDatabasesSlotStaysInBroker(): void { $connection = new InMemoryConnection(); @@ -106,39 +132,7 @@ final class WorkerConcurrencyTest extends TestCase } /** - * The fake has to serve a batched receive, not just a single pop. - * - * queue 3.0 gave Broker\Redis a receiveBatch() that takes the message it - * waited for plus whatever is already behind it in one round trip, over - * Connection::rightPopMany(). A fake that answered that with an empty batch - * would leave every batched wiring untestable here, and would do it quietly: - * an empty receive is what an idle queue looks like. - */ - public function testTheFakeServesABatchedReceive(): void - { - $connection = new InMemoryConnection(); - $broker = new Redis($connection, $connection); - $queue = new Queue('v1-functions', self::NAMESPACE); - - $broker->publish($queue, ['n' => 0]); - $broker->publish($queue, ['n' => 1]); - $broker->publish($queue, ['n' => 2]); - - // Asking for more than the list holds: a short batch is the ordinary - // answer, not an error, because only the first message is waited for. - $claimed = $broker->receiveBatch($queue, 1, 5); - - $this->assertCount(3, $claimed); - $this->assertSame( - [0, 1, 2], - \array_map(static fn ($message): int => $message->getPayload()['n'], $claimed), - 'a batch keeps the order the messages were published in', - ); - $this->assertSame(0, $broker->getQueueSize($queue), 'one receive claimed the whole backlog'); - } - - /** - * @param list $queues + * @param list $queues * @return array{0: array, 1: array} [processedByQueue, maxActiveByQueue] */ private function runQueues(array $queues): array @@ -167,7 +161,11 @@ final class WorkerConcurrencyTest extends TestCase for ($i = 0; $i < $spec['messages']; $i++) { $broker->publish($queue, ['n' => $i]); } - $specs[] = ['queue' => $queue, 'maxCoroutines' => $spec['maxCoroutines']]; + $specs[] = [ + 'queue' => $queue, + 'maxCoroutines' => $spec['maxCoroutines'], + 'batch' => $spec['batch'] ?? 1, + ]; } $adapter = new Swoole($broker, 1, self::NAMESPACE);