diff --git a/composer.json b/composer.json index 2397a2cf06..28053cdb1a 100644 --- a/composer.json +++ b/composer.json @@ -104,10 +104,10 @@ "utopia-php/messaging": "^2.1", "utopia-php/migration": "^2.0.0", "utopia-php/mqtt": "0.0.*", - "utopia-php/platform": "1.0.0-rc20", + "utopia-php/platform": "1.0.0-rc21", "utopia-php/pools": "2.*", "utopia-php/span": "^4.2", - "utopia-php/queue": "^2.2.4", + "utopia-php/queue": "^3.0.0", "utopia-php/servers": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/storage": "^4.0", diff --git a/composer.lock b/composer.lock index 31388535d7..486bd945e4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8a4455dd9986b24d5ab9788f1d97a2bf", + "content-hash": "b6de094fa4baf857915a252387762572", "packages": [ { "name": "adhocore/jwt", @@ -4248,16 +4248,16 @@ }, { "name": "utopia-php/platform", - "version": "1.0.0-rc20", + "version": "1.0.0-rc21", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "746d403d417bb8ae42e5f8d94e1bbe0e75e4c87d" + "reference": "b5bb2f661efa35ef06a6e86e97eeaea20ead66f8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/platform/zipball/746d403d417bb8ae42e5f8d94e1bbe0e75e4c87d", - "reference": "746d403d417bb8ae42e5f8d94e1bbe0e75e4c87d", + "url": "https://api.github.com/repos/utopia-php/platform/zipball/b5bb2f661efa35ef06a6e86e97eeaea20ead66f8", + "reference": "b5bb2f661efa35ef06a6e86e97eeaea20ead66f8", "shasum": "" }, "require": { @@ -4266,7 +4266,7 @@ "php": ">=8.5", "utopia-php/cli": "^0.24", "utopia-php/http": "^2.0@RC", - "utopia-php/queue": "0.23.* || 0.24.* || ^1.0.0 || ^2.0.0", + "utopia-php/queue": "0.23.* || 0.24.* || ^1.0.0 || ^2.0.0 || ^3.0.0", "utopia-php/servers": "^0.4" }, "type": "library", @@ -4289,9 +4289,9 @@ ], "support": { "issues": "https://github.com/utopia-php/platform/issues", - "source": "https://github.com/utopia-php/platform/tree/1.0.0-rc20" + "source": "https://github.com/utopia-php/platform/tree/1.0.0-rc21" }, - "time": "2026-09-04T17:01:57+00:00" + "time": "2026-09-18T13:07:00+00:00" }, { "name": "utopia-php/pools", @@ -4400,16 +4400,16 @@ }, { "name": "utopia-php/queue", - "version": "2.2.4", + "version": "3.0.0", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "c6994fe68c58d48f303693dad2762d1f80d5bf97" + "reference": "ee2c461a8244173bcfba7ad5e299d5e11c406b02" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/c6994fe68c58d48f303693dad2762d1f80d5bf97", - "reference": "c6994fe68c58d48f303693dad2762d1f80d5bf97", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/ee2c461a8244173bcfba7ad5e299d5e11c406b02", + "reference": "ee2c461a8244173bcfba7ad5e299d5e11c406b02", "shasum": "" }, "require": { @@ -4428,6 +4428,7 @@ "workerman/workerman": "^4.0" }, "suggest": { + "ext-igbinary": "Required for the Igbinary codec (Codec\\Igbinary).", "ext-pcntl": "Fallback for Adapter\\KubernetesJob graceful shutdown when Swoole is unavailable.", "ext-swoole": "Needed to support Swoole.", "utopia-php/nats": "Needed to support the NATS JetStream broker (Broker\\Nats).", @@ -4460,9 +4461,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/2.2.4" + "source": "https://github.com/utopia-php/queue/tree/3.0.0" }, - "time": "2026-09-11T17:50:20+00:00" + "time": "2026-09-18T11:36:46+00:00" }, { "name": "utopia-php/registry", diff --git a/tests/e2e/General/WorkerConcurrencyTest.php b/tests/e2e/General/WorkerConcurrencyTest.php index 5b742b9796..20ad5e1226 100644 --- a/tests/e2e/General/WorkerConcurrencyTest.php +++ b/tests/e2e/General/WorkerConcurrencyTest.php @@ -63,6 +63,32 @@ final class WorkerConcurrencyTest extends TestCase $this->assertSame(3, $maxActive['v1-functions'], 'sibling queues keep their own higher caps'); } + /** + * A batched sibling must not cost the databases queue its serialisation. + * + * Batching changes how many messages one receive claims, not how many run at + * once -- the cap still decides that. The way it could go wrong is at the + * boundary between queues: a batch claimed for one queue must not occupy the + * slots another one is serialised by, and must not lose or duplicate the + * messages it claimed. So this is the combined-mode case with the parallel + * sibling batching, asserted on what came out rather than on how it was + * fetched. + */ + public function testABatchedSiblingLeavesTheDatabasesQueueSerial(): void + { + [$processed, $maxActive] = $this->runQueues( + queues: [ + ['name' => 'database_db_main', 'messages' => 6, 'maxCoroutines' => 1], + ['name' => 'v1-functions', 'messages' => 9, 'maxCoroutines' => 3, 'batch' => 3], + ], + ); + + $this->assertSame(6, $processed['database_db_main']); + $this->assertSame(9, $processed['v1-functions'], 'a batch must deliver every message it claimed, exactly once'); + $this->assertSame(1, $maxActive['database_db_main'], 'databases stays serial beside a batching sibling'); + $this->assertSame(3, $maxActive['v1-functions'], 'a batch is claimed three at a time, not run three-deep past the cap'); + } + public function testMessageWithoutFreeDatabasesSlotStaysInBroker(): void { $connection = new InMemoryConnection(); @@ -106,7 +132,7 @@ final class WorkerConcurrencyTest extends TestCase } /** - * @param list $queues + * @param list $queues * @return array{0: array, 1: array} [processedByQueue, maxActiveByQueue] */ private function runQueues(array $queues): array @@ -135,7 +161,11 @@ final class WorkerConcurrencyTest extends TestCase for ($i = 0; $i < $spec['messages']; $i++) { $broker->publish($queue, ['n' => $i]); } - $specs[] = ['queue' => $queue, 'maxCoroutines' => $spec['maxCoroutines']]; + $specs[] = [ + 'queue' => $queue, + 'maxCoroutines' => $spec['maxCoroutines'], + 'batch' => $spec['batch'] ?? 1, + ]; } $adapter = new Swoole($broker, 1, self::NAMESPACE); diff --git a/tests/extensions/Queue/InMemoryConnection.php b/tests/extensions/Queue/InMemoryConnection.php index bb76a899e5..08d079c700 100644 --- a/tests/extensions/Queue/InMemoryConnection.php +++ b/tests/extensions/Queue/InMemoryConnection.php @@ -88,6 +88,45 @@ final class InMemoryConnection implements Connection return \is_string($value) ? $value : false; } + /** + * Pop up to $count payloads from the tail, in pop order. + * + * Fewer than asked — including none — is the ordinary answer on a list that + * holds fewer, matching what LMPOP does for the real connection: it waits + * only for the first message and then takes whatever company it finds. + * + * $timeout is unused, as it is in every other pop here. Nothing in this fake + * blocks; an empty pop yields instead, which is what keeps a busy receive + * loop from starving the handlers it shares a thread with. + * + * @return list + */ + public function rightPopMany(string $queue, int $count, int $timeout): array + { + // The first one goes through pop(), so an empty queue costs exactly what + // it costs rightPop(). + $first = $this->pop($queue, fromTail: true); + if (!\is_string($first)) { + return []; + } + + $batch = [$first]; + + // The rest is only what is already on the list. Going back through pop() + // would yield once more on the message that is not there, turning a + // short batch into a sleep the real connection never takes. + while (\count($batch) < $count && !empty($this->lists[$queue])) { + $value = array_pop($this->lists[$queue]); + if (!\is_string($value)) { + break; + } + + $batch[] = $value; + } + + return $batch; + } + public function rightPopLeftPush(string $queue, string $destination, int $timeout): string|false { $value = $this->rightPop($queue, $timeout); @@ -185,6 +224,11 @@ final class InMemoryConnection implements Connection return $this->counters[$key] = ($this->counters[$key] ?? 0) + 1; } + public function incrementBy(string $key, int $by): int + { + return $this->counters[$key] = ($this->counters[$key] ?? 0) + $by; + } + public function decrement(string $key): int { return $this->counters[$key] = ($this->counters[$key] ?? 0) - 1;