From 7c8d7dcb80f36f271e996e93f9a8a1498c824f15 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Jun 2026 04:13:49 +1200 Subject: [PATCH] fix(messaging): address fanout scale review comments Apply five reviewer fixes to the large-topic send path without touching the adapter-agnostic messaging library: - Empty recipient pages no longer count as recipients: a page that resolves to zero identifiers is skipped instead of running batch([]) and masking the "No valid recipients found." outcome. - Best-effort expired-device-token cleanup is isolated from the provider send: the send try/catch wraps only adapter->send(), results are processed outside it, and the findOne/updateDocument cleanup has its own try/catch so a transient DB error can never be misattributed as a send failure or trigger a spurious retry. Delivery accounting stays exact across attempts. - resolveProvider caches the fallback provider under the requested id so a topic full of targets pointing at one disabled/missing provider does not re-query the providers collection once per page. - The test Swoole channel polyfill enforces capacity (push returns false when full) so a saturated semaphore surfaces Contention and the scheduler defers the task, making the concurrency cap genuinely bounded under the polyfill. - ReflectionMethod::setAccessible(true) added for explicitness/back-compat. Adds a regression test proving a thrown expired-token cleanup is swallowed and the real provider failure is recorded instead. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/Appwrite/Platform/Workers/Messaging.php | 63 +++++++++++----- tests/unit/Messaging/MessagingFanoutTest.php | 73 +++++++++++++++++++ .../Messaging/SwooleCoroutinePolyfill.php | 8 ++ 3 files changed, 124 insertions(+), 20 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index b239a69c75..0154716021 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -177,8 +177,6 @@ class Messaging extends Action $hasRecipients = false; foreach ($this->streamRecipients($dbForProject, $topicIds, $userIds, $targetIds, $providerType, $default) as $page) { - $hasRecipients = true; - /** * @var array $tasks */ @@ -217,6 +215,14 @@ class Messaging extends Action } } + // A page can resolve to zero identifiers (e.g. subscribers whose targetInternalId matches no targets + // row), so an empty task list must not count as recipients or run batch() for nothing. + if (empty($tasks)) { + continue; + } + + $hasRecipients = true; + /** * @var array}> $results */ @@ -486,6 +492,10 @@ class Messaging extends Action $provider = $dbForProject->getDocument('providers', $providerId); if ($provider->isEmpty() || !$provider->getAttribute('enabled')) { + // Cache the fallback under this id too, so a topic full of targets pointing at the same + // disabled/missing provider does not re-query the providers collection once per page. + $providers[$providerId] = $default; + return $default; } @@ -592,8 +602,24 @@ class Messaging extends Action $retry = []; + // The try/catch wraps ONLY the provider send. A whole-batch throw is retryable when transient, + // otherwise it records one representative terminal error. The previous behaviour of resetting + // $delivered to 0 on a throw is gone — the retry refactor sums delivered across attempts, and the + // expired-device-token cleanup below is isolated in its own try so a DB hiccup there can never be + // misattributed as a send failure. try { $response = $adapter->send($data); + } catch (\Throwable $e) { + if ($hasRetriesLeft && $this->isRetryableError($e->getMessage())) { + $retry = $pending; + } else { + $this->recordError($errors, 'Failed sending to targets with error: ' . $e->getMessage()); + } + + $response = null; + } + + if ($response !== null) { $delivered += (int) $response['deliveredTo']; foreach ($response['results'] as $result) { @@ -604,18 +630,23 @@ class Messaging extends Action $error = $result['error'] ?? null; $recipient = $result['recipient']; - // Deleting push targets when token has expired. + // Best-effort: deleting push targets when the token has expired. Isolated so a transient DB + // error here never affects delivery accounting or the retry decision below. if ($error === 'Expired device token') { - $target = $dbForProject->findOne('targets', [ - Query::equal('identifier', [$recipient]) - ]); + try { + $target = $dbForProject->findOne('targets', [ + Query::equal('identifier', [$recipient]) + ]); - if (!$target->isEmpty()) { - $dbForProject->updateDocument( - 'targets', - $target->getId(), - $target->setAttribute('expired', true) - ); + if (!$target->isEmpty()) { + $dbForProject->updateDocument( + 'targets', + $target->getId(), + $target->setAttribute('expired', true) + ); + } + } catch (\Throwable) { + // Best-effort; must not affect accounting or retries. } } @@ -626,14 +657,6 @@ class Messaging extends Action $this->recordError($errors, "Failed sending to target {$recipient} with error: {$error}"); } - } catch (\Throwable $e) { - // Whole-batch failure: retry every still-pending recipient when the throw looks transient, - // otherwise record one representative terminal error covering them all. - if ($hasRetriesLeft && $this->isRetryableError($e->getMessage())) { - $retry = $pending; - } else { - $this->recordError($errors, 'Failed sending to targets with error: ' . $e->getMessage()); - } } $pending = $retry; diff --git a/tests/unit/Messaging/MessagingFanoutTest.php b/tests/unit/Messaging/MessagingFanoutTest.php index f2779b9a85..b7529433c5 100644 --- a/tests/unit/Messaging/MessagingFanoutTest.php +++ b/tests/unit/Messaging/MessagingFanoutTest.php @@ -328,6 +328,37 @@ class RecordingDatabase extends Database } } +/** + * {@see RecordingDatabase} variant whose expired-device-token cleanup always fails: `findOne('targets')` + * resolves to a real target so the worker attempts the update, and `updateDocument('targets')` throws a + * transient error. Used to prove that a DB hiccup during best-effort cleanup never leaks into delivery + * accounting or the retry decision. + */ +class ExpiredTokenCleanupFailingDatabase extends RecordingDatabase +{ + public int $targetUpdateAttempts = 0; + + public function findOne(string $collection, array $queries = []): Document + { + if ($collection === 'targets') { + return new Document(['$id' => 'target-expired', '$sequence' => '1']); + } + + return parent::findOne($collection, $queries); + } + + public function updateDocument(string $collection, string $id, Document $document): Document + { + if ($collection === 'targets') { + $this->targetUpdateAttempts++; + + throw new \RuntimeException('Transient database error during target cleanup'); + } + + return parent::updateDocument($collection, $id, $document); + } +} + class MessagingFanoutTest extends TestCase { private const RECIPIENT_COUNT = 4500; @@ -405,6 +436,7 @@ class MessagingFanoutTest extends TestCase ): void { $worker = new TestableMessaging($adapter); $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); + $method->setAccessible(true); $publisher = $this->createMock(UsagePublisher::class); $publisher->method('enqueue')->willReturn(true); @@ -543,6 +575,7 @@ class MessagingFanoutTest extends TestCase $worker = new TestableMessaging($adapter); $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); + $method->setAccessible(true); $publisher = $this->createMock(UsagePublisher::class); $device = new Local(\sys_get_temp_dir()); $project = new Document(['$id' => 'project1', '$sequence' => '1']); @@ -672,6 +705,46 @@ class MessagingFanoutTest extends TestCase $this->assertCount($count, $message->getAttribute('deliveryErrors')); } + public function testExpiredTokenCleanupFailureDoesNotCorruptAccounting(): void + { + $count = 3; + $data = $this->topicDataset($count); + // The cleanup path always throws, but the worker must isolate it: the send result is what's accounted. + $database = new ExpiredTokenCleanupFailingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + + // user1@ comes back with an expired device token (triggering cleanup, which then throws); the other + // two recipients are delivered. The expired-token error is terminal, never retryable. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => $recipient === 'user1@example.com' + ? 'Expired device token' + : null, + ]); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + + // The worker reached the cleanup (so the isolation is genuinely exercised) and it threw. + $this->assertGreaterThan(0, $database->targetUpdateAttempts, 'Expired-token cleanup must have been attempted'); + + // The send was attempted exactly once: a thrown cleanup must not look like a transient send failure + // and must not trigger a retry. + $this->assertSame(1, $adapter->sendCalls, 'Cleanup failure must not trigger a send retry'); + + // Accounting reflects the send result, not the cleanup throw: two delivered, one terminal failure. + $this->assertSame($count - 1, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::FAILED, $message->getAttribute('status')); + + // The recorded error is the provider's expired-token failure for the real recipient — proving the + // database exception was swallowed rather than misattributed as the send error. + $errors = $message->getAttribute('deliveryErrors'); + $this->assertCount(1, $errors); + $this->assertStringContainsString('user1@example.com', $errors[0]); + $this->assertStringContainsString('Expired device token', $errors[0]); + $this->assertStringNotContainsString('Transient database error', $errors[0]); + } + /** * @param array $queries * @return array diff --git a/tests/unit/Messaging/SwooleCoroutinePolyfill.php b/tests/unit/Messaging/SwooleCoroutinePolyfill.php index b509d84bec..fcf520a911 100644 --- a/tests/unit/Messaging/SwooleCoroutinePolyfill.php +++ b/tests/unit/Messaging/SwooleCoroutinePolyfill.php @@ -54,6 +54,14 @@ namespace Swoole\Coroutine { public function push(mixed $data, float $timeout = 0): bool { + // Enforce capacity so a full channel reports back-pressure exactly like a real Swoole channel. + // A full Semaphore permit channel then fails to acquire, which surfaces as a + // {@see \Utopia\Lock\Exception\Contention} that the batch() scheduler defers to a later tick — + // making the concurrency bound genuinely enforced rather than silently unbounded. + if (\count($this->buffer) >= $this->capacity) { + return false; + } + $this->buffer[] = $data; return true;