fix(messaging): address fanout scale review comments

Apply five reviewer fixes to the large-topic send path without touching
the adapter-agnostic messaging library:

- Empty recipient pages no longer count as recipients: a page that
  resolves to zero identifiers is skipped instead of running batch([])
  and masking the "No valid recipients found." outcome.
- Best-effort expired-device-token cleanup is isolated from the provider
  send: the send try/catch wraps only adapter->send(), results are
  processed outside it, and the findOne/updateDocument cleanup has its
  own try/catch so a transient DB error can never be misattributed as a
  send failure or trigger a spurious retry. Delivery accounting stays
  exact across attempts.
- resolveProvider caches the fallback provider under the requested id so
  a topic full of targets pointing at one disabled/missing provider does
  not re-query the providers collection once per page.
- The test Swoole channel polyfill enforces capacity (push returns false
  when full) so a saturated semaphore surfaces Contention and the
  scheduler defers the task, making the concurrency cap genuinely
  bounded under the polyfill.
- ReflectionMethod::setAccessible(true) added for explicitness/back-compat.

Adds a regression test proving a thrown expired-token cleanup is
swallowed and the real provider failure is recorded instead.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jake BarnbyandClaude Opus 4.8 committed 2026-06-03 04:13:49 +12:00
1 parent 2cd1abc273
commit 7c8d7dcb80
3 files changed
+115 -11

No files matched your search

+34 -11
View File
@@ -177,8 +177,6 @@ class Messaging extends Action
$hasRecipients = false; $hasRecipients = false;
foreach ($this->streamRecipients($dbForProject, $topicIds, $userIds, $targetIds, $providerType, $default) as $page) { foreach ($this->streamRecipients($dbForProject, $topicIds, $userIds, $targetIds, $providerType, $default) as $page) {
$hasRecipients = true;
/** /**
* @var array<callable> $tasks * @var array<callable> $tasks
*/ */
@@ -217,6 +215,14 @@ class Messaging extends Action
} }
} }
// A page can resolve to zero identifiers (e.g. subscribers whose targetInternalId matches no targets
// row), so an empty task list must not count as recipients or run batch() for nothing.
if (empty($tasks)) {
continue;
}
$hasRecipients = true;
/** /**
* @var array<array{delivered: int, recipients: int, errors: array<string>}> $results * @var array<array{delivered: int, recipients: int, errors: array<string>}> $results
*/ */
@@ -486,6 +492,10 @@ class Messaging extends Action
$provider = $dbForProject->getDocument('providers', $providerId); $provider = $dbForProject->getDocument('providers', $providerId);
if ($provider->isEmpty() || !$provider->getAttribute('enabled')) { if ($provider->isEmpty() || !$provider->getAttribute('enabled')) {
// Cache the fallback under this id too, so a topic full of targets pointing at the same
// disabled/missing provider does not re-query the providers collection once per page.
$providers[$providerId] = $default;
return $default; return $default;
} }
@@ -592,8 +602,24 @@ class Messaging extends Action
$retry = []; $retry = [];
// The try/catch wraps ONLY the provider send. A whole-batch throw is retryable when transient,
// otherwise it records one representative terminal error. The previous behaviour of resetting
// $delivered to 0 on a throw is gone — the retry refactor sums delivered across attempts, and the
// expired-device-token cleanup below is isolated in its own try so a DB hiccup there can never be
// misattributed as a send failure.
try { try {
$response = $adapter->send($data); $response = $adapter->send($data);
} catch (\Throwable $e) {
if ($hasRetriesLeft && $this->isRetryableError($e->getMessage())) {
$retry = $pending;
} else {
$this->recordError($errors, 'Failed sending to targets with error: ' . $e->getMessage());
}
$response = null;
}
if ($response !== null) {
$delivered += (int) $response['deliveredTo']; $delivered += (int) $response['deliveredTo'];
foreach ($response['results'] as $result) { foreach ($response['results'] as $result) {
@@ -604,8 +630,10 @@ class Messaging extends Action
$error = $result['error'] ?? null; $error = $result['error'] ?? null;
$recipient = $result['recipient']; $recipient = $result['recipient'];
// Deleting push targets when token has expired. // Best-effort: deleting push targets when the token has expired. Isolated so a transient DB
// error here never affects delivery accounting or the retry decision below.
if ($error === 'Expired device token') { if ($error === 'Expired device token') {
try {
$target = $dbForProject->findOne('targets', [ $target = $dbForProject->findOne('targets', [
Query::equal('identifier', [$recipient]) Query::equal('identifier', [$recipient])
]); ]);
@@ -617,6 +645,9 @@ class Messaging extends Action
$target->setAttribute('expired', true) $target->setAttribute('expired', true)
); );
} }
} catch (\Throwable) {
// Best-effort; must not affect accounting or retries.
}
} }
if ($hasRetriesLeft && $this->isRetryableError($error)) { if ($hasRetriesLeft && $this->isRetryableError($error)) {
@@ -626,14 +657,6 @@ class Messaging extends Action
$this->recordError($errors, "Failed sending to target {$recipient} with error: {$error}"); $this->recordError($errors, "Failed sending to target {$recipient} with error: {$error}");
} }
} catch (\Throwable $e) {
// Whole-batch failure: retry every still-pending recipient when the throw looks transient,
// otherwise record one representative terminal error covering them all.
if ($hasRetriesLeft && $this->isRetryableError($e->getMessage())) {
$retry = $pending;
} else {
$this->recordError($errors, 'Failed sending to targets with error: ' . $e->getMessage());
}
} }
$pending = $retry; $pending = $retry;
@@ -328,6 +328,37 @@ class RecordingDatabase extends Database
} }
} }
/**
* {@see RecordingDatabase} variant whose expired-device-token cleanup always fails: `findOne('targets')`
* resolves to a real target so the worker attempts the update, and `updateDocument('targets')` throws a
* transient error. Used to prove that a DB hiccup during best-effort cleanup never leaks into delivery
* accounting or the retry decision.
*/
class ExpiredTokenCleanupFailingDatabase extends RecordingDatabase
{
public int $targetUpdateAttempts = 0;
public function findOne(string $collection, array $queries = []): Document
{
if ($collection === 'targets') {
return new Document(['$id' => 'target-expired', '$sequence' => '1']);
}
return parent::findOne($collection, $queries);
}
public function updateDocument(string $collection, string $id, Document $document): Document
{
if ($collection === 'targets') {
$this->targetUpdateAttempts++;
throw new \RuntimeException('Transient database error during target cleanup');
}
return parent::updateDocument($collection, $id, $document);
}
}
class MessagingFanoutTest extends TestCase class MessagingFanoutTest extends TestCase
{ {
private const RECIPIENT_COUNT = 4500; private const RECIPIENT_COUNT = 4500;
@@ -405,6 +436,7 @@ class MessagingFanoutTest extends TestCase
): void { ): void {
$worker = new TestableMessaging($adapter); $worker = new TestableMessaging($adapter);
$method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage');
$method->setAccessible(true);
$publisher = $this->createMock(UsagePublisher::class); $publisher = $this->createMock(UsagePublisher::class);
$publisher->method('enqueue')->willReturn(true); $publisher->method('enqueue')->willReturn(true);
@@ -543,6 +575,7 @@ class MessagingFanoutTest extends TestCase
$worker = new TestableMessaging($adapter); $worker = new TestableMessaging($adapter);
$method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage');
$method->setAccessible(true);
$publisher = $this->createMock(UsagePublisher::class); $publisher = $this->createMock(UsagePublisher::class);
$device = new Local(\sys_get_temp_dir()); $device = new Local(\sys_get_temp_dir());
$project = new Document(['$id' => 'project1', '$sequence' => '1']); $project = new Document(['$id' => 'project1', '$sequence' => '1']);
@@ -672,6 +705,46 @@ class MessagingFanoutTest extends TestCase
$this->assertCount($count, $message->getAttribute('deliveryErrors')); $this->assertCount($count, $message->getAttribute('deliveryErrors'));
} }
public function testExpiredTokenCleanupFailureDoesNotCorruptAccounting(): void
{
$count = 3;
$data = $this->topicDataset($count);
// The cleanup path always throws, but the worker must isolate it: the send result is what's accounted.
$database = new ExpiredTokenCleanupFailingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
// user1@ comes back with an expired device token (triggering cleanup, which then throws); the other
// two recipients are delivered. The expired-token error is terminal, never retryable.
$adapter = new ScriptedEmailAdapter([
static fn (string $recipient): ?string => $recipient === 'user1@example.com'
? 'Expired device token'
: null,
]);
$this->sendTopic($database, $adapter);
$message = $database->updatedMessage;
$this->assertNotNull($message);
// The worker reached the cleanup (so the isolation is genuinely exercised) and it threw.
$this->assertGreaterThan(0, $database->targetUpdateAttempts, 'Expired-token cleanup must have been attempted');
// The send was attempted exactly once: a thrown cleanup must not look like a transient send failure
// and must not trigger a retry.
$this->assertSame(1, $adapter->sendCalls, 'Cleanup failure must not trigger a send retry');
// Accounting reflects the send result, not the cleanup throw: two delivered, one terminal failure.
$this->assertSame($count - 1, $message->getAttribute('deliveredTotal'));
$this->assertSame(MessageStatus::FAILED, $message->getAttribute('status'));
// The recorded error is the provider's expired-token failure for the real recipient — proving the
// database exception was swallowed rather than misattributed as the send error.
$errors = $message->getAttribute('deliveryErrors');
$this->assertCount(1, $errors);
$this->assertStringContainsString('user1@example.com', $errors[0]);
$this->assertStringContainsString('Expired device token', $errors[0]);
$this->assertStringNotContainsString('Transient database error', $errors[0]);
}
/** /**
* @param array<Query> $queries * @param array<Query> $queries
* @return array<string> * @return array<string>
@@ -54,6 +54,14 @@ namespace Swoole\Coroutine {
public function push(mixed $data, float $timeout = 0): bool public function push(mixed $data, float $timeout = 0): bool
{ {
// Enforce capacity so a full channel reports back-pressure exactly like a real Swoole channel.
// A full Semaphore permit channel then fails to acquire, which surfaces as a
// {@see \Utopia\Lock\Exception\Contention} that the batch() scheduler defers to a later tick —
// making the concurrency bound genuinely enforced rather than silently unbounded.
if (\count($this->buffer) >= $this->capacity) {
return false;
}
$this->buffer[] = $data; $this->buffer[] = $data;
return true; return true;