feat(messaging): retry throttled recipients with backoff in send path

Provider rate limiting and transient errors previously failed a whole
batch outright. Wrap the adapter send in a backoff/retry loop that
partitions per-recipient results: delivered recipients are summed and
never retried, retryable failures (throttle/rate-limit/quota/503/timeout
phrasing) are re-sent with only the still-failing identifiers in `to`,
and permanent failures stay terminal. Exponential backoff with jitter
runs on the non-blocking Swoole coroutine sleep so sibling sends keep
progressing; Retry-After is intentionally not honored to keep the
messaging library adapter-agnostic.

Accounting stays exact: recipients is the original batch size and
delivered is summed across attempts without double-counting.
This commit is contained in:
Jake Barnby committed 2026-06-03 03:57:39 +12:00
1 parent da061ba7d2
commit 2cd1abc273
3 files changed
+409 -58

No files matched your search

+2
View File
@@ -246,6 +246,8 @@ const MESSAGE_SEND_TYPE_EXTERNAL = 'external';
const MESSAGE_RECIPIENTS_PAGE_SIZE = 1000; // Recipients resolved from the database per page while streaming a send
const MESSAGE_SEND_CONCURRENCY = 10; // Maximum adapter send requests running concurrently within a single send job
const MESSAGE_DELIVERY_ERRORS_LIMIT = 100; // Maximum number of per-recipient delivery errors retained on a message
const MESSAGE_SEND_MAX_RETRIES = 5; // Maximum number of attempts to deliver a batch when the provider throttles or fails transiently
const MESSAGE_SEND_RETRY_DELAY = 1.0; // Base seconds for exponential backoff between batch send retries; overridable in tests for an instant suite
// Mail Types
const MAIL_TYPE_VERIFICATION = 'verification';
const MAIL_TYPE_MAGIC_SESSION = 'magicSession';
+217 -57
View File
@@ -497,6 +497,11 @@ class Messaging extends Action
/**
* Send a single adapter-sized batch and report delivery counts plus a bounded error list.
*
* Wraps the provider call in a backoff/retry loop that reacts to provider rate limiting and transient
* failures (see {@see retrySend()}). Accounting stays exact: only still-failing recipients are ever
* retried, so `delivered` is summed across attempts without double-counting, and `recipients` always
* reports the original batch size so the caller's `failed = recipients - delivered` holds.
*
* @param array<string> $batch
* @return array{delivered: int, recipients: int, errors: array<string>}
*/
@@ -512,66 +517,29 @@ class Messaging extends Action
UsagePublisher $publisherForUsage
): array {
$recipients = \count($batch);
$delivered = 0;
$errors = [];
$messageData = clone $message;
$messageData->setAttribute('to', $batch);
[
'delivered' => $delivered,
'errors' => $errors,
] = $this->retrySend($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project);
$data = match ($providerType) {
MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider),
MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData),
MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project),
default => throw new \Exception('Provider with the requested ID is of the incorrect type')
};
$failed = $recipients - $delivered;
$usage = new UsageContext();
$usage
->addMetric(METRIC_MESSAGES, $recipients)
->addMetric(METRIC_MESSAGES_SENT, $delivered)
->addMetric(METRIC_MESSAGES_FAILED, $failed)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed);
try {
$response = $adapter->send($data);
$delivered = (int) $response['deliveredTo'];
foreach ($response['results'] as $result) {
if ($result['status'] === 'failure' && \count($errors) < MESSAGE_DELIVERY_ERRORS_LIMIT) {
$errors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}";
}
// Deleting push targets when token has expired.
if (($result['error'] ?? '') === 'Expired device token') {
$target = $dbForProject->findOne('targets', [
Query::equal('identifier', [$result['recipient']])
]);
if (!$target->isEmpty()) {
$dbForProject->updateDocument(
'targets',
$target->getId(),
$target->setAttribute('expired', true)
);
}
}
}
} catch (\Throwable $e) {
// Whole-batch failure: record one representative error and count every recipient as failed.
$delivered = 0;
$errors = ['Failed sending to targets with error: ' . $e->getMessage()];
} finally {
$failed = $recipients - $delivered;
$usage = new UsageContext();
$usage
->addMetric(METRIC_MESSAGES, $recipients)
->addMetric(METRIC_MESSAGES_SENT, $delivered)
->addMetric(METRIC_MESSAGES_FAILED, $failed)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed);
$publisherForUsage->enqueue(new Usage(
project: $project,
metrics: $usage->getMetrics(),
));
}
$publisherForUsage->enqueue(new Usage(
project: $project,
metrics: $usage->getMetrics(),
));
return [
'delivered' => $delivered,
@@ -580,6 +548,198 @@ class Messaging extends Action
];
}
/**
* Drive a batch through provider sends with exponential backoff, reacting to throttling and transient
* errors by retrying only the recipients that are still failing.
*
* Partitioning per attempt: the provider's per-recipient results are split into delivered (summed into
* the running total, never retried) and failures. A failure whose error matches {@see isRetryableError()}
* stays pending for the next attempt; any other failure becomes a terminal error immediately. When
* `send()` throws, the throw is retryable only if its message matches; otherwise every still-pending
* recipient is recorded as terminal. The next attempt rebuilds the provider message with only the pending
* recipients in `to`, so an already-delivered recipient is never re-sent. After {@see MESSAGE_SEND_MAX_RETRIES}
* attempts, any recipients still pending are flushed to terminal errors.
*
* Note: provider `Retry-After` hints are not honored — the agnostic adapter {@see \Utopia\Messaging\Response}
* exposes only generic per-recipient strings, never a structured retry delay, so we keep the library
* adapter-agnostic and rely on exponential backoff alone.
*
* @param array<string> $batch
* @return array{delivered: int, errors: array<string>}
*/
private function retrySend(
array $batch,
Document $message,
Document $provider,
string $providerType,
EmailAdapter|SMSAdapter|PushAdapter $adapter,
Database $dbForProject,
Device $deviceForFiles,
Document $project
): array {
$delivered = 0;
$errors = [];
// Recipients still awaiting a successful (or terminal) outcome; shrinks to only the failing ones each attempt.
$pending = $batch;
for ($attempt = 1; $attempt <= MESSAGE_SEND_MAX_RETRIES; $attempt++) {
$hasRetriesLeft = $attempt < MESSAGE_SEND_MAX_RETRIES;
// Rebuild the provider message scoped to only the still-pending recipients so a partially-delivered
// batch never re-sends to recipients that already succeeded on an earlier attempt.
$data = $this->buildMessage($pending, $message, $provider, $providerType, $dbForProject, $deviceForFiles, $project);
$retry = [];
try {
$response = $adapter->send($data);
$delivered += (int) $response['deliveredTo'];
foreach ($response['results'] as $result) {
if ($result['status'] !== 'failure') {
continue;
}
$error = $result['error'] ?? null;
$recipient = $result['recipient'];
// Deleting push targets when token has expired.
if ($error === 'Expired device token') {
$target = $dbForProject->findOne('targets', [
Query::equal('identifier', [$recipient])
]);
if (!$target->isEmpty()) {
$dbForProject->updateDocument(
'targets',
$target->getId(),
$target->setAttribute('expired', true)
);
}
}
if ($hasRetriesLeft && $this->isRetryableError($error)) {
$retry[] = $recipient;
continue;
}
$this->recordError($errors, "Failed sending to target {$recipient} with error: {$error}");
}
} catch (\Throwable $e) {
// Whole-batch failure: retry every still-pending recipient when the throw looks transient,
// otherwise record one representative terminal error covering them all.
if ($hasRetriesLeft && $this->isRetryableError($e->getMessage())) {
$retry = $pending;
} else {
$this->recordError($errors, 'Failed sending to targets with error: ' . $e->getMessage());
}
}
$pending = $retry;
if (empty($pending)) {
break;
}
// Exponential backoff with jitter; non-blocking under Swoole so sibling sends keep progressing.
$delay = $this->retryDelay() * (2 ** ($attempt - 1));
$delay += $delay * (\random_int(0, 100) / 1000);
\Swoole\Coroutine::sleep($delay);
}
return [
'delivered' => $delivered,
'errors' => $errors,
];
}
/**
* Append a delivery error while keeping the retained list bounded by {@see MESSAGE_DELIVERY_ERRORS_LIMIT}.
*
* @param array<string> $errors
*/
private function recordError(array &$errors, string $error): void
{
if (\count($errors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) {
return;
}
$errors[] = $error;
}
/**
* Conservatively classify a provider error string as retryable. The agnostic adapter
* {@see \Utopia\Messaging\Response} returns only free-form error strings, so string matching is the only
* provider-agnostic signal available. The pattern list is intentionally narrow — throttling, rate limits,
* quota, service-unavailable and timeout phrasing — so permanent failures (e.g. invalid recipients) are
* never retried.
*/
private function isRetryableError(?string $error): bool
{
if ($error === null || $error === '') {
return false;
}
$patterns = [
'throttl',
'rate exceeded',
'rate limit',
'too many requests',
'429',
'quota',
'service unavailable',
'503',
'timed out',
'timeout',
'temporarily',
];
$needle = \strtolower($error);
foreach ($patterns as $pattern) {
if (\str_contains($needle, $pattern)) {
return true;
}
}
return false;
}
/**
* Base seconds for exponential backoff between send retries. Isolated so tests can override it to keep the
* suite instant without touching the production constant.
*/
protected function retryDelay(): float
{
return MESSAGE_SEND_RETRY_DELAY;
}
/**
* Build the provider-specific message for a set of recipients.
*
* @param array<string> $to
*/
private function buildMessage(
array $to,
Document $message,
Document $provider,
string $providerType,
Database $dbForProject,
Device $deviceForFiles,
Document $project
): Email|SMS|Push {
$messageData = clone $message;
$messageData->setAttribute('to', $to);
return match ($providerType) {
MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider),
MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData),
MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project),
default => throw new \Exception('Provider with the requested ID is of the incorrect type')
};
}
private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void
{
if ($this->adapter === null) {
+190 -1
View File
@@ -91,6 +91,7 @@ class ConcurrencyProbeAdapter extends EmailAdapter
/**
* Worker subclass that injects the instrumented email adapter instead of a network-backed provider adapter.
* Retry backoff is zeroed so the throttle/retry tests stay instant.
*/
class TestableMessaging extends Messaging
{
@@ -102,6 +103,80 @@ class TestableMessaging extends Messaging
{
return $this->emailAdapter;
}
protected function retryDelay(): float
{
return 0.0;
}
}
/**
* Email adapter double whose per-attempt outcome is scripted. Each {@see process()} call records the exact
* `to` recipients it received and consults the next scripted step, which either throws (simulating a
* connection-level provider error) or returns a per-recipient success/failure verdict. When the script runs
* out, every recipient succeeds — modelling a provider that has stopped throttling.
*/
class ScriptedEmailAdapter extends EmailAdapter
{
public int $sendCalls = 0;
/**
* @var array<array<string>> Recipient email lists captured per {@see process()} call, in call order.
*/
public array $recipientsPerCall = [];
/**
* @param array<\Throwable|(callable(string): ?string)> $script One step per attempt. A Throwable is thrown
* for that attempt; a callable returns an error string for a failing recipient or null for success.
*/
public function __construct(private readonly array $script = [])
{
}
public function getName(): string
{
return 'ScriptedEmail';
}
public function getMaxMessagesPerRequest(): int
{
return 1000;
}
/**
* @param EmailMessage $message
* @return array{deliveredTo: int, type: string, results: array<array<string, mixed>>}
*/
protected function process(EmailMessage $message): array
{
$recipients = \array_map(static fn (array $entry): string => $entry['email'], $message->getTo());
$this->recipientsPerCall[] = $recipients;
$step = $this->script[$this->sendCalls] ?? null;
$this->sendCalls++;
if ($step instanceof \Throwable) {
throw $step;
}
$response = new Response($this->getType());
$delivered = 0;
foreach ($recipients as $recipient) {
$error = \is_callable($step) ? $step($recipient) : null;
if ($error === null) {
$delivered++;
$response->addResult($recipient);
} else {
$response->addResult($recipient, $error);
}
}
$response->setDeliveredTo($delivered);
return $response->toArray();
}
}
/**
@@ -326,7 +401,7 @@ class MessagingFanoutTest extends TestCase
private function sendTopic(
RecordingDatabase $database,
ConcurrencyProbeAdapter $adapter
EmailAdapter $adapter
): void {
$worker = new TestableMessaging($adapter);
$method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage');
@@ -483,6 +558,120 @@ class MessagingFanoutTest extends TestCase
$this->assertNull($database->updatedMessage, 'Terminal messages must not be rewritten');
}
public function testThrottledBatchIsRetriedThenFullyDelivered(): void
{
$count = 5;
$data = $this->topicDataset($count);
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
// First attempt throttles every recipient; the script then runs out so the second attempt succeeds.
$adapter = new ScriptedEmailAdapter([
static fn (string $recipient): string => 'Rate limit exceeded, please retry',
]);
$this->sendTopic($database, $adapter);
$message = $database->updatedMessage;
$this->assertNotNull($message);
// The batch was retried exactly once and then fully delivered with no double-counting.
$this->assertSame(2, $adapter->sendCalls, 'Throttled batch must be retried once');
$this->assertSame($count, $message->getAttribute('deliveredTotal'));
$this->assertSame(MessageStatus::SENT, $message->getAttribute('status'));
$this->assertEmpty($message->getAttribute('deliveryErrors'));
// Both attempts targeted the full recipient set (none had succeeded on the first attempt).
$this->assertCount($count, $adapter->recipientsPerCall[0]);
$this->assertCount($count, $adapter->recipientsPerCall[1]);
}
public function testRetryTargetsOnlyThrottledRecipients(): void
{
$count = 5;
$data = $this->topicDataset($count);
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
// user2@ and user4@ throttle on the first attempt; everyone else succeeds. The second attempt has no
// script step, so the two retried recipients then succeed.
$throttled = ['user2@example.com', 'user4@example.com'];
$adapter = new ScriptedEmailAdapter([
static fn (string $recipient): ?string => \in_array($recipient, $throttled, true)
? 'Too Many Requests'
: null,
]);
$this->sendTopic($database, $adapter);
$message = $database->updatedMessage;
$this->assertNotNull($message);
$this->assertSame(2, $adapter->sendCalls, 'A partially throttled batch must be retried once');
// The retry must re-send to ONLY the throttled recipients — never the ones already delivered.
\sort($adapter->recipientsPerCall[1]);
$this->assertSame($throttled, $adapter->recipientsPerCall[1]);
// Every recipient is ultimately delivered and counted exactly once.
$this->assertSame($count, $message->getAttribute('deliveredTotal'));
$this->assertSame(MessageStatus::SENT, $message->getAttribute('status'));
$this->assertEmpty($message->getAttribute('deliveryErrors'));
}
public function testNonRetryableFailureIsNotRetried(): void
{
$count = 3;
$data = $this->topicDataset($count);
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
// user1@ permanently fails with a non-retryable error; the rest succeed.
$adapter = new ScriptedEmailAdapter([
static fn (string $recipient): ?string => $recipient === 'user1@example.com'
? 'Invalid recipient address'
: null,
]);
$this->sendTopic($database, $adapter);
$message = $database->updatedMessage;
$this->assertNotNull($message);
// A permanent failure is recorded immediately, never retried.
$this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry');
$this->assertSame($count - 1, $message->getAttribute('deliveredTotal'));
$this->assertSame(MessageStatus::FAILED, $message->getAttribute('status'));
$errors = $message->getAttribute('deliveryErrors');
$this->assertCount(1, $errors);
$this->assertStringContainsString('user1@example.com', $errors[0]);
$this->assertStringContainsString('Invalid recipient address', $errors[0]);
}
public function testExhaustedRetriesRecordTerminalFailure(): void
{
$count = 2;
$data = $this->topicDataset($count);
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
// Every attempt throttles, for more steps than MESSAGE_SEND_MAX_RETRIES, so retries are exhausted.
$script = \array_fill(
0,
MESSAGE_SEND_MAX_RETRIES + 2,
static fn (string $recipient): string => 'Service unavailable, throttled'
);
$adapter = new ScriptedEmailAdapter($script);
$this->sendTopic($database, $adapter);
$message = $database->updatedMessage;
$this->assertNotNull($message);
// The batch is attempted exactly MESSAGE_SEND_MAX_RETRIES times, then the still-throttled recipients
// become terminal failures with nothing delivered.
$this->assertSame(MESSAGE_SEND_MAX_RETRIES, $adapter->sendCalls, 'Attempts must be capped at MESSAGE_SEND_MAX_RETRIES');
$this->assertSame(0, $message->getAttribute('deliveredTotal'));
$this->assertSame(MessageStatus::FAILED, $message->getAttribute('status'));
$this->assertCount($count, $message->getAttribute('deliveryErrors'));
}
/**
* @param array<Query> $queries
* @return array<string>