From 2cd1abc2732c4b3d8246426abbd4333ee079e406 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Jun 2026 03:57:39 +1200 Subject: [PATCH] feat(messaging): retry throttled recipients with backoff in send path Provider rate limiting and transient errors previously failed a whole batch outright. Wrap the adapter send in a backoff/retry loop that partitions per-recipient results: delivered recipients are summed and never retried, retryable failures (throttle/rate-limit/quota/503/timeout phrasing) are re-sent with only the still-failing identifiers in `to`, and permanent failures stay terminal. Exponential backoff with jitter runs on the non-blocking Swoole coroutine sleep so sibling sends keep progressing; Retry-After is intentionally not honored to keep the messaging library adapter-agnostic. Accounting stays exact: recipients is the original batch size and delivered is summed across attempts without double-counting. --- app/init/constants.php | 2 + src/Appwrite/Platform/Workers/Messaging.php | 274 +++++++++++++++---- tests/unit/Messaging/MessagingFanoutTest.php | 191 ++++++++++++- 3 files changed, 409 insertions(+), 58 deletions(-) diff --git a/app/init/constants.php b/app/init/constants.php index 2aa0d347f1..2b72f8814f 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -246,6 +246,8 @@ const MESSAGE_SEND_TYPE_EXTERNAL = 'external'; const MESSAGE_RECIPIENTS_PAGE_SIZE = 1000; // Recipients resolved from the database per page while streaming a send const MESSAGE_SEND_CONCURRENCY = 10; // Maximum adapter send requests running concurrently within a single send job const MESSAGE_DELIVERY_ERRORS_LIMIT = 100; // Maximum number of per-recipient delivery errors retained on a message +const MESSAGE_SEND_MAX_RETRIES = 5; // Maximum number of attempts to deliver a batch when the provider throttles or fails transiently +const MESSAGE_SEND_RETRY_DELAY = 1.0; // Base seconds for exponential backoff between batch send retries; overridable in tests for an instant suite // Mail Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index a80110681b..b239a69c75 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -497,6 +497,11 @@ class Messaging extends Action /** * Send a single adapter-sized batch and report delivery counts plus a bounded error list. * + * Wraps the provider call in a backoff/retry loop that reacts to provider rate limiting and transient + * failures (see {@see retrySend()}). Accounting stays exact: only still-failing recipients are ever + * retried, so `delivered` is summed across attempts without double-counting, and `recipients` always + * reports the original batch size so the caller's `failed = recipients - delivered` holds. + * * @param array $batch * @return array{delivered: int, recipients: int, errors: array} */ @@ -512,66 +517,29 @@ class Messaging extends Action UsagePublisher $publisherForUsage ): array { $recipients = \count($batch); - $delivered = 0; - $errors = []; - $messageData = clone $message; - $messageData->setAttribute('to', $batch); + [ + 'delivered' => $delivered, + 'errors' => $errors, + ] = $this->retrySend($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project); - $data = match ($providerType) { - MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider), - MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData), - MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project), - default => throw new \Exception('Provider with the requested ID is of the incorrect type') - }; + $failed = $recipients - $delivered; + $usage = new UsageContext(); + $usage + ->addMetric(METRIC_MESSAGES, $recipients) + ->addMetric(METRIC_MESSAGES_SENT, $delivered) + ->addMetric(METRIC_MESSAGES_FAILED, $failed) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed); - try { - $response = $adapter->send($data); - $delivered = (int) $response['deliveredTo']; - - foreach ($response['results'] as $result) { - if ($result['status'] === 'failure' && \count($errors) < MESSAGE_DELIVERY_ERRORS_LIMIT) { - $errors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}"; - } - - // Deleting push targets when token has expired. - if (($result['error'] ?? '') === 'Expired device token') { - $target = $dbForProject->findOne('targets', [ - Query::equal('identifier', [$result['recipient']]) - ]); - - if (!$target->isEmpty()) { - $dbForProject->updateDocument( - 'targets', - $target->getId(), - $target->setAttribute('expired', true) - ); - } - } - } - } catch (\Throwable $e) { - // Whole-batch failure: record one representative error and count every recipient as failed. - $delivered = 0; - $errors = ['Failed sending to targets with error: ' . $e->getMessage()]; - } finally { - $failed = $recipients - $delivered; - $usage = new UsageContext(); - $usage - ->addMetric(METRIC_MESSAGES, $recipients) - ->addMetric(METRIC_MESSAGES_SENT, $delivered) - ->addMetric(METRIC_MESSAGES_FAILED, $failed) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed); - - $publisherForUsage->enqueue(new Usage( - project: $project, - metrics: $usage->getMetrics(), - )); - } + $publisherForUsage->enqueue(new Usage( + project: $project, + metrics: $usage->getMetrics(), + )); return [ 'delivered' => $delivered, @@ -580,6 +548,198 @@ class Messaging extends Action ]; } + /** + * Drive a batch through provider sends with exponential backoff, reacting to throttling and transient + * errors by retrying only the recipients that are still failing. + * + * Partitioning per attempt: the provider's per-recipient results are split into delivered (summed into + * the running total, never retried) and failures. A failure whose error matches {@see isRetryableError()} + * stays pending for the next attempt; any other failure becomes a terminal error immediately. When + * `send()` throws, the throw is retryable only if its message matches; otherwise every still-pending + * recipient is recorded as terminal. The next attempt rebuilds the provider message with only the pending + * recipients in `to`, so an already-delivered recipient is never re-sent. After {@see MESSAGE_SEND_MAX_RETRIES} + * attempts, any recipients still pending are flushed to terminal errors. + * + * Note: provider `Retry-After` hints are not honored — the agnostic adapter {@see \Utopia\Messaging\Response} + * exposes only generic per-recipient strings, never a structured retry delay, so we keep the library + * adapter-agnostic and rely on exponential backoff alone. + * + * @param array $batch + * @return array{delivered: int, errors: array} + */ + private function retrySend( + array $batch, + Document $message, + Document $provider, + string $providerType, + EmailAdapter|SMSAdapter|PushAdapter $adapter, + Database $dbForProject, + Device $deviceForFiles, + Document $project + ): array { + $delivered = 0; + $errors = []; + + // Recipients still awaiting a successful (or terminal) outcome; shrinks to only the failing ones each attempt. + $pending = $batch; + + for ($attempt = 1; $attempt <= MESSAGE_SEND_MAX_RETRIES; $attempt++) { + $hasRetriesLeft = $attempt < MESSAGE_SEND_MAX_RETRIES; + + // Rebuild the provider message scoped to only the still-pending recipients so a partially-delivered + // batch never re-sends to recipients that already succeeded on an earlier attempt. + $data = $this->buildMessage($pending, $message, $provider, $providerType, $dbForProject, $deviceForFiles, $project); + + $retry = []; + + try { + $response = $adapter->send($data); + $delivered += (int) $response['deliveredTo']; + + foreach ($response['results'] as $result) { + if ($result['status'] !== 'failure') { + continue; + } + + $error = $result['error'] ?? null; + $recipient = $result['recipient']; + + // Deleting push targets when token has expired. + if ($error === 'Expired device token') { + $target = $dbForProject->findOne('targets', [ + Query::equal('identifier', [$recipient]) + ]); + + if (!$target->isEmpty()) { + $dbForProject->updateDocument( + 'targets', + $target->getId(), + $target->setAttribute('expired', true) + ); + } + } + + if ($hasRetriesLeft && $this->isRetryableError($error)) { + $retry[] = $recipient; + continue; + } + + $this->recordError($errors, "Failed sending to target {$recipient} with error: {$error}"); + } + } catch (\Throwable $e) { + // Whole-batch failure: retry every still-pending recipient when the throw looks transient, + // otherwise record one representative terminal error covering them all. + if ($hasRetriesLeft && $this->isRetryableError($e->getMessage())) { + $retry = $pending; + } else { + $this->recordError($errors, 'Failed sending to targets with error: ' . $e->getMessage()); + } + } + + $pending = $retry; + + if (empty($pending)) { + break; + } + + // Exponential backoff with jitter; non-blocking under Swoole so sibling sends keep progressing. + $delay = $this->retryDelay() * (2 ** ($attempt - 1)); + $delay += $delay * (\random_int(0, 100) / 1000); + \Swoole\Coroutine::sleep($delay); + } + + return [ + 'delivered' => $delivered, + 'errors' => $errors, + ]; + } + + /** + * Append a delivery error while keeping the retained list bounded by {@see MESSAGE_DELIVERY_ERRORS_LIMIT}. + * + * @param array $errors + */ + private function recordError(array &$errors, string $error): void + { + if (\count($errors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) { + return; + } + + $errors[] = $error; + } + + /** + * Conservatively classify a provider error string as retryable. The agnostic adapter + * {@see \Utopia\Messaging\Response} returns only free-form error strings, so string matching is the only + * provider-agnostic signal available. The pattern list is intentionally narrow — throttling, rate limits, + * quota, service-unavailable and timeout phrasing — so permanent failures (e.g. invalid recipients) are + * never retried. + */ + private function isRetryableError(?string $error): bool + { + if ($error === null || $error === '') { + return false; + } + + $patterns = [ + 'throttl', + 'rate exceeded', + 'rate limit', + 'too many requests', + '429', + 'quota', + 'service unavailable', + '503', + 'timed out', + 'timeout', + 'temporarily', + ]; + + $needle = \strtolower($error); + + foreach ($patterns as $pattern) { + if (\str_contains($needle, $pattern)) { + return true; + } + } + + return false; + } + + /** + * Base seconds for exponential backoff between send retries. Isolated so tests can override it to keep the + * suite instant without touching the production constant. + */ + protected function retryDelay(): float + { + return MESSAGE_SEND_RETRY_DELAY; + } + + /** + * Build the provider-specific message for a set of recipients. + * + * @param array $to + */ + private function buildMessage( + array $to, + Document $message, + Document $provider, + string $providerType, + Database $dbForProject, + Device $deviceForFiles, + Document $project + ): Email|SMS|Push { + $messageData = clone $message; + $messageData->setAttribute('to', $to); + + return match ($providerType) { + MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider), + MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData), + MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project), + default => throw new \Exception('Provider with the requested ID is of the incorrect type') + }; + } + private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void { if ($this->adapter === null) { diff --git a/tests/unit/Messaging/MessagingFanoutTest.php b/tests/unit/Messaging/MessagingFanoutTest.php index 024cdbf43e..f2779b9a85 100644 --- a/tests/unit/Messaging/MessagingFanoutTest.php +++ b/tests/unit/Messaging/MessagingFanoutTest.php @@ -91,6 +91,7 @@ class ConcurrencyProbeAdapter extends EmailAdapter /** * Worker subclass that injects the instrumented email adapter instead of a network-backed provider adapter. + * Retry backoff is zeroed so the throttle/retry tests stay instant. */ class TestableMessaging extends Messaging { @@ -102,6 +103,80 @@ class TestableMessaging extends Messaging { return $this->emailAdapter; } + + protected function retryDelay(): float + { + return 0.0; + } +} + +/** + * Email adapter double whose per-attempt outcome is scripted. Each {@see process()} call records the exact + * `to` recipients it received and consults the next scripted step, which either throws (simulating a + * connection-level provider error) or returns a per-recipient success/failure verdict. When the script runs + * out, every recipient succeeds — modelling a provider that has stopped throttling. + */ +class ScriptedEmailAdapter extends EmailAdapter +{ + public int $sendCalls = 0; + + /** + * @var array> Recipient email lists captured per {@see process()} call, in call order. + */ + public array $recipientsPerCall = []; + + /** + * @param array<\Throwable|(callable(string): ?string)> $script One step per attempt. A Throwable is thrown + * for that attempt; a callable returns an error string for a failing recipient or null for success. + */ + public function __construct(private readonly array $script = []) + { + } + + public function getName(): string + { + return 'ScriptedEmail'; + } + + public function getMaxMessagesPerRequest(): int + { + return 1000; + } + + /** + * @param EmailMessage $message + * @return array{deliveredTo: int, type: string, results: array>} + */ + protected function process(EmailMessage $message): array + { + $recipients = \array_map(static fn (array $entry): string => $entry['email'], $message->getTo()); + $this->recipientsPerCall[] = $recipients; + + $step = $this->script[$this->sendCalls] ?? null; + $this->sendCalls++; + + if ($step instanceof \Throwable) { + throw $step; + } + + $response = new Response($this->getType()); + $delivered = 0; + + foreach ($recipients as $recipient) { + $error = \is_callable($step) ? $step($recipient) : null; + + if ($error === null) { + $delivered++; + $response->addResult($recipient); + } else { + $response->addResult($recipient, $error); + } + } + + $response->setDeliveredTo($delivered); + + return $response->toArray(); + } } /** @@ -326,7 +401,7 @@ class MessagingFanoutTest extends TestCase private function sendTopic( RecordingDatabase $database, - ConcurrencyProbeAdapter $adapter + EmailAdapter $adapter ): void { $worker = new TestableMessaging($adapter); $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); @@ -483,6 +558,120 @@ class MessagingFanoutTest extends TestCase $this->assertNull($database->updatedMessage, 'Terminal messages must not be rewritten'); } + public function testThrottledBatchIsRetriedThenFullyDelivered(): void + { + $count = 5; + $data = $this->topicDataset($count); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + + // First attempt throttles every recipient; the script then runs out so the second attempt succeeds. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): string => 'Rate limit exceeded, please retry', + ]); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + + // The batch was retried exactly once and then fully delivered with no double-counting. + $this->assertSame(2, $adapter->sendCalls, 'Throttled batch must be retried once'); + $this->assertSame($count, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::SENT, $message->getAttribute('status')); + $this->assertEmpty($message->getAttribute('deliveryErrors')); + + // Both attempts targeted the full recipient set (none had succeeded on the first attempt). + $this->assertCount($count, $adapter->recipientsPerCall[0]); + $this->assertCount($count, $adapter->recipientsPerCall[1]); + } + + public function testRetryTargetsOnlyThrottledRecipients(): void + { + $count = 5; + $data = $this->topicDataset($count); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + + // user2@ and user4@ throttle on the first attempt; everyone else succeeds. The second attempt has no + // script step, so the two retried recipients then succeed. + $throttled = ['user2@example.com', 'user4@example.com']; + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => \in_array($recipient, $throttled, true) + ? 'Too Many Requests' + : null, + ]); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + + $this->assertSame(2, $adapter->sendCalls, 'A partially throttled batch must be retried once'); + + // The retry must re-send to ONLY the throttled recipients — never the ones already delivered. + \sort($adapter->recipientsPerCall[1]); + $this->assertSame($throttled, $adapter->recipientsPerCall[1]); + + // Every recipient is ultimately delivered and counted exactly once. + $this->assertSame($count, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::SENT, $message->getAttribute('status')); + $this->assertEmpty($message->getAttribute('deliveryErrors')); + } + + public function testNonRetryableFailureIsNotRetried(): void + { + $count = 3; + $data = $this->topicDataset($count); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + + // user1@ permanently fails with a non-retryable error; the rest succeed. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => $recipient === 'user1@example.com' + ? 'Invalid recipient address' + : null, + ]); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + + // A permanent failure is recorded immediately, never retried. + $this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry'); + $this->assertSame($count - 1, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::FAILED, $message->getAttribute('status')); + $errors = $message->getAttribute('deliveryErrors'); + $this->assertCount(1, $errors); + $this->assertStringContainsString('user1@example.com', $errors[0]); + $this->assertStringContainsString('Invalid recipient address', $errors[0]); + } + + public function testExhaustedRetriesRecordTerminalFailure(): void + { + $count = 2; + $data = $this->topicDataset($count); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + + // Every attempt throttles, for more steps than MESSAGE_SEND_MAX_RETRIES, so retries are exhausted. + $script = \array_fill( + 0, + MESSAGE_SEND_MAX_RETRIES + 2, + static fn (string $recipient): string => 'Service unavailable, throttled' + ); + $adapter = new ScriptedEmailAdapter($script); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + + // The batch is attempted exactly MESSAGE_SEND_MAX_RETRIES times, then the still-throttled recipients + // become terminal failures with nothing delivered. + $this->assertSame(MESSAGE_SEND_MAX_RETRIES, $adapter->sendCalls, 'Attempts must be capped at MESSAGE_SEND_MAX_RETRIES'); + $this->assertSame(0, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::FAILED, $message->getAttribute('status')); + $this->assertCount($count, $message->getAttribute('deliveryErrors')); + } + /** * @param array $queries * @return array