mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
feat(messaging): retry throttled recipients with backoff in send path
Provider rate limiting and transient errors previously failed a whole batch outright. Wrap the adapter send in a backoff/retry loop that partitions per-recipient results: delivered recipients are summed and never retried, retryable failures (throttle/rate-limit/quota/503/timeout phrasing) are re-sent with only the still-failing identifiers in `to`, and permanent failures stay terminal. Exponential backoff with jitter runs on the non-blocking Swoole coroutine sleep so sibling sends keep progressing; Retry-After is intentionally not honored to keep the messaging library adapter-agnostic. Accounting stays exact: recipients is the original batch size and delivered is summed across attempts without double-counting.
This commit is contained in:
1 parent
da061ba7d2
commit
2cd1abc273
3 files changed
+409
-58
No files matched your search
@@ -246,6 +246,8 @@ const MESSAGE_SEND_TYPE_EXTERNAL = 'external';
|
||||
const MESSAGE_RECIPIENTS_PAGE_SIZE = 1000; // Recipients resolved from the database per page while streaming a send
|
||||
const MESSAGE_SEND_CONCURRENCY = 10; // Maximum adapter send requests running concurrently within a single send job
|
||||
const MESSAGE_DELIVERY_ERRORS_LIMIT = 100; // Maximum number of per-recipient delivery errors retained on a message
|
||||
const MESSAGE_SEND_MAX_RETRIES = 5; // Maximum number of attempts to deliver a batch when the provider throttles or fails transiently
|
||||
const MESSAGE_SEND_RETRY_DELAY = 1.0; // Base seconds for exponential backoff between batch send retries; overridable in tests for an instant suite
|
||||
// Mail Types
|
||||
const MAIL_TYPE_VERIFICATION = 'verification';
|
||||
const MAIL_TYPE_MAGIC_SESSION = 'magicSession';
|
||||
|
||||
@@ -497,6 +497,11 @@ class Messaging extends Action
|
||||
/**
|
||||
* Send a single adapter-sized batch and report delivery counts plus a bounded error list.
|
||||
*
|
||||
* Wraps the provider call in a backoff/retry loop that reacts to provider rate limiting and transient
|
||||
* failures (see {@see retrySend()}). Accounting stays exact: only still-failing recipients are ever
|
||||
* retried, so `delivered` is summed across attempts without double-counting, and `recipients` always
|
||||
* reports the original batch size so the caller's `failed = recipients - delivered` holds.
|
||||
*
|
||||
* @param array<string> $batch
|
||||
* @return array{delivered: int, recipients: int, errors: array<string>}
|
||||
*/
|
||||
@@ -512,66 +517,29 @@ class Messaging extends Action
|
||||
UsagePublisher $publisherForUsage
|
||||
): array {
|
||||
$recipients = \count($batch);
|
||||
$delivered = 0;
|
||||
$errors = [];
|
||||
|
||||
$messageData = clone $message;
|
||||
$messageData->setAttribute('to', $batch);
|
||||
[
|
||||
'delivered' => $delivered,
|
||||
'errors' => $errors,
|
||||
] = $this->retrySend($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project);
|
||||
|
||||
$data = match ($providerType) {
|
||||
MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider),
|
||||
MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData),
|
||||
MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project),
|
||||
default => throw new \Exception('Provider with the requested ID is of the incorrect type')
|
||||
};
|
||||
$failed = $recipients - $delivered;
|
||||
$usage = new UsageContext();
|
||||
$usage
|
||||
->addMetric(METRIC_MESSAGES, $recipients)
|
||||
->addMetric(METRIC_MESSAGES_SENT, $delivered)
|
||||
->addMetric(METRIC_MESSAGES_FAILED, $failed)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed);
|
||||
|
||||
try {
|
||||
$response = $adapter->send($data);
|
||||
$delivered = (int) $response['deliveredTo'];
|
||||
|
||||
foreach ($response['results'] as $result) {
|
||||
if ($result['status'] === 'failure' && \count($errors) < MESSAGE_DELIVERY_ERRORS_LIMIT) {
|
||||
$errors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}";
|
||||
}
|
||||
|
||||
// Deleting push targets when token has expired.
|
||||
if (($result['error'] ?? '') === 'Expired device token') {
|
||||
$target = $dbForProject->findOne('targets', [
|
||||
Query::equal('identifier', [$result['recipient']])
|
||||
]);
|
||||
|
||||
if (!$target->isEmpty()) {
|
||||
$dbForProject->updateDocument(
|
||||
'targets',
|
||||
$target->getId(),
|
||||
$target->setAttribute('expired', true)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
// Whole-batch failure: record one representative error and count every recipient as failed.
|
||||
$delivered = 0;
|
||||
$errors = ['Failed sending to targets with error: ' . $e->getMessage()];
|
||||
} finally {
|
||||
$failed = $recipients - $delivered;
|
||||
$usage = new UsageContext();
|
||||
$usage
|
||||
->addMetric(METRIC_MESSAGES, $recipients)
|
||||
->addMetric(METRIC_MESSAGES_SENT, $delivered)
|
||||
->addMetric(METRIC_MESSAGES_FAILED, $failed)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed);
|
||||
|
||||
$publisherForUsage->enqueue(new Usage(
|
||||
project: $project,
|
||||
metrics: $usage->getMetrics(),
|
||||
));
|
||||
}
|
||||
$publisherForUsage->enqueue(new Usage(
|
||||
project: $project,
|
||||
metrics: $usage->getMetrics(),
|
||||
));
|
||||
|
||||
return [
|
||||
'delivered' => $delivered,
|
||||
@@ -580,6 +548,198 @@ class Messaging extends Action
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Drive a batch through provider sends with exponential backoff, reacting to throttling and transient
|
||||
* errors by retrying only the recipients that are still failing.
|
||||
*
|
||||
* Partitioning per attempt: the provider's per-recipient results are split into delivered (summed into
|
||||
* the running total, never retried) and failures. A failure whose error matches {@see isRetryableError()}
|
||||
* stays pending for the next attempt; any other failure becomes a terminal error immediately. When
|
||||
* `send()` throws, the throw is retryable only if its message matches; otherwise every still-pending
|
||||
* recipient is recorded as terminal. The next attempt rebuilds the provider message with only the pending
|
||||
* recipients in `to`, so an already-delivered recipient is never re-sent. After {@see MESSAGE_SEND_MAX_RETRIES}
|
||||
* attempts, any recipients still pending are flushed to terminal errors.
|
||||
*
|
||||
* Note: provider `Retry-After` hints are not honored — the agnostic adapter {@see \Utopia\Messaging\Response}
|
||||
* exposes only generic per-recipient strings, never a structured retry delay, so we keep the library
|
||||
* adapter-agnostic and rely on exponential backoff alone.
|
||||
*
|
||||
* @param array<string> $batch
|
||||
* @return array{delivered: int, errors: array<string>}
|
||||
*/
|
||||
private function retrySend(
|
||||
array $batch,
|
||||
Document $message,
|
||||
Document $provider,
|
||||
string $providerType,
|
||||
EmailAdapter|SMSAdapter|PushAdapter $adapter,
|
||||
Database $dbForProject,
|
||||
Device $deviceForFiles,
|
||||
Document $project
|
||||
): array {
|
||||
$delivered = 0;
|
||||
$errors = [];
|
||||
|
||||
// Recipients still awaiting a successful (or terminal) outcome; shrinks to only the failing ones each attempt.
|
||||
$pending = $batch;
|
||||
|
||||
for ($attempt = 1; $attempt <= MESSAGE_SEND_MAX_RETRIES; $attempt++) {
|
||||
$hasRetriesLeft = $attempt < MESSAGE_SEND_MAX_RETRIES;
|
||||
|
||||
// Rebuild the provider message scoped to only the still-pending recipients so a partially-delivered
|
||||
// batch never re-sends to recipients that already succeeded on an earlier attempt.
|
||||
$data = $this->buildMessage($pending, $message, $provider, $providerType, $dbForProject, $deviceForFiles, $project);
|
||||
|
||||
$retry = [];
|
||||
|
||||
try {
|
||||
$response = $adapter->send($data);
|
||||
$delivered += (int) $response['deliveredTo'];
|
||||
|
||||
foreach ($response['results'] as $result) {
|
||||
if ($result['status'] !== 'failure') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$error = $result['error'] ?? null;
|
||||
$recipient = $result['recipient'];
|
||||
|
||||
// Deleting push targets when token has expired.
|
||||
if ($error === 'Expired device token') {
|
||||
$target = $dbForProject->findOne('targets', [
|
||||
Query::equal('identifier', [$recipient])
|
||||
]);
|
||||
|
||||
if (!$target->isEmpty()) {
|
||||
$dbForProject->updateDocument(
|
||||
'targets',
|
||||
$target->getId(),
|
||||
$target->setAttribute('expired', true)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if ($hasRetriesLeft && $this->isRetryableError($error)) {
|
||||
$retry[] = $recipient;
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->recordError($errors, "Failed sending to target {$recipient} with error: {$error}");
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
// Whole-batch failure: retry every still-pending recipient when the throw looks transient,
|
||||
// otherwise record one representative terminal error covering them all.
|
||||
if ($hasRetriesLeft && $this->isRetryableError($e->getMessage())) {
|
||||
$retry = $pending;
|
||||
} else {
|
||||
$this->recordError($errors, 'Failed sending to targets with error: ' . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
$pending = $retry;
|
||||
|
||||
if (empty($pending)) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Exponential backoff with jitter; non-blocking under Swoole so sibling sends keep progressing.
|
||||
$delay = $this->retryDelay() * (2 ** ($attempt - 1));
|
||||
$delay += $delay * (\random_int(0, 100) / 1000);
|
||||
\Swoole\Coroutine::sleep($delay);
|
||||
}
|
||||
|
||||
return [
|
||||
'delivered' => $delivered,
|
||||
'errors' => $errors,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a delivery error while keeping the retained list bounded by {@see MESSAGE_DELIVERY_ERRORS_LIMIT}.
|
||||
*
|
||||
* @param array<string> $errors
|
||||
*/
|
||||
private function recordError(array &$errors, string $error): void
|
||||
{
|
||||
if (\count($errors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) {
|
||||
return;
|
||||
}
|
||||
|
||||
$errors[] = $error;
|
||||
}
|
||||
|
||||
/**
|
||||
* Conservatively classify a provider error string as retryable. The agnostic adapter
|
||||
* {@see \Utopia\Messaging\Response} returns only free-form error strings, so string matching is the only
|
||||
* provider-agnostic signal available. The pattern list is intentionally narrow — throttling, rate limits,
|
||||
* quota, service-unavailable and timeout phrasing — so permanent failures (e.g. invalid recipients) are
|
||||
* never retried.
|
||||
*/
|
||||
private function isRetryableError(?string $error): bool
|
||||
{
|
||||
if ($error === null || $error === '') {
|
||||
return false;
|
||||
}
|
||||
|
||||
$patterns = [
|
||||
'throttl',
|
||||
'rate exceeded',
|
||||
'rate limit',
|
||||
'too many requests',
|
||||
'429',
|
||||
'quota',
|
||||
'service unavailable',
|
||||
'503',
|
||||
'timed out',
|
||||
'timeout',
|
||||
'temporarily',
|
||||
];
|
||||
|
||||
$needle = \strtolower($error);
|
||||
|
||||
foreach ($patterns as $pattern) {
|
||||
if (\str_contains($needle, $pattern)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Base seconds for exponential backoff between send retries. Isolated so tests can override it to keep the
|
||||
* suite instant without touching the production constant.
|
||||
*/
|
||||
protected function retryDelay(): float
|
||||
{
|
||||
return MESSAGE_SEND_RETRY_DELAY;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the provider-specific message for a set of recipients.
|
||||
*
|
||||
* @param array<string> $to
|
||||
*/
|
||||
private function buildMessage(
|
||||
array $to,
|
||||
Document $message,
|
||||
Document $provider,
|
||||
string $providerType,
|
||||
Database $dbForProject,
|
||||
Device $deviceForFiles,
|
||||
Document $project
|
||||
): Email|SMS|Push {
|
||||
$messageData = clone $message;
|
||||
$messageData->setAttribute('to', $to);
|
||||
|
||||
return match ($providerType) {
|
||||
MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider),
|
||||
MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData),
|
||||
MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project),
|
||||
default => throw new \Exception('Provider with the requested ID is of the incorrect type')
|
||||
};
|
||||
}
|
||||
|
||||
private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void
|
||||
{
|
||||
if ($this->adapter === null) {
|
||||
|
||||
@@ -91,6 +91,7 @@ class ConcurrencyProbeAdapter extends EmailAdapter
|
||||
|
||||
/**
|
||||
* Worker subclass that injects the instrumented email adapter instead of a network-backed provider adapter.
|
||||
* Retry backoff is zeroed so the throttle/retry tests stay instant.
|
||||
*/
|
||||
class TestableMessaging extends Messaging
|
||||
{
|
||||
@@ -102,6 +103,80 @@ class TestableMessaging extends Messaging
|
||||
{
|
||||
return $this->emailAdapter;
|
||||
}
|
||||
|
||||
protected function retryDelay(): float
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Email adapter double whose per-attempt outcome is scripted. Each {@see process()} call records the exact
|
||||
* `to` recipients it received and consults the next scripted step, which either throws (simulating a
|
||||
* connection-level provider error) or returns a per-recipient success/failure verdict. When the script runs
|
||||
* out, every recipient succeeds — modelling a provider that has stopped throttling.
|
||||
*/
|
||||
class ScriptedEmailAdapter extends EmailAdapter
|
||||
{
|
||||
public int $sendCalls = 0;
|
||||
|
||||
/**
|
||||
* @var array<array<string>> Recipient email lists captured per {@see process()} call, in call order.
|
||||
*/
|
||||
public array $recipientsPerCall = [];
|
||||
|
||||
/**
|
||||
* @param array<\Throwable|(callable(string): ?string)> $script One step per attempt. A Throwable is thrown
|
||||
* for that attempt; a callable returns an error string for a failing recipient or null for success.
|
||||
*/
|
||||
public function __construct(private readonly array $script = [])
|
||||
{
|
||||
}
|
||||
|
||||
public function getName(): string
|
||||
{
|
||||
return 'ScriptedEmail';
|
||||
}
|
||||
|
||||
public function getMaxMessagesPerRequest(): int
|
||||
{
|
||||
return 1000;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param EmailMessage $message
|
||||
* @return array{deliveredTo: int, type: string, results: array<array<string, mixed>>}
|
||||
*/
|
||||
protected function process(EmailMessage $message): array
|
||||
{
|
||||
$recipients = \array_map(static fn (array $entry): string => $entry['email'], $message->getTo());
|
||||
$this->recipientsPerCall[] = $recipients;
|
||||
|
||||
$step = $this->script[$this->sendCalls] ?? null;
|
||||
$this->sendCalls++;
|
||||
|
||||
if ($step instanceof \Throwable) {
|
||||
throw $step;
|
||||
}
|
||||
|
||||
$response = new Response($this->getType());
|
||||
$delivered = 0;
|
||||
|
||||
foreach ($recipients as $recipient) {
|
||||
$error = \is_callable($step) ? $step($recipient) : null;
|
||||
|
||||
if ($error === null) {
|
||||
$delivered++;
|
||||
$response->addResult($recipient);
|
||||
} else {
|
||||
$response->addResult($recipient, $error);
|
||||
}
|
||||
}
|
||||
|
||||
$response->setDeliveredTo($delivered);
|
||||
|
||||
return $response->toArray();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -326,7 +401,7 @@ class MessagingFanoutTest extends TestCase
|
||||
|
||||
private function sendTopic(
|
||||
RecordingDatabase $database,
|
||||
ConcurrencyProbeAdapter $adapter
|
||||
EmailAdapter $adapter
|
||||
): void {
|
||||
$worker = new TestableMessaging($adapter);
|
||||
$method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage');
|
||||
@@ -483,6 +558,120 @@ class MessagingFanoutTest extends TestCase
|
||||
$this->assertNull($database->updatedMessage, 'Terminal messages must not be rewritten');
|
||||
}
|
||||
|
||||
public function testThrottledBatchIsRetriedThenFullyDelivered(): void
|
||||
{
|
||||
$count = 5;
|
||||
$data = $this->topicDataset($count);
|
||||
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
|
||||
|
||||
// First attempt throttles every recipient; the script then runs out so the second attempt succeeds.
|
||||
$adapter = new ScriptedEmailAdapter([
|
||||
static fn (string $recipient): string => 'Rate limit exceeded, please retry',
|
||||
]);
|
||||
|
||||
$this->sendTopic($database, $adapter);
|
||||
|
||||
$message = $database->updatedMessage;
|
||||
$this->assertNotNull($message);
|
||||
|
||||
// The batch was retried exactly once and then fully delivered with no double-counting.
|
||||
$this->assertSame(2, $adapter->sendCalls, 'Throttled batch must be retried once');
|
||||
$this->assertSame($count, $message->getAttribute('deliveredTotal'));
|
||||
$this->assertSame(MessageStatus::SENT, $message->getAttribute('status'));
|
||||
$this->assertEmpty($message->getAttribute('deliveryErrors'));
|
||||
|
||||
// Both attempts targeted the full recipient set (none had succeeded on the first attempt).
|
||||
$this->assertCount($count, $adapter->recipientsPerCall[0]);
|
||||
$this->assertCount($count, $adapter->recipientsPerCall[1]);
|
||||
}
|
||||
|
||||
public function testRetryTargetsOnlyThrottledRecipients(): void
|
||||
{
|
||||
$count = 5;
|
||||
$data = $this->topicDataset($count);
|
||||
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
|
||||
|
||||
// user2@ and user4@ throttle on the first attempt; everyone else succeeds. The second attempt has no
|
||||
// script step, so the two retried recipients then succeed.
|
||||
$throttled = ['user2@example.com', 'user4@example.com'];
|
||||
$adapter = new ScriptedEmailAdapter([
|
||||
static fn (string $recipient): ?string => \in_array($recipient, $throttled, true)
|
||||
? 'Too Many Requests'
|
||||
: null,
|
||||
]);
|
||||
|
||||
$this->sendTopic($database, $adapter);
|
||||
|
||||
$message = $database->updatedMessage;
|
||||
$this->assertNotNull($message);
|
||||
|
||||
$this->assertSame(2, $adapter->sendCalls, 'A partially throttled batch must be retried once');
|
||||
|
||||
// The retry must re-send to ONLY the throttled recipients — never the ones already delivered.
|
||||
\sort($adapter->recipientsPerCall[1]);
|
||||
$this->assertSame($throttled, $adapter->recipientsPerCall[1]);
|
||||
|
||||
// Every recipient is ultimately delivered and counted exactly once.
|
||||
$this->assertSame($count, $message->getAttribute('deliveredTotal'));
|
||||
$this->assertSame(MessageStatus::SENT, $message->getAttribute('status'));
|
||||
$this->assertEmpty($message->getAttribute('deliveryErrors'));
|
||||
}
|
||||
|
||||
public function testNonRetryableFailureIsNotRetried(): void
|
||||
{
|
||||
$count = 3;
|
||||
$data = $this->topicDataset($count);
|
||||
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
|
||||
|
||||
// user1@ permanently fails with a non-retryable error; the rest succeed.
|
||||
$adapter = new ScriptedEmailAdapter([
|
||||
static fn (string $recipient): ?string => $recipient === 'user1@example.com'
|
||||
? 'Invalid recipient address'
|
||||
: null,
|
||||
]);
|
||||
|
||||
$this->sendTopic($database, $adapter);
|
||||
|
||||
$message = $database->updatedMessage;
|
||||
$this->assertNotNull($message);
|
||||
|
||||
// A permanent failure is recorded immediately, never retried.
|
||||
$this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry');
|
||||
$this->assertSame($count - 1, $message->getAttribute('deliveredTotal'));
|
||||
$this->assertSame(MessageStatus::FAILED, $message->getAttribute('status'));
|
||||
$errors = $message->getAttribute('deliveryErrors');
|
||||
$this->assertCount(1, $errors);
|
||||
$this->assertStringContainsString('user1@example.com', $errors[0]);
|
||||
$this->assertStringContainsString('Invalid recipient address', $errors[0]);
|
||||
}
|
||||
|
||||
public function testExhaustedRetriesRecordTerminalFailure(): void
|
||||
{
|
||||
$count = 2;
|
||||
$data = $this->topicDataset($count);
|
||||
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
|
||||
|
||||
// Every attempt throttles, for more steps than MESSAGE_SEND_MAX_RETRIES, so retries are exhausted.
|
||||
$script = \array_fill(
|
||||
0,
|
||||
MESSAGE_SEND_MAX_RETRIES + 2,
|
||||
static fn (string $recipient): string => 'Service unavailable, throttled'
|
||||
);
|
||||
$adapter = new ScriptedEmailAdapter($script);
|
||||
|
||||
$this->sendTopic($database, $adapter);
|
||||
|
||||
$message = $database->updatedMessage;
|
||||
$this->assertNotNull($message);
|
||||
|
||||
// The batch is attempted exactly MESSAGE_SEND_MAX_RETRIES times, then the still-throttled recipients
|
||||
// become terminal failures with nothing delivered.
|
||||
$this->assertSame(MESSAGE_SEND_MAX_RETRIES, $adapter->sendCalls, 'Attempts must be capped at MESSAGE_SEND_MAX_RETRIES');
|
||||
$this->assertSame(0, $message->getAttribute('deliveredTotal'));
|
||||
$this->assertSame(MessageStatus::FAILED, $message->getAttribute('status'));
|
||||
$this->assertCount($count, $message->getAttribute('deliveryErrors'));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Query> $queries
|
||||
* @return array<string>
|
||||
|
||||
Reference in new issue
Block a user