mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Merge pull request #12465 from appwrite/feat-messaging-scale-fanout-19x
perf(messaging): stream large-topic sends and bound fan-out concurrency
This commit is contained in:
3 files changed
+1348
-177
No files matched your search
@@ -242,6 +242,12 @@ const RULE_STATUS_VERIFIED = 'verified';
|
||||
// Message types
|
||||
const MESSAGE_SEND_TYPE_INTERNAL = 'internal';
|
||||
const MESSAGE_SEND_TYPE_EXTERNAL = 'external';
|
||||
// External message fan-out tuning
|
||||
const MESSAGE_RECIPIENTS_PAGE_SIZE = 5000; // Recipients resolved from the database per page while streaming a send; aligned with APP_LIMIT_COUNT to minimise round-trips on large topics
|
||||
const MESSAGE_SEND_CONCURRENCY = 10; // Maximum adapter send requests running concurrently within a single send job
|
||||
const MESSAGE_DELIVERY_ERRORS_LIMIT = 100; // Maximum number of per-recipient delivery errors retained on a message
|
||||
const MESSAGE_SEND_MAX_RETRIES = 5; // Maximum number of attempts to deliver a batch when the provider throttles or fails transiently
|
||||
const MESSAGE_SEND_RETRY_DELAY = 1.0; // Base seconds for exponential backoff between batch send retries; overridable in tests for an instant suite
|
||||
// Mail Types
|
||||
const MAIL_TYPE_VERIFICATION = 'verification';
|
||||
const MAIL_TYPE_MAGIC_SESSION = 'magicSession';
|
||||
|
||||
@@ -16,6 +16,7 @@ use Utopia\Database\Document;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\DSN\DSN;
|
||||
use Utopia\Lock\Semaphore;
|
||||
use Utopia\Logger\Log;
|
||||
use Utopia\Messaging\Adapter\Email as EmailAdapter;
|
||||
use Utopia\Messaging\Adapter\Email\Mailgun;
|
||||
@@ -129,69 +130,19 @@ class Messaging extends Action
|
||||
Document $project,
|
||||
UsagePublisher $publisherForUsage
|
||||
): void {
|
||||
$status = $message->getAttribute('status');
|
||||
|
||||
// Idempotency guard: queue redelivery can hand us a message that already finished.
|
||||
if (\in_array($status, [MessageStatus::SENT, MessageStatus::FAILED], true)) {
|
||||
Span::add('message.skipped', 'already_processed');
|
||||
return;
|
||||
}
|
||||
|
||||
$topicIds = $message->getAttribute('topics', []);
|
||||
$targetIds = $message->getAttribute('targets', []);
|
||||
$userIds = $message->getAttribute('users', []);
|
||||
$providerType = $message->getAttribute('providerType');
|
||||
|
||||
/**
|
||||
* @var array<Document> $allTargets
|
||||
*/
|
||||
$allTargets = [];
|
||||
|
||||
if (\count($topicIds) > 0) {
|
||||
$topics = $dbForProject->find('topics', [
|
||||
Query::equal('$id', $topicIds),
|
||||
Query::limit(\count($topicIds)),
|
||||
]);
|
||||
foreach ($topics as $topic) {
|
||||
$targets = \array_filter($topic->getAttribute('targets'), function (Document $target) use ($providerType) {
|
||||
return $target->getAttribute('providerType') === $providerType;
|
||||
});
|
||||
|
||||
\array_push($allTargets, ...$targets);
|
||||
}
|
||||
}
|
||||
|
||||
if (\count($userIds) > 0) {
|
||||
$limit = 1000;
|
||||
$offset = 0;
|
||||
|
||||
do {
|
||||
$targets = $dbForProject->find('targets', [
|
||||
Query::equal('userId', $userIds),
|
||||
Query::select(['providerId', 'identifier']),
|
||||
Query::equal('providerType', [$providerType]),
|
||||
Query::limit($limit),
|
||||
Query::offset($offset),
|
||||
]);
|
||||
|
||||
\array_push($allTargets, ...$targets);
|
||||
$offset += \count($targets);
|
||||
} while (\count($targets) === $limit);
|
||||
}
|
||||
|
||||
if (\count($targetIds) > 0) {
|
||||
$targets = $dbForProject->find('targets', [
|
||||
Query::select(['providerId', 'identifier']),
|
||||
Query::equal('$id', $targetIds),
|
||||
Query::equal('providerType', [$providerType]),
|
||||
Query::limit(\count($targetIds)),
|
||||
]);
|
||||
|
||||
\array_push($allTargets, ...$targets);
|
||||
}
|
||||
|
||||
if (empty($allTargets)) {
|
||||
$dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([
|
||||
'status' => MessageStatus::FAILED,
|
||||
'deliveryErrors' => ['No valid recipients found.']
|
||||
]));
|
||||
|
||||
Span::add('message.skipped', 'no_valid_recipients');
|
||||
return;
|
||||
}
|
||||
|
||||
$default = $dbForProject->findOne('providers', [
|
||||
Query::equal('enabled', [true]),
|
||||
Query::equal('type', [$providerType]),
|
||||
@@ -208,55 +159,33 @@ class Messaging extends Action
|
||||
}
|
||||
|
||||
/**
|
||||
* @var array<string, array<string, null>> $identifiers
|
||||
*/
|
||||
$identifiers = [];
|
||||
|
||||
/**
|
||||
* @var array<Document> $providers
|
||||
* Resolved providers cached for the lifetime of this job, keyed by provider id.
|
||||
* Seeded with the default provider so most sends never touch the providers collection.
|
||||
*
|
||||
* @var array<string, Document> $providers
|
||||
*/
|
||||
$providers = [
|
||||
$default->getId() => $default
|
||||
$default->getId() => $default,
|
||||
];
|
||||
|
||||
foreach ($allTargets as $target) {
|
||||
$providerId = $target->getAttribute('providerId');
|
||||
$semaphore = new Semaphore(MESSAGE_SEND_CONCURRENCY);
|
||||
|
||||
if (!$providerId) {
|
||||
$providerId = $default->getId();
|
||||
}
|
||||
$deliveredTotal = 0;
|
||||
$failedTotal = 0;
|
||||
$deliveryErrors = [];
|
||||
$hasRecipients = false;
|
||||
|
||||
if ($providerId) {
|
||||
if (!\array_key_exists($providerId, $identifiers)) {
|
||||
$identifiers[$providerId] = [];
|
||||
}
|
||||
// Use null as value to avoid duplicate keys
|
||||
$identifiers[$providerId][$target->getAttribute('identifier')] = null;
|
||||
}
|
||||
}
|
||||
foreach ($this->streamRecipients($dbForProject, $topicIds, $userIds, $targetIds, $providerType, $default) as $page) {
|
||||
/**
|
||||
* @var array<callable> $tasks
|
||||
*/
|
||||
$tasks = [];
|
||||
|
||||
/**
|
||||
* @var array<array> $results
|
||||
*/
|
||||
$results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
|
||||
return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
|
||||
if (\array_key_exists($providerId, $providers)) {
|
||||
$provider = $providers[$providerId];
|
||||
} else {
|
||||
$provider = $dbForProject->getDocument('providers', $providerId);
|
||||
foreach ($page as $providerId => $identifiers) {
|
||||
$provider = $this->resolveProvider($dbForProject, $providerId, $providers, $default);
|
||||
$resolvedProviderType = $provider->getAttribute('type');
|
||||
|
||||
if ($provider->isEmpty() || !$provider->getAttribute('enabled')) {
|
||||
$provider = $default;
|
||||
} else {
|
||||
$providers[$providerId] = $provider;
|
||||
}
|
||||
}
|
||||
|
||||
$identifiersForProvider = $identifiers[$providerId];
|
||||
|
||||
$providerType = $provider->getAttribute('type');
|
||||
|
||||
$adapter = match ($providerType) {
|
||||
$adapter = match ($resolvedProviderType) {
|
||||
MESSAGE_TYPE_SMS => $this->getSmsAdapter($provider),
|
||||
MESSAGE_TYPE_PUSH => $this->getPushAdapter($provider),
|
||||
MESSAGE_TYPE_EMAIL => $this->getEmailAdapter($provider),
|
||||
@@ -264,101 +193,74 @@ class Messaging extends Action
|
||||
};
|
||||
|
||||
$batches = \array_chunk(
|
||||
\array_keys($identifiersForProvider),
|
||||
\array_keys($identifiers),
|
||||
$adapter->getMaxMessagesPerRequest()
|
||||
);
|
||||
|
||||
return batch(\array_map(function ($batch) use ($message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
|
||||
return function () use ($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
|
||||
$deliveredTotal = 0;
|
||||
$deliveryErrors = [];
|
||||
$messageData = clone $message;
|
||||
$messageData->setAttribute('to', $batch);
|
||||
foreach ($batches as $batch) {
|
||||
$tasks[] = fn (): array => $semaphore->withLock(
|
||||
fn (): array => $this->sendBatch(
|
||||
$batch,
|
||||
$message,
|
||||
$provider,
|
||||
$resolvedProviderType,
|
||||
$adapter,
|
||||
$dbForProject,
|
||||
$deviceForFiles,
|
||||
$project,
|
||||
$publisherForUsage
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
$data = match ($providerType) {
|
||||
MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider),
|
||||
MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData),
|
||||
MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project),
|
||||
};
|
||||
// A page can resolve to zero identifiers (e.g. subscribers whose targetInternalId matches no targets
|
||||
// row), so an empty task list must not count as recipients or run batch() for nothing.
|
||||
if (empty($tasks)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
$response = $adapter->send($data);
|
||||
$deliveredTotal += (int) $response['deliveredTo'];
|
||||
foreach ($response['results'] as $result) {
|
||||
if ($result['status'] === 'failure') {
|
||||
$deliveryErrors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}";
|
||||
}
|
||||
$hasRecipients = true;
|
||||
|
||||
// Deleting push targets when token has expired.
|
||||
if (($result['error'] ?? '') === 'Expired device token') {
|
||||
$target = $dbForProject->findOne('targets', [
|
||||
Query::equal('identifier', [$result['recipient']])
|
||||
]);
|
||||
/**
|
||||
* @var array<array{delivered: int, recipients: int, errors: array<string>}> $results
|
||||
*/
|
||||
$results = batch($tasks);
|
||||
|
||||
if (!$target->isEmpty()) {
|
||||
$dbForProject->updateDocument(
|
||||
'targets',
|
||||
$target->getId(),
|
||||
$target->setAttribute('expired', true)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
$deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage();
|
||||
} finally {
|
||||
$errorTotal = \count($deliveryErrors);
|
||||
$usage = new UsageContext();
|
||||
$usage
|
||||
->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal))
|
||||
->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal)
|
||||
->addMetric(METRIC_MESSAGES_FAILED, $errorTotal)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), ($deliveredTotal + $errorTotal))
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $deliveredTotal)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $errorTotal)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), ($deliveredTotal + $errorTotal))
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $deliveredTotal)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal);
|
||||
foreach ($results as $result) {
|
||||
$deliveredTotal += $result['delivered'];
|
||||
$failedTotal += $result['recipients'] - $result['delivered'];
|
||||
|
||||
$publisherForUsage->enqueue(new Usage(
|
||||
project: $project,
|
||||
metrics: $usage->getMetrics(),
|
||||
));
|
||||
foreach ($result['errors'] as $error) {
|
||||
if (\count($deliveryErrors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) {
|
||||
break;
|
||||
}
|
||||
|
||||
return [
|
||||
'deliveredTotal' => $deliveredTotal,
|
||||
'deliveryErrors' => $deliveryErrors,
|
||||
];
|
||||
}
|
||||
};
|
||||
}, $batches));
|
||||
};
|
||||
}, \array_keys($identifiers)));
|
||||
$deliveryErrors[] = $error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$results = \array_merge(...$results);
|
||||
if (!$hasRecipients) {
|
||||
$dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([
|
||||
'status' => MessageStatus::FAILED,
|
||||
'deliveryErrors' => ['No valid recipients found.']
|
||||
]));
|
||||
|
||||
$deliveredTotal = 0;
|
||||
$deliveryErrors = [];
|
||||
|
||||
foreach ($results as $result) {
|
||||
$deliveredTotal += $result['deliveredTotal'];
|
||||
$deliveryErrors = \array_merge($deliveryErrors, $result['deliveryErrors']);
|
||||
Span::add('message.skipped', 'no_valid_recipients');
|
||||
return;
|
||||
}
|
||||
|
||||
if (empty($deliveryErrors) && $deliveredTotal === 0) {
|
||||
$deliveryErrors[] = 'Unknown error';
|
||||
}
|
||||
|
||||
$hasFailures = $failedTotal > 0 || \count($deliveryErrors) > 0;
|
||||
$message->setAttribute('status', $hasFailures ? MessageStatus::FAILED : MessageStatus::SENT);
|
||||
$message->setAttribute('deliveryErrors', $deliveryErrors);
|
||||
|
||||
if (\count($message->getAttribute('deliveryErrors')) > 0) {
|
||||
$message->setAttribute('status', MessageStatus::FAILED);
|
||||
} else {
|
||||
$message->setAttribute('status', MessageStatus::SENT);
|
||||
}
|
||||
|
||||
Span::add('message.delivered_total', $deliveredTotal);
|
||||
Span::add('message.errors_total', \count($deliveryErrors));
|
||||
Span::add('message.errors_total', $failedTotal);
|
||||
|
||||
$message->removeAttribute('to');
|
||||
|
||||
@@ -409,6 +311,461 @@ class Messaging extends Action
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream a message's recipients in bounded pages, grouped by provider and deduplicated by identifier within each page.
|
||||
*
|
||||
* Peak memory is O(MESSAGE_RECIPIENTS_PAGE_SIZE), never O(topic size): topics are walked through the
|
||||
* subscribers collection with cursor pagination rather than reading the topic's `targets` attribute, which
|
||||
* triggers the subQueryTopicTargets filter and loads up to APP_LIMIT_SUBSCRIBERS_SUBQUERY rows at once.
|
||||
*
|
||||
* @param array<string> $topicIds
|
||||
* @param array<string> $userIds
|
||||
* @param array<string> $targetIds
|
||||
* @return \Generator<array<string, array<string, null>>>
|
||||
* @throws \Exception
|
||||
*/
|
||||
private function streamRecipients(
|
||||
Database $dbForProject,
|
||||
array $topicIds,
|
||||
array $userIds,
|
||||
array $targetIds,
|
||||
string $providerType,
|
||||
Document $default
|
||||
): \Generator {
|
||||
if (\count($topicIds) > 0) {
|
||||
$topics = $dbForProject->find('topics', [
|
||||
Query::select(['$sequence']),
|
||||
Query::equal('$id', $topicIds),
|
||||
Query::limit(\count($topicIds)),
|
||||
]);
|
||||
|
||||
foreach ($topics as $topic) {
|
||||
$cursor = null;
|
||||
|
||||
do {
|
||||
$queries = [
|
||||
Query::equal('topicInternalId', [$topic->getSequence()]),
|
||||
Query::equal('providerType', [$providerType]),
|
||||
Query::select(['$sequence', 'targetInternalId']),
|
||||
Query::orderAsc('$sequence'),
|
||||
Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE),
|
||||
];
|
||||
|
||||
if ($cursor !== null) {
|
||||
$queries[] = Query::cursorAfter($cursor);
|
||||
}
|
||||
|
||||
$subscribers = $dbForProject->getAuthorization()->skip(
|
||||
fn () => $dbForProject->find('subscribers', $queries)
|
||||
);
|
||||
|
||||
$count = \count($subscribers);
|
||||
|
||||
if ($count === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
$cursor = $subscribers[$count - 1];
|
||||
|
||||
$targetInternalIds = \array_map(
|
||||
fn (Document $subscriber) => $subscriber->getAttribute('targetInternalId'),
|
||||
$subscribers
|
||||
);
|
||||
|
||||
$targets = $dbForProject->skipValidation(
|
||||
fn () => $dbForProject->getAuthorization()->skip(
|
||||
fn () => $dbForProject->find('targets', [
|
||||
Query::equal('$sequence', $targetInternalIds),
|
||||
Query::select(['providerId', 'identifier']),
|
||||
Query::limit(\count($targetInternalIds)),
|
||||
])
|
||||
)
|
||||
);
|
||||
|
||||
yield $this->groupTargetsByProvider($targets, $default);
|
||||
} while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE);
|
||||
}
|
||||
}
|
||||
|
||||
if (\count($userIds) > 0) {
|
||||
$cursor = null;
|
||||
|
||||
do {
|
||||
$queries = [
|
||||
Query::equal('userId', $userIds),
|
||||
Query::equal('providerType', [$providerType]),
|
||||
Query::select(['$sequence', 'providerId', 'identifier']),
|
||||
Query::orderAsc('$sequence'),
|
||||
Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE),
|
||||
];
|
||||
|
||||
if ($cursor !== null) {
|
||||
$queries[] = Query::cursorAfter($cursor);
|
||||
}
|
||||
|
||||
$targets = $dbForProject->find('targets', $queries);
|
||||
$count = \count($targets);
|
||||
|
||||
if ($count === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
$cursor = $targets[$count - 1];
|
||||
|
||||
yield $this->groupTargetsByProvider($targets, $default);
|
||||
} while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE);
|
||||
}
|
||||
|
||||
if (\count($targetIds) > 0) {
|
||||
$cursor = null;
|
||||
|
||||
do {
|
||||
$queries = [
|
||||
Query::equal('$id', $targetIds),
|
||||
Query::equal('providerType', [$providerType]),
|
||||
Query::select(['$sequence', 'providerId', 'identifier']),
|
||||
Query::orderAsc('$sequence'),
|
||||
Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE),
|
||||
];
|
||||
|
||||
if ($cursor !== null) {
|
||||
$queries[] = Query::cursorAfter($cursor);
|
||||
}
|
||||
|
||||
$targets = $dbForProject->find('targets', $queries);
|
||||
$count = \count($targets);
|
||||
|
||||
if ($count === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
$cursor = $targets[$count - 1];
|
||||
|
||||
yield $this->groupTargetsByProvider($targets, $default);
|
||||
} while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Group a page of target documents by provider id, deduplicating identifiers within the page.
|
||||
*
|
||||
* @param array<Document> $targets
|
||||
* @return array<string, array<string, null>>
|
||||
*/
|
||||
private function groupTargetsByProvider(array $targets, Document $default): array
|
||||
{
|
||||
/**
|
||||
* @var array<string, array<string, null>> $identifiers
|
||||
*/
|
||||
$identifiers = [];
|
||||
|
||||
foreach ($targets as $target) {
|
||||
$providerId = $target->getAttribute('providerId') ?: $default->getId();
|
||||
|
||||
if (!\array_key_exists($providerId, $identifiers)) {
|
||||
$identifiers[$providerId] = [];
|
||||
}
|
||||
|
||||
// Null values keep identifiers unique without a second lookup structure.
|
||||
$identifiers[$providerId][$target->getAttribute('identifier')] = null;
|
||||
}
|
||||
|
||||
return $identifiers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve and cache a provider for the lifetime of a send job, falling back to the default provider.
|
||||
*
|
||||
* @param array<string, Document> $providers
|
||||
*/
|
||||
private function resolveProvider(
|
||||
Database $dbForProject,
|
||||
string $providerId,
|
||||
array &$providers,
|
||||
Document $default
|
||||
): Document {
|
||||
if (\array_key_exists($providerId, $providers)) {
|
||||
return $providers[$providerId];
|
||||
}
|
||||
|
||||
$provider = $dbForProject->getDocument('providers', $providerId);
|
||||
|
||||
if ($provider->isEmpty() || !$provider->getAttribute('enabled')) {
|
||||
// Cache the fallback under this id too, so a topic full of targets pointing at the same
|
||||
// disabled/missing provider does not re-query the providers collection once per page.
|
||||
$providers[$providerId] = $default;
|
||||
|
||||
return $default;
|
||||
}
|
||||
|
||||
$providers[$providerId] = $provider;
|
||||
|
||||
return $provider;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a single adapter-sized batch and report delivery counts plus a bounded error list.
|
||||
*
|
||||
* Wraps the provider call in a backoff/retry loop that reacts to provider rate limiting and transient
|
||||
* failures (see {@see retrySend()}). Accounting stays exact: only still-failing recipients are ever
|
||||
* retried, so `delivered` is summed across attempts without double-counting, and `recipients` always
|
||||
* reports the original batch size so the caller's `failed = recipients - delivered` holds.
|
||||
*
|
||||
* @param array<string> $batch
|
||||
* @return array{delivered: int, recipients: int, errors: array<string>}
|
||||
*/
|
||||
private function sendBatch(
|
||||
array $batch,
|
||||
Document $message,
|
||||
Document $provider,
|
||||
string $providerType,
|
||||
EmailAdapter|SMSAdapter|PushAdapter $adapter,
|
||||
Database $dbForProject,
|
||||
Device $deviceForFiles,
|
||||
Document $project,
|
||||
UsagePublisher $publisherForUsage
|
||||
): array {
|
||||
$recipients = \count($batch);
|
||||
|
||||
[
|
||||
'delivered' => $delivered,
|
||||
'errors' => $errors,
|
||||
] = $this->retrySend($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project);
|
||||
|
||||
$failed = $recipients - $delivered;
|
||||
$usage = new UsageContext();
|
||||
$usage
|
||||
->addMetric(METRIC_MESSAGES, $recipients)
|
||||
->addMetric(METRIC_MESSAGES_SENT, $delivered)
|
||||
->addMetric(METRIC_MESSAGES_FAILED, $failed)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed);
|
||||
|
||||
$publisherForUsage->enqueue(new Usage(
|
||||
project: $project,
|
||||
metrics: $usage->getMetrics(),
|
||||
));
|
||||
|
||||
return [
|
||||
'delivered' => $delivered,
|
||||
'recipients' => $recipients,
|
||||
'errors' => $errors,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Drive a batch through provider sends with exponential backoff, reacting to throttling and transient
|
||||
* errors by retrying only the recipients that are still failing.
|
||||
*
|
||||
* Partitioning per attempt: the provider's per-recipient results are split into delivered (summed into
|
||||
* the running total, never retried) and failures. A failure whose error matches {@see isRetryableError()}
|
||||
* stays pending for the next attempt; any other failure becomes a terminal error immediately. When
|
||||
* `send()` throws, the throw is retryable only if its message matches; otherwise every still-pending
|
||||
* recipient is recorded as terminal. The next attempt rebuilds the provider message with only the pending
|
||||
* recipients in `to`, so an already-delivered recipient is never re-sent. After {@see MESSAGE_SEND_MAX_RETRIES}
|
||||
* attempts, any recipients still pending are flushed to terminal errors.
|
||||
*
|
||||
* Note: provider `Retry-After` hints are not honored — the agnostic adapter {@see \Utopia\Messaging\Response}
|
||||
* exposes only generic per-recipient strings, never a structured retry delay, so we keep the library
|
||||
* adapter-agnostic and rely on exponential backoff alone.
|
||||
*
|
||||
* @param array<string> $batch
|
||||
* @return array{delivered: int, errors: array<string>}
|
||||
*/
|
||||
private function retrySend(
|
||||
array $batch,
|
||||
Document $message,
|
||||
Document $provider,
|
||||
string $providerType,
|
||||
EmailAdapter|SMSAdapter|PushAdapter $adapter,
|
||||
Database $dbForProject,
|
||||
Device $deviceForFiles,
|
||||
Document $project
|
||||
): array {
|
||||
$delivered = 0;
|
||||
$errors = [];
|
||||
|
||||
// Recipients still awaiting a successful (or terminal) outcome; shrinks to only the failing ones each attempt.
|
||||
$pending = $batch;
|
||||
|
||||
for ($attempt = 1; $attempt <= MESSAGE_SEND_MAX_RETRIES; $attempt++) {
|
||||
$hasRetriesLeft = $attempt < MESSAGE_SEND_MAX_RETRIES;
|
||||
|
||||
// Rebuild the provider message scoped to only the still-pending recipients so a partially-delivered
|
||||
// batch never re-sends to recipients that already succeeded on an earlier attempt.
|
||||
$data = $this->buildMessage($pending, $message, $provider, $providerType, $dbForProject, $deviceForFiles, $project);
|
||||
|
||||
$retry = [];
|
||||
|
||||
// The try/catch wraps ONLY the provider send. A whole-batch throw is retryable when transient,
|
||||
// otherwise it records one representative terminal error. The previous behaviour of resetting
|
||||
// $delivered to 0 on a throw is gone — the retry refactor sums delivered across attempts, and the
|
||||
// expired-device-token cleanup below is isolated in its own try so a DB hiccup there can never be
|
||||
// misattributed as a send failure.
|
||||
try {
|
||||
$response = $adapter->send($data);
|
||||
} catch (\Throwable $e) {
|
||||
if ($hasRetriesLeft && $this->isRetryableError($e->getMessage())) {
|
||||
$retry = $pending;
|
||||
} else {
|
||||
$this->recordError($errors, 'Failed sending to targets with error: ' . $e->getMessage());
|
||||
}
|
||||
|
||||
$response = null;
|
||||
}
|
||||
|
||||
if ($response !== null) {
|
||||
$delivered += (int) $response['deliveredTo'];
|
||||
|
||||
foreach ($response['results'] as $result) {
|
||||
if ($result['status'] !== 'failure') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$error = $result['error'] ?? null;
|
||||
$recipient = $result['recipient'];
|
||||
|
||||
// Best-effort: deleting push targets when the token has expired. Isolated so a transient DB
|
||||
// error here never affects delivery accounting or the retry decision below.
|
||||
if ($error === 'Expired device token') {
|
||||
try {
|
||||
$target = $dbForProject->findOne('targets', [
|
||||
Query::equal('identifier', [$recipient])
|
||||
]);
|
||||
|
||||
if (!$target->isEmpty()) {
|
||||
$dbForProject->updateDocument(
|
||||
'targets',
|
||||
$target->getId(),
|
||||
$target->setAttribute('expired', true)
|
||||
);
|
||||
}
|
||||
} catch (\Throwable) {
|
||||
// Best-effort; must not affect accounting or retries.
|
||||
}
|
||||
}
|
||||
|
||||
if ($hasRetriesLeft && $this->isRetryableError($error)) {
|
||||
$retry[] = $recipient;
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->recordError($errors, "Failed sending to target {$recipient} with error: {$error}");
|
||||
}
|
||||
}
|
||||
|
||||
$pending = $retry;
|
||||
|
||||
if (empty($pending)) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Exponential backoff with jitter; non-blocking under Swoole so sibling sends keep progressing.
|
||||
// Skip a non-positive delay: Swoole\Coroutine::sleep() rejects 0/negative values (which tests
|
||||
// produce by overriding the base delay to 0 for speed), and it would otherwise emit a warning.
|
||||
$delay = $this->retryDelay() * (2 ** ($attempt - 1));
|
||||
$delay += $delay * (\random_int(0, 100) / 1000);
|
||||
if ($delay > 0) {
|
||||
\Swoole\Coroutine::sleep($delay);
|
||||
}
|
||||
}
|
||||
|
||||
return [
|
||||
'delivered' => $delivered,
|
||||
'errors' => $errors,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a delivery error while keeping the retained list bounded by {@see MESSAGE_DELIVERY_ERRORS_LIMIT}.
|
||||
*
|
||||
* @param array<string> $errors
|
||||
*/
|
||||
private function recordError(array &$errors, string $error): void
|
||||
{
|
||||
if (\count($errors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) {
|
||||
return;
|
||||
}
|
||||
|
||||
$errors[] = $error;
|
||||
}
|
||||
|
||||
/**
|
||||
* Conservatively classify a provider error string as retryable. The agnostic adapter
|
||||
* {@see \Utopia\Messaging\Response} returns only free-form error strings, so string matching is the only
|
||||
* provider-agnostic signal available. The pattern list is intentionally narrow — throttling, rate limits,
|
||||
* quota, service-unavailable and timeout phrasing — so permanent failures (e.g. invalid recipients) are
|
||||
* never retried.
|
||||
*/
|
||||
private function isRetryableError(?string $error): bool
|
||||
{
|
||||
if ($error === null || $error === '') {
|
||||
return false;
|
||||
}
|
||||
|
||||
$patterns = [
|
||||
'throttl',
|
||||
'rate exceeded',
|
||||
'rate limit',
|
||||
'too many requests',
|
||||
'429',
|
||||
'quota',
|
||||
'service unavailable',
|
||||
'503',
|
||||
'timed out',
|
||||
'timeout',
|
||||
'temporarily',
|
||||
];
|
||||
|
||||
$needle = \strtolower($error);
|
||||
|
||||
foreach ($patterns as $pattern) {
|
||||
if (\str_contains($needle, $pattern)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Base seconds for exponential backoff between send retries. Isolated so tests can override it to keep the
|
||||
* suite instant without touching the production constant.
|
||||
*/
|
||||
protected function retryDelay(): float
|
||||
{
|
||||
return MESSAGE_SEND_RETRY_DELAY;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the provider-specific message for a set of recipients.
|
||||
*
|
||||
* @param array<string> $to
|
||||
*/
|
||||
private function buildMessage(
|
||||
array $to,
|
||||
Document $message,
|
||||
Document $provider,
|
||||
string $providerType,
|
||||
Database $dbForProject,
|
||||
Device $deviceForFiles,
|
||||
Document $project
|
||||
): Email|SMS|Push {
|
||||
$messageData = clone $message;
|
||||
$messageData->setAttribute('to', $to);
|
||||
|
||||
return match ($providerType) {
|
||||
MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider),
|
||||
MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData),
|
||||
MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project),
|
||||
default => throw new \Exception('Provider with the requested ID is of the incorrect type')
|
||||
};
|
||||
}
|
||||
|
||||
private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void
|
||||
{
|
||||
if ($this->adapter === null) {
|
||||
@@ -450,7 +807,7 @@ class Messaging extends Action
|
||||
}
|
||||
|
||||
|
||||
private function getSmsAdapter(Document $provider): ?SMSAdapter
|
||||
protected function getSmsAdapter(Document $provider): ?SMSAdapter
|
||||
{
|
||||
$credentials = $provider->getAttribute('credentials');
|
||||
|
||||
@@ -493,7 +850,7 @@ class Messaging extends Action
|
||||
};
|
||||
}
|
||||
|
||||
private function getPushAdapter(Document $provider): ?PushAdapter
|
||||
protected function getPushAdapter(Document $provider): ?PushAdapter
|
||||
{
|
||||
$credentials = $provider->getAttribute('credentials');
|
||||
$options = $provider->getAttribute('options');
|
||||
@@ -512,7 +869,7 @@ class Messaging extends Action
|
||||
};
|
||||
}
|
||||
|
||||
private function getEmailAdapter(Document $provider): ?EmailAdapter
|
||||
protected function getEmailAdapter(Document $provider): ?EmailAdapter
|
||||
{
|
||||
$credentials = $provider->getAttribute('credentials', []);
|
||||
$options = $provider->getAttribute('options', []);
|
||||
|
||||
@@ -0,0 +1,808 @@
|
||||
<?php
|
||||
|
||||
namespace Tests\Unit\Messaging;
|
||||
|
||||
use Appwrite\Platform\Workers\Messaging;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Messaging\Adapter\Email as EmailAdapter;
|
||||
use Utopia\Messaging\Messages\Email as EmailMessage;
|
||||
use Utopia\Messaging\Response;
|
||||
use Utopia\Storage\Device\Local;
|
||||
|
||||
/**
|
||||
* Pure unit tests for the messaging worker's fan-out building blocks: the retryable-error classifier, the
|
||||
* batch retry/backoff loop, and the cursor-paginated recipient stream. Each method is exercised directly
|
||||
* through reflection with no Swoole coroutines.
|
||||
*
|
||||
* The coroutine fan-out integration itself — the bounded-concurrency {@see \Utopia\Lock\Semaphore} driving
|
||||
* `Swoole\Coroutine\batch()` — is intentionally NOT covered here. Executing real Swoole coroutines inside the
|
||||
* shared unit process is unsafe: the in-process scheduler destabilises later, unrelated tests in the suite.
|
||||
* That integration is covered by the e2e Messaging suite instead. See PR #12465.
|
||||
*
|
||||
* Private methods are invoked via {@see \ReflectionMethod::invoke()} WITHOUT {@see \ReflectionMethod::setAccessible()}:
|
||||
* reflection invokes private methods directly on PHP 8.1+, and `setAccessible(true)` is deprecated on PHP 8.5
|
||||
* (the CI unit-test runtime), where the suite fails on triggered deprecations.
|
||||
*/
|
||||
class MessagingFanoutTest extends TestCase
|
||||
{
|
||||
private function provider(): Document
|
||||
{
|
||||
return new Document([
|
||||
'$id' => 'provider1',
|
||||
'$sequence' => '1',
|
||||
'enabled' => true,
|
||||
'type' => MESSAGE_TYPE_EMAIL,
|
||||
'provider' => 'mock',
|
||||
'name' => 'Mock Email',
|
||||
'credentials' => [],
|
||||
'options' => [
|
||||
'fromName' => 'Appwrite',
|
||||
'fromEmail' => 'noreply@example.com',
|
||||
],
|
||||
]);
|
||||
}
|
||||
|
||||
private function message(): Document
|
||||
{
|
||||
return new Document([
|
||||
'$id' => 'message1',
|
||||
'$sequence' => '1',
|
||||
'providerType' => MESSAGE_TYPE_EMAIL,
|
||||
'topics' => ['topic1'],
|
||||
'targets' => [],
|
||||
'users' => [],
|
||||
'data' => [
|
||||
'subject' => 'Hello',
|
||||
'content' => 'World',
|
||||
'html' => false,
|
||||
],
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Invoke the worker's private {@see Messaging::retrySend()} for an email batch.
|
||||
*
|
||||
* @param array<string> $batch
|
||||
* @return array{delivered: int, errors: array<string>}
|
||||
*/
|
||||
private function retrySend(array $batch, EmailAdapter $adapter, Database $database): array
|
||||
{
|
||||
$worker = new TestableMessaging();
|
||||
$method = new \ReflectionMethod(Messaging::class, 'retrySend');
|
||||
|
||||
$device = new Local(\sys_get_temp_dir());
|
||||
$project = new Document(['$id' => 'project1', '$sequence' => '1']);
|
||||
|
||||
/** @var array{delivered: int, errors: array<string>} $result */
|
||||
$result = $method->invoke(
|
||||
$worker,
|
||||
$batch,
|
||||
$this->message(),
|
||||
$this->provider(),
|
||||
MESSAGE_TYPE_EMAIL,
|
||||
$adapter,
|
||||
$database,
|
||||
$device,
|
||||
$project
|
||||
);
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
public function testIsRetryableErrorMatchesTransientFailures(): void
|
||||
{
|
||||
$worker = new TestableMessaging();
|
||||
$method = new \ReflectionMethod(Messaging::class, 'isRetryableError');
|
||||
|
||||
$retryable = [
|
||||
'Throttling',
|
||||
'rate exceeded',
|
||||
'Rate limit reached',
|
||||
'Too Many Requests',
|
||||
'429 Too Many Requests',
|
||||
'quota exceeded',
|
||||
'Service Unavailable',
|
||||
'503 Service Unavailable',
|
||||
'Connection timed out',
|
||||
'Request timeout',
|
||||
'Service temporarily unavailable',
|
||||
];
|
||||
|
||||
foreach ($retryable as $error) {
|
||||
$this->assertTrue(
|
||||
$method->invoke($worker, $error),
|
||||
"Expected '{$error}' to be classified as retryable"
|
||||
);
|
||||
}
|
||||
|
||||
$permanent = [
|
||||
'invalid recipient',
|
||||
'Mailbox does not exist',
|
||||
'Authentication failed',
|
||||
'Expired device token',
|
||||
];
|
||||
|
||||
foreach ($permanent as $error) {
|
||||
$this->assertFalse(
|
||||
$method->invoke($worker, $error),
|
||||
"Expected '{$error}' to be classified as permanent"
|
||||
);
|
||||
}
|
||||
|
||||
$this->assertFalse($method->invoke($worker, null), 'A null error must never be retryable');
|
||||
$this->assertFalse($method->invoke($worker, ''), 'An empty error must never be retryable');
|
||||
}
|
||||
|
||||
public function testThrottledBatchIsRetriedThenFullyDelivered(): void
|
||||
{
|
||||
$batch = ['user1@example.com', 'user2@example.com', 'user3@example.com', 'user4@example.com', 'user5@example.com'];
|
||||
|
||||
// First attempt throttles every recipient; the script then runs out so the second attempt succeeds.
|
||||
$adapter = new ScriptedEmailAdapter([
|
||||
static fn (string $recipient): string => 'Rate limit exceeded, please retry',
|
||||
]);
|
||||
|
||||
$result = $this->retrySend($batch, $adapter, new InMemoryDatabase());
|
||||
|
||||
// Retried exactly once, then everyone delivered with no double-counting and no terminal errors.
|
||||
$this->assertSame(2, $adapter->sendCalls, 'Throttled batch must be retried once');
|
||||
$this->assertSame(\count($batch), $result['delivered']);
|
||||
$this->assertEmpty($result['errors']);
|
||||
|
||||
// Both attempts targeted the full recipient set (none had succeeded on the first attempt).
|
||||
$this->assertSame($batch, $adapter->recipientsPerCall[0]);
|
||||
$this->assertSame($batch, $adapter->recipientsPerCall[1]);
|
||||
}
|
||||
|
||||
public function testRetryTargetsOnlyThrottledRecipients(): void
|
||||
{
|
||||
$batch = ['user1@example.com', 'user2@example.com', 'user3@example.com', 'user4@example.com', 'user5@example.com'];
|
||||
|
||||
// user2@ and user4@ throttle on the first attempt; everyone else succeeds. The second attempt has no
|
||||
// script step, so the two retried recipients then succeed.
|
||||
$throttled = ['user2@example.com', 'user4@example.com'];
|
||||
$adapter = new ScriptedEmailAdapter([
|
||||
static fn (string $recipient): ?string => \in_array($recipient, $throttled, true)
|
||||
? 'Too Many Requests'
|
||||
: null,
|
||||
]);
|
||||
|
||||
$result = $this->retrySend($batch, $adapter, new InMemoryDatabase());
|
||||
|
||||
$this->assertSame(2, $adapter->sendCalls, 'A partially throttled batch must be retried once');
|
||||
|
||||
// The retry must re-send to EXACTLY the throttled recipients — never the ones already delivered.
|
||||
$this->assertSame($throttled, $adapter->recipientsPerCall[1]);
|
||||
|
||||
// Every recipient is ultimately delivered and counted exactly once.
|
||||
$this->assertSame(\count($batch), $result['delivered']);
|
||||
$this->assertEmpty($result['errors']);
|
||||
}
|
||||
|
||||
public function testNonRetryableFailureIsNotRetried(): void
|
||||
{
|
||||
$batch = ['user1@example.com', 'user2@example.com', 'user3@example.com'];
|
||||
|
||||
// user1@ permanently fails with a non-retryable error; the rest succeed.
|
||||
$adapter = new ScriptedEmailAdapter([
|
||||
static fn (string $recipient): ?string => $recipient === 'user1@example.com'
|
||||
? 'Invalid recipient address'
|
||||
: null,
|
||||
]);
|
||||
|
||||
$result = $this->retrySend($batch, $adapter, new InMemoryDatabase());
|
||||
|
||||
// A permanent failure is recorded immediately, never retried.
|
||||
$this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry');
|
||||
$this->assertSame(\count($batch) - 1, $result['delivered']);
|
||||
$this->assertCount(1, $result['errors']);
|
||||
$this->assertStringContainsString('user1@example.com', $result['errors'][0]);
|
||||
$this->assertStringContainsString('Invalid recipient address', $result['errors'][0]);
|
||||
}
|
||||
|
||||
public function testExhaustedRetriesRecordTerminalFailure(): void
|
||||
{
|
||||
$batch = ['user1@example.com', 'user2@example.com'];
|
||||
|
||||
// Every attempt throttles, for more steps than MESSAGE_SEND_MAX_RETRIES, so retries are exhausted.
|
||||
$script = \array_fill(
|
||||
0,
|
||||
MESSAGE_SEND_MAX_RETRIES + 2,
|
||||
static fn (string $recipient): string => 'Service unavailable, throttled'
|
||||
);
|
||||
$adapter = new ScriptedEmailAdapter($script);
|
||||
|
||||
$result = $this->retrySend($batch, $adapter, new InMemoryDatabase());
|
||||
|
||||
// The batch is attempted exactly MESSAGE_SEND_MAX_RETRIES times, then the still-throttled recipients
|
||||
// become terminal failures with nothing delivered.
|
||||
$this->assertSame(MESSAGE_SEND_MAX_RETRIES, $adapter->sendCalls, 'Attempts must be capped at MESSAGE_SEND_MAX_RETRIES');
|
||||
$this->assertSame(0, $result['delivered']);
|
||||
$this->assertCount(\count($batch), $result['errors']);
|
||||
}
|
||||
|
||||
public function testDeliveryErrorsAreCappedWhileDeliveredCountIsExact(): void
|
||||
{
|
||||
// More failing recipients than the retained-error cap, plus a known number of successes, so the
|
||||
// delivered tally stays exact even though the error list is truncated.
|
||||
$failingCount = MESSAGE_DELIVERY_ERRORS_LIMIT + 20;
|
||||
$succeedingCount = 100;
|
||||
|
||||
$batch = [];
|
||||
for ($i = 1; $i <= $failingCount; $i++) {
|
||||
$batch[] = "fail{$i}@example.com";
|
||||
}
|
||||
for ($i = 1; $i <= $succeedingCount; $i++) {
|
||||
$batch[] = "ok{$i}@example.com";
|
||||
}
|
||||
|
||||
// Permanent (non-retryable) failure for every "fail*" recipient; everyone else is delivered.
|
||||
$adapter = new ScriptedEmailAdapter([
|
||||
static fn (string $recipient): ?string => \str_starts_with($recipient, 'fail')
|
||||
? 'Invalid recipient address'
|
||||
: null,
|
||||
]);
|
||||
|
||||
$result = $this->retrySend($batch, $adapter, new InMemoryDatabase());
|
||||
|
||||
// Errors are capped, but the delivered count reflects every success exactly.
|
||||
$this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $result['errors'], 'Delivery errors must be capped');
|
||||
$this->assertSame($succeedingCount, $result['delivered'], 'Delivered count stays exact despite the error cap');
|
||||
$this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry');
|
||||
}
|
||||
|
||||
public function testExpiredTokenCleanupFailureDoesNotCorruptAccounting(): void
|
||||
{
|
||||
$batch = ['user1@example.com', 'user2@example.com', 'user3@example.com'];
|
||||
|
||||
// The cleanup path always throws, but the worker must isolate it: the send result is what's accounted.
|
||||
$database = new ExpiredTokenCleanupFailingDatabase();
|
||||
|
||||
// user1@ comes back with an expired device token (triggering cleanup, which then throws); the other
|
||||
// two recipients are delivered. The expired-token error is terminal, never retryable.
|
||||
$adapter = new ScriptedEmailAdapter([
|
||||
static fn (string $recipient): ?string => $recipient === 'user1@example.com'
|
||||
? 'Expired device token'
|
||||
: null,
|
||||
]);
|
||||
|
||||
$result = $this->retrySend($batch, $adapter, $database);
|
||||
|
||||
// The worker reached the cleanup (so the isolation is genuinely exercised) and it threw.
|
||||
$this->assertGreaterThan(0, $database->targetUpdateAttempts, 'Expired-token cleanup must have been attempted');
|
||||
|
||||
// The send was attempted exactly once: a thrown cleanup must not look like a transient send failure
|
||||
// and must not trigger a retry.
|
||||
$this->assertSame(1, $adapter->sendCalls, 'Cleanup failure must not trigger a send retry');
|
||||
|
||||
// Accounting reflects the send result, not the cleanup throw: two delivered, one terminal failure.
|
||||
$this->assertSame(\count($batch) - 1, $result['delivered']);
|
||||
|
||||
// The recorded error is the provider's expired-token failure for the real recipient — proving the
|
||||
// database exception was swallowed rather than misattributed as the send error.
|
||||
$this->assertCount(1, $result['errors']);
|
||||
$this->assertStringContainsString('user1@example.com', $result['errors'][0]);
|
||||
$this->assertStringContainsString('Expired device token', $result['errors'][0]);
|
||||
$this->assertStringNotContainsString('Transient database error', $result['errors'][0]);
|
||||
}
|
||||
|
||||
public function testRecipientsArePaginatedWithoutSubqueryFilter(): void
|
||||
{
|
||||
// Always span several pages regardless of the configured page size, so cursor pagination is exercised.
|
||||
$count = MESSAGE_RECIPIENTS_PAGE_SIZE * 2 + 500;
|
||||
$data = $this->topicDataset($count);
|
||||
$database = new RecordingDatabase($data['subscribers'], $data['targets']);
|
||||
|
||||
$identifiers = $this->collectStream($database, topicIds: ['topic1']);
|
||||
|
||||
$topicReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics'));
|
||||
$subscriberReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'subscribers'));
|
||||
|
||||
// The topic's `targets` attribute (the subQueryTopicTargets filter) is never materialized: topics are
|
||||
// only read with a $sequence projection, and recipients come from the subscribers collection instead.
|
||||
$this->assertNotEmpty($topicReads);
|
||||
foreach ($topicReads as $call) {
|
||||
$this->assertNotContains('targets', $this->selectFields($call['queries']), 'Topic read must not select the targets subquery attribute');
|
||||
}
|
||||
|
||||
// Subscribers are walked in bounded pages via cursorAfter; no query ever requests a runaway page size.
|
||||
$expectedPages = (int) \ceil($count / MESSAGE_RECIPIENTS_PAGE_SIZE);
|
||||
$this->assertCount($expectedPages, $subscriberReads);
|
||||
|
||||
foreach ($subscriberReads as $index => $call) {
|
||||
$this->assertSame(MESSAGE_RECIPIENTS_PAGE_SIZE, $this->limitOf($call['queries']));
|
||||
$this->assertSame('100', $this->topicInternalIdOf($call['queries']), 'Subscribers must be filtered by the topic internal id');
|
||||
|
||||
if ($index === 0) {
|
||||
$this->assertFalse($this->hasCursor($call['queries']), 'First page must not carry a cursor');
|
||||
} else {
|
||||
$this->assertTrue($this->hasCursor($call['queries']), 'Subsequent pages must paginate with cursorAfter');
|
||||
}
|
||||
}
|
||||
|
||||
// No query in the whole stream issues a runaway (>= 1,000,000) limit.
|
||||
foreach ($database->findCalls as $call) {
|
||||
$this->assertLessThan(1_000_000, $this->limitOf($call['queries']), 'No query may request a million-row page');
|
||||
}
|
||||
|
||||
// The duplicate identifier collapses: one row per distinct subscriber identifier resolved.
|
||||
$this->assertCount($count, $identifiers);
|
||||
for ($i = 1; $i <= $count; $i++) {
|
||||
$this->assertArrayHasKey("user{$i}@example.com", $identifiers);
|
||||
}
|
||||
}
|
||||
|
||||
public function testStreamDeduplicatesIdentifiersWithinPage(): void
|
||||
{
|
||||
// Two subscribers in a single page resolve to targets sharing one identifier; the page must collapse
|
||||
// them to a single recipient entry.
|
||||
$subscribers = [
|
||||
new Document(['$id' => 'sub1', '$sequence' => '1', 'topicInternalId' => '100', 'targetInternalId' => '1', 'providerType' => MESSAGE_TYPE_EMAIL]),
|
||||
new Document(['$id' => 'sub2', '$sequence' => '2', 'topicInternalId' => '100', 'targetInternalId' => '2', 'providerType' => MESSAGE_TYPE_EMAIL]),
|
||||
];
|
||||
$targets = [
|
||||
'1' => new Document(['$id' => 'target1', '$sequence' => '1', 'providerId' => null, 'identifier' => 'dup@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]),
|
||||
'2' => new Document(['$id' => 'target2', '$sequence' => '2', 'providerId' => null, 'identifier' => 'dup@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]),
|
||||
];
|
||||
$database = new RecordingDatabase($subscribers, $targets);
|
||||
|
||||
$pages = $this->collectPages($database, topicIds: ['topic1']);
|
||||
|
||||
$this->assertCount(1, $pages, 'A single subscriber page yields a single grouped result');
|
||||
$providerId = $this->provider()->getId();
|
||||
$this->assertArrayHasKey($providerId, $pages[0]);
|
||||
$this->assertSame(['dup@example.com'], \array_keys($pages[0][$providerId]), 'Duplicate identifiers must collapse to one');
|
||||
}
|
||||
|
||||
public function testStreamResolvesUsersAndDirectTargets(): void
|
||||
{
|
||||
$userTargets = [
|
||||
new Document(['$id' => 'utarget1', '$sequence' => '10', 'providerId' => null, 'identifier' => 'byuser@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]),
|
||||
];
|
||||
$directTargets = [
|
||||
new Document(['$id' => 'dtarget1', '$sequence' => '20', 'providerId' => null, 'identifier' => 'bytarget@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]),
|
||||
];
|
||||
$database = new RecordingDatabase([], [], $userTargets, $directTargets);
|
||||
|
||||
$identifiers = $this->collectStream($database, userIds: ['user-a'], targetIds: ['target-b']);
|
||||
|
||||
// No topics in play: only the user and direct-target queries run, and both identifiers resolve.
|
||||
$this->assertSame([], \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics')));
|
||||
$this->assertArrayHasKey('byuser@example.com', $identifiers);
|
||||
$this->assertArrayHasKey('bytarget@example.com', $identifiers);
|
||||
$this->assertCount(2, $identifiers);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a topic-backed dataset: $count subscribers, each pointing at one email target. A single identifier
|
||||
* is duplicated so per-page dedup is exercised when the dataset fits in one page.
|
||||
*
|
||||
* @return array{subscribers: array<Document>, targets: array<string, Document>}
|
||||
*/
|
||||
private function topicDataset(int $count): array
|
||||
{
|
||||
$subscribers = [];
|
||||
$targets = [];
|
||||
|
||||
for ($i = 1; $i <= $count; $i++) {
|
||||
$sequence = (string) $i;
|
||||
$subscribers[] = new Document([
|
||||
'$id' => 'sub' . $i,
|
||||
'$sequence' => $sequence,
|
||||
'topicInternalId' => '100',
|
||||
'targetInternalId' => $sequence,
|
||||
'providerType' => MESSAGE_TYPE_EMAIL,
|
||||
]);
|
||||
$targets[$sequence] = new Document([
|
||||
'$id' => 'target' . $i,
|
||||
'$sequence' => $sequence,
|
||||
'providerId' => null,
|
||||
'identifier' => "user{$i}@example.com",
|
||||
'providerType' => MESSAGE_TYPE_EMAIL,
|
||||
]);
|
||||
}
|
||||
|
||||
return ['subscribers' => $subscribers, 'targets' => $targets];
|
||||
}
|
||||
|
||||
/**
|
||||
* Drive {@see Messaging::streamRecipients()} and merge every yielded page into a single identifier map.
|
||||
*
|
||||
* @param array<string> $topicIds
|
||||
* @param array<string> $userIds
|
||||
* @param array<string> $targetIds
|
||||
* @return array<string, null>
|
||||
*/
|
||||
private function collectStream(
|
||||
RecordingDatabase $database,
|
||||
array $topicIds = [],
|
||||
array $userIds = [],
|
||||
array $targetIds = []
|
||||
): array {
|
||||
$identifiers = [];
|
||||
|
||||
foreach ($this->collectPages($database, $topicIds, $userIds, $targetIds) as $page) {
|
||||
foreach ($page as $perProvider) {
|
||||
foreach ($perProvider as $identifier => $_) {
|
||||
$identifiers[$identifier] = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $identifiers;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string> $topicIds
|
||||
* @param array<string> $userIds
|
||||
* @param array<string> $targetIds
|
||||
* @return array<array<string, array<string, null>>>
|
||||
*/
|
||||
private function collectPages(
|
||||
RecordingDatabase $database,
|
||||
array $topicIds = [],
|
||||
array $userIds = [],
|
||||
array $targetIds = []
|
||||
): array {
|
||||
$method = new \ReflectionMethod(Messaging::class, 'streamRecipients');
|
||||
$worker = new TestableMessaging();
|
||||
|
||||
/** @var \Generator<array<string, array<string, null>>> $generator */
|
||||
$generator = $method->invoke(
|
||||
$worker,
|
||||
$database,
|
||||
$topicIds,
|
||||
$userIds,
|
||||
$targetIds,
|
||||
MESSAGE_TYPE_EMAIL,
|
||||
$this->provider()
|
||||
);
|
||||
|
||||
return \iterator_to_array($generator, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Query> $queries
|
||||
* @return array<string>
|
||||
*/
|
||||
private function selectFields(array $queries): array
|
||||
{
|
||||
foreach ($queries as $query) {
|
||||
if ($query->getMethod() === Query::TYPE_SELECT) {
|
||||
return $query->getValues();
|
||||
}
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Query> $queries
|
||||
*/
|
||||
private function limitOf(array $queries): int
|
||||
{
|
||||
foreach ($queries as $query) {
|
||||
if ($query->getMethod() === Query::TYPE_LIMIT) {
|
||||
return (int) ($query->getValues()[0] ?? 0);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Query> $queries
|
||||
*/
|
||||
private function hasCursor(array $queries): bool
|
||||
{
|
||||
foreach ($queries as $query) {
|
||||
if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Query> $queries
|
||||
*/
|
||||
private function topicInternalIdOf(array $queries): ?string
|
||||
{
|
||||
foreach ($queries as $query) {
|
||||
if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'topicInternalId') {
|
||||
return (string) ($query->getValues()[0] ?? null);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Worker subclass that zeroes the retry backoff so the throttle/retry tests stay instant. The worker already
|
||||
* skips a non-positive delay, so no coroutine sleep is ever reached.
|
||||
*/
|
||||
class TestableMessaging extends Messaging
|
||||
{
|
||||
protected function retryDelay(): float
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Email adapter double whose per-attempt outcome is scripted. Each {@see process()} call records the exact
|
||||
* `to` recipients it received and consults the next scripted step, which either throws (simulating a
|
||||
* connection-level provider error) or returns a per-recipient success/failure verdict. When the script runs
|
||||
* out, every recipient succeeds — modelling a provider that has stopped throttling.
|
||||
*/
|
||||
class ScriptedEmailAdapter extends EmailAdapter
|
||||
{
|
||||
public int $sendCalls = 0;
|
||||
|
||||
/**
|
||||
* @var array<array<string>> Recipient email lists captured per {@see process()} call, in call order.
|
||||
*/
|
||||
public array $recipientsPerCall = [];
|
||||
|
||||
/**
|
||||
* @param array<\Throwable|(callable(string): ?string)> $script One step per attempt. A Throwable is thrown
|
||||
* for that attempt; a callable returns an error string for a failing recipient or null for success.
|
||||
*/
|
||||
public function __construct(private readonly array $script = [])
|
||||
{
|
||||
}
|
||||
|
||||
public function getName(): string
|
||||
{
|
||||
return 'ScriptedEmail';
|
||||
}
|
||||
|
||||
public function getMaxMessagesPerRequest(): int
|
||||
{
|
||||
return 1000;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param EmailMessage $message
|
||||
* @return array{deliveredTo: int, type: string, results: array<array<string, mixed>>}
|
||||
*/
|
||||
protected function process(EmailMessage $message): array
|
||||
{
|
||||
$recipients = \array_map(static fn (array $entry): string => $entry['email'], $message->getTo());
|
||||
$this->recipientsPerCall[] = $recipients;
|
||||
|
||||
$step = $this->script[$this->sendCalls] ?? null;
|
||||
$this->sendCalls++;
|
||||
|
||||
if ($step instanceof \Throwable) {
|
||||
throw $step;
|
||||
}
|
||||
|
||||
$response = new Response($this->getType());
|
||||
$delivered = 0;
|
||||
|
||||
foreach ($recipients as $recipient) {
|
||||
$error = \is_callable($step) ? $step($recipient) : null;
|
||||
|
||||
if ($error === null) {
|
||||
$delivered++;
|
||||
$response->addResult($recipient);
|
||||
} else {
|
||||
$response->addResult($recipient, $error);
|
||||
}
|
||||
}
|
||||
|
||||
$response->setDeliveredTo($delivered);
|
||||
|
||||
return $response->toArray();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal in-memory {@see Database} double for the retry tests, which only build email messages with no
|
||||
* attachments and never touch the expired-token cleanup path. Every read resolves empty.
|
||||
*/
|
||||
class InMemoryDatabase extends Database
|
||||
{
|
||||
private readonly Authorization $authorization;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->authorization = new Authorization();
|
||||
}
|
||||
|
||||
public function getAuthorization(): Authorization
|
||||
{
|
||||
return $this->authorization;
|
||||
}
|
||||
|
||||
public function skipValidation(callable $callback): mixed
|
||||
{
|
||||
return $callback();
|
||||
}
|
||||
|
||||
public function findOne(string $collection, array $queries = []): Document
|
||||
{
|
||||
return new Document();
|
||||
}
|
||||
|
||||
public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): Document
|
||||
{
|
||||
return new Document();
|
||||
}
|
||||
|
||||
public function updateDocument(string $collection, string $id, Document $document): Document
|
||||
{
|
||||
return $document;
|
||||
}
|
||||
|
||||
public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array
|
||||
{
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* {@see InMemoryDatabase} variant whose expired-device-token cleanup always fails: `findOne('targets')`
|
||||
* resolves to a real target so the worker attempts the update, and `updateDocument('targets')` throws a
|
||||
* transient error. Used to prove that a DB hiccup during best-effort cleanup never leaks into delivery
|
||||
* accounting or the retry decision.
|
||||
*/
|
||||
class ExpiredTokenCleanupFailingDatabase extends InMemoryDatabase
|
||||
{
|
||||
public int $targetUpdateAttempts = 0;
|
||||
|
||||
public function findOne(string $collection, array $queries = []): Document
|
||||
{
|
||||
if ($collection === 'targets') {
|
||||
return new Document(['$id' => 'target-expired', '$sequence' => '1']);
|
||||
}
|
||||
|
||||
return parent::findOne($collection, $queries);
|
||||
}
|
||||
|
||||
public function updateDocument(string $collection, string $id, Document $document): Document
|
||||
{
|
||||
if ($collection === 'targets') {
|
||||
$this->targetUpdateAttempts++;
|
||||
|
||||
throw new \RuntimeException('Transient database error during target cleanup');
|
||||
}
|
||||
|
||||
return parent::updateDocument($collection, $id, $document);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* In-memory {@see Database} double for the streaming tests. Serves subscribers/targets through cursor
|
||||
* pagination and records every query issued so the test can assert the streaming contract (cursorAfter +
|
||||
* bounded page size, topic reads never selecting the 1M-row `targets` subquery, no runaway limits).
|
||||
*/
|
||||
class RecordingDatabase extends Database
|
||||
{
|
||||
/**
|
||||
* @var array<array{collection: string, queries: array<Query>}>
|
||||
*/
|
||||
public array $findCalls = [];
|
||||
|
||||
private readonly Authorization $authorization;
|
||||
|
||||
/**
|
||||
* @param array<Document> $subscribers Subscriber rows (carry $sequence + targetInternalId + providerType).
|
||||
* @param array<string, Document> $targets Target rows keyed by $sequence (resolved from a subscriber page).
|
||||
* @param array<Document> $userTargets Targets addressed by userId.
|
||||
* @param array<Document> $directTargets Targets addressed directly by id.
|
||||
*/
|
||||
public function __construct(
|
||||
private readonly array $subscribers = [],
|
||||
private readonly array $targets = [],
|
||||
private readonly array $userTargets = [],
|
||||
private readonly array $directTargets = []
|
||||
) {
|
||||
$this->authorization = new Authorization();
|
||||
}
|
||||
|
||||
public function getAuthorization(): Authorization
|
||||
{
|
||||
return $this->authorization;
|
||||
}
|
||||
|
||||
public function skipValidation(callable $callback): mixed
|
||||
{
|
||||
return $callback();
|
||||
}
|
||||
|
||||
public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array
|
||||
{
|
||||
$this->findCalls[] = ['collection' => $collection, 'queries' => $queries];
|
||||
|
||||
return match ($collection) {
|
||||
'topics' => [new Document(['$id' => 'topic1', '$sequence' => '100'])],
|
||||
'subscribers' => $this->paginate($this->subscribers, $queries),
|
||||
'targets' => $this->resolveTargets($queries),
|
||||
default => [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Document> $rows
|
||||
* @param array<Query> $queries
|
||||
* @return array<Document>
|
||||
*/
|
||||
private function paginate(array $rows, array $queries): array
|
||||
{
|
||||
$limit = $this->limitOf($queries);
|
||||
$cursor = $this->cursorOf($queries);
|
||||
|
||||
if ($cursor !== null) {
|
||||
$rows = \array_values(\array_filter(
|
||||
$rows,
|
||||
fn (Document $row) => $row->getSequence() > $cursor->getSequence()
|
||||
));
|
||||
}
|
||||
|
||||
return \array_slice($rows, 0, $limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Either resolve a subscriber page's targets by $sequence, or paginate user/direct targets.
|
||||
*
|
||||
* @param array<Query> $queries
|
||||
* @return array<Document>
|
||||
*/
|
||||
private function resolveTargets(array $queries): array
|
||||
{
|
||||
foreach ($queries as $query) {
|
||||
if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === '$sequence') {
|
||||
$sequences = $query->getValues();
|
||||
|
||||
return \array_values(\array_map(
|
||||
fn ($sequence) => $this->targets[$sequence],
|
||||
\array_filter($sequences, fn ($sequence) => isset($this->targets[$sequence]))
|
||||
));
|
||||
}
|
||||
|
||||
if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'userId') {
|
||||
return $this->paginate($this->userTargets, $queries);
|
||||
}
|
||||
}
|
||||
|
||||
return $this->paginate($this->directTargets, $queries);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Query> $queries
|
||||
*/
|
||||
private function limitOf(array $queries): int
|
||||
{
|
||||
foreach ($queries as $query) {
|
||||
if ($query->getMethod() === Query::TYPE_LIMIT) {
|
||||
return (int) ($query->getValues()[0] ?? 0);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Query> $queries
|
||||
*/
|
||||
private function cursorOf(array $queries): ?Document
|
||||
{
|
||||
foreach ($queries as $query) {
|
||||
if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) {
|
||||
$value = $query->getValues()[0] ?? null;
|
||||
|
||||
return $value instanceof Document ? $value : null;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user