Merge pull request #12465 from appwrite/feat-messaging-scale-fanout-19x

perf(messaging): stream large-topic sends and bound fan-out concurrency
This commit is contained in:
Jake Barnby authored and GitHub committed 2026-06-03 14:36:54 +12:00
commit a6ead889ac
3 files changed
+1348 -177

No files matched your search

+6
View File
@@ -242,6 +242,12 @@ const RULE_STATUS_VERIFIED = 'verified';
// Message types
const MESSAGE_SEND_TYPE_INTERNAL = 'internal';
const MESSAGE_SEND_TYPE_EXTERNAL = 'external';
// External message fan-out tuning
const MESSAGE_RECIPIENTS_PAGE_SIZE = 5000; // Recipients resolved from the database per page while streaming a send; aligned with APP_LIMIT_COUNT to minimise round-trips on large topics
const MESSAGE_SEND_CONCURRENCY = 10; // Maximum adapter send requests running concurrently within a single send job
const MESSAGE_DELIVERY_ERRORS_LIMIT = 100; // Maximum number of per-recipient delivery errors retained on a message
const MESSAGE_SEND_MAX_RETRIES = 5; // Maximum number of attempts to deliver a batch when the provider throttles or fails transiently
const MESSAGE_SEND_RETRY_DELAY = 1.0; // Base seconds for exponential backoff between batch send retries; overridable in tests for an instant suite
// Mail Types
const MAIL_TYPE_VERIFICATION = 'verification';
const MAIL_TYPE_MAGIC_SESSION = 'magicSession';
+534 -177
View File
@@ -16,6 +16,7 @@ use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\DSN\DSN;
use Utopia\Lock\Semaphore;
use Utopia\Logger\Log;
use Utopia\Messaging\Adapter\Email as EmailAdapter;
use Utopia\Messaging\Adapter\Email\Mailgun;
@@ -129,69 +130,19 @@ class Messaging extends Action
Document $project,
UsagePublisher $publisherForUsage
): void {
$status = $message->getAttribute('status');
// Idempotency guard: queue redelivery can hand us a message that already finished.
if (\in_array($status, [MessageStatus::SENT, MessageStatus::FAILED], true)) {
Span::add('message.skipped', 'already_processed');
return;
}
$topicIds = $message->getAttribute('topics', []);
$targetIds = $message->getAttribute('targets', []);
$userIds = $message->getAttribute('users', []);
$providerType = $message->getAttribute('providerType');
/**
* @var array<Document> $allTargets
*/
$allTargets = [];
if (\count($topicIds) > 0) {
$topics = $dbForProject->find('topics', [
Query::equal('$id', $topicIds),
Query::limit(\count($topicIds)),
]);
foreach ($topics as $topic) {
$targets = \array_filter($topic->getAttribute('targets'), function (Document $target) use ($providerType) {
return $target->getAttribute('providerType') === $providerType;
});
\array_push($allTargets, ...$targets);
}
}
if (\count($userIds) > 0) {
$limit = 1000;
$offset = 0;
do {
$targets = $dbForProject->find('targets', [
Query::equal('userId', $userIds),
Query::select(['providerId', 'identifier']),
Query::equal('providerType', [$providerType]),
Query::limit($limit),
Query::offset($offset),
]);
\array_push($allTargets, ...$targets);
$offset += \count($targets);
} while (\count($targets) === $limit);
}
if (\count($targetIds) > 0) {
$targets = $dbForProject->find('targets', [
Query::select(['providerId', 'identifier']),
Query::equal('$id', $targetIds),
Query::equal('providerType', [$providerType]),
Query::limit(\count($targetIds)),
]);
\array_push($allTargets, ...$targets);
}
if (empty($allTargets)) {
$dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([
'status' => MessageStatus::FAILED,
'deliveryErrors' => ['No valid recipients found.']
]));
Span::add('message.skipped', 'no_valid_recipients');
return;
}
$default = $dbForProject->findOne('providers', [
Query::equal('enabled', [true]),
Query::equal('type', [$providerType]),
@@ -208,55 +159,33 @@ class Messaging extends Action
}
/**
* @var array<string, array<string, null>> $identifiers
*/
$identifiers = [];
/**
* @var array<Document> $providers
* Resolved providers cached for the lifetime of this job, keyed by provider id.
* Seeded with the default provider so most sends never touch the providers collection.
*
* @var array<string, Document> $providers
*/
$providers = [
$default->getId() => $default
$default->getId() => $default,
];
foreach ($allTargets as $target) {
$providerId = $target->getAttribute('providerId');
$semaphore = new Semaphore(MESSAGE_SEND_CONCURRENCY);
if (!$providerId) {
$providerId = $default->getId();
}
$deliveredTotal = 0;
$failedTotal = 0;
$deliveryErrors = [];
$hasRecipients = false;
if ($providerId) {
if (!\array_key_exists($providerId, $identifiers)) {
$identifiers[$providerId] = [];
}
// Use null as value to avoid duplicate keys
$identifiers[$providerId][$target->getAttribute('identifier')] = null;
}
}
foreach ($this->streamRecipients($dbForProject, $topicIds, $userIds, $targetIds, $providerType, $default) as $page) {
/**
* @var array<callable> $tasks
*/
$tasks = [];
/**
* @var array<array> $results
*/
$results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
if (\array_key_exists($providerId, $providers)) {
$provider = $providers[$providerId];
} else {
$provider = $dbForProject->getDocument('providers', $providerId);
foreach ($page as $providerId => $identifiers) {
$provider = $this->resolveProvider($dbForProject, $providerId, $providers, $default);
$resolvedProviderType = $provider->getAttribute('type');
if ($provider->isEmpty() || !$provider->getAttribute('enabled')) {
$provider = $default;
} else {
$providers[$providerId] = $provider;
}
}
$identifiersForProvider = $identifiers[$providerId];
$providerType = $provider->getAttribute('type');
$adapter = match ($providerType) {
$adapter = match ($resolvedProviderType) {
MESSAGE_TYPE_SMS => $this->getSmsAdapter($provider),
MESSAGE_TYPE_PUSH => $this->getPushAdapter($provider),
MESSAGE_TYPE_EMAIL => $this->getEmailAdapter($provider),
@@ -264,101 +193,74 @@ class Messaging extends Action
};
$batches = \array_chunk(
\array_keys($identifiersForProvider),
\array_keys($identifiers),
$adapter->getMaxMessagesPerRequest()
);
return batch(\array_map(function ($batch) use ($message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
return function () use ($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
$deliveredTotal = 0;
$deliveryErrors = [];
$messageData = clone $message;
$messageData->setAttribute('to', $batch);
foreach ($batches as $batch) {
$tasks[] = fn (): array => $semaphore->withLock(
fn (): array => $this->sendBatch(
$batch,
$message,
$provider,
$resolvedProviderType,
$adapter,
$dbForProject,
$deviceForFiles,
$project,
$publisherForUsage
)
);
}
}
$data = match ($providerType) {
MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider),
MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData),
MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project),
};
// A page can resolve to zero identifiers (e.g. subscribers whose targetInternalId matches no targets
// row), so an empty task list must not count as recipients or run batch() for nothing.
if (empty($tasks)) {
continue;
}
try {
$response = $adapter->send($data);
$deliveredTotal += (int) $response['deliveredTo'];
foreach ($response['results'] as $result) {
if ($result['status'] === 'failure') {
$deliveryErrors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}";
}
$hasRecipients = true;
// Deleting push targets when token has expired.
if (($result['error'] ?? '') === 'Expired device token') {
$target = $dbForProject->findOne('targets', [
Query::equal('identifier', [$result['recipient']])
]);
/**
* @var array<array{delivered: int, recipients: int, errors: array<string>}> $results
*/
$results = batch($tasks);
if (!$target->isEmpty()) {
$dbForProject->updateDocument(
'targets',
$target->getId(),
$target->setAttribute('expired', true)
);
}
}
}
} catch (\Throwable $e) {
$deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage();
} finally {
$errorTotal = \count($deliveryErrors);
$usage = new UsageContext();
$usage
->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal))
->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal)
->addMetric(METRIC_MESSAGES_FAILED, $errorTotal)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), ($deliveredTotal + $errorTotal))
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $deliveredTotal)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $errorTotal)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), ($deliveredTotal + $errorTotal))
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $deliveredTotal)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal);
foreach ($results as $result) {
$deliveredTotal += $result['delivered'];
$failedTotal += $result['recipients'] - $result['delivered'];
$publisherForUsage->enqueue(new Usage(
project: $project,
metrics: $usage->getMetrics(),
));
foreach ($result['errors'] as $error) {
if (\count($deliveryErrors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) {
break;
}
return [
'deliveredTotal' => $deliveredTotal,
'deliveryErrors' => $deliveryErrors,
];
}
};
}, $batches));
};
}, \array_keys($identifiers)));
$deliveryErrors[] = $error;
}
}
}
$results = \array_merge(...$results);
if (!$hasRecipients) {
$dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([
'status' => MessageStatus::FAILED,
'deliveryErrors' => ['No valid recipients found.']
]));
$deliveredTotal = 0;
$deliveryErrors = [];
foreach ($results as $result) {
$deliveredTotal += $result['deliveredTotal'];
$deliveryErrors = \array_merge($deliveryErrors, $result['deliveryErrors']);
Span::add('message.skipped', 'no_valid_recipients');
return;
}
if (empty($deliveryErrors) && $deliveredTotal === 0) {
$deliveryErrors[] = 'Unknown error';
}
$hasFailures = $failedTotal > 0 || \count($deliveryErrors) > 0;
$message->setAttribute('status', $hasFailures ? MessageStatus::FAILED : MessageStatus::SENT);
$message->setAttribute('deliveryErrors', $deliveryErrors);
if (\count($message->getAttribute('deliveryErrors')) > 0) {
$message->setAttribute('status', MessageStatus::FAILED);
} else {
$message->setAttribute('status', MessageStatus::SENT);
}
Span::add('message.delivered_total', $deliveredTotal);
Span::add('message.errors_total', \count($deliveryErrors));
Span::add('message.errors_total', $failedTotal);
$message->removeAttribute('to');
@@ -409,6 +311,461 @@ class Messaging extends Action
}
}
/**
* Stream a message's recipients in bounded pages, grouped by provider and deduplicated by identifier within each page.
*
* Peak memory is O(MESSAGE_RECIPIENTS_PAGE_SIZE), never O(topic size): topics are walked through the
* subscribers collection with cursor pagination rather than reading the topic's `targets` attribute, which
* triggers the subQueryTopicTargets filter and loads up to APP_LIMIT_SUBSCRIBERS_SUBQUERY rows at once.
*
* @param array<string> $topicIds
* @param array<string> $userIds
* @param array<string> $targetIds
* @return \Generator<array<string, array<string, null>>>
* @throws \Exception
*/
private function streamRecipients(
Database $dbForProject,
array $topicIds,
array $userIds,
array $targetIds,
string $providerType,
Document $default
): \Generator {
if (\count($topicIds) > 0) {
$topics = $dbForProject->find('topics', [
Query::select(['$sequence']),
Query::equal('$id', $topicIds),
Query::limit(\count($topicIds)),
]);
foreach ($topics as $topic) {
$cursor = null;
do {
$queries = [
Query::equal('topicInternalId', [$topic->getSequence()]),
Query::equal('providerType', [$providerType]),
Query::select(['$sequence', 'targetInternalId']),
Query::orderAsc('$sequence'),
Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE),
];
if ($cursor !== null) {
$queries[] = Query::cursorAfter($cursor);
}
$subscribers = $dbForProject->getAuthorization()->skip(
fn () => $dbForProject->find('subscribers', $queries)
);
$count = \count($subscribers);
if ($count === 0) {
break;
}
$cursor = $subscribers[$count - 1];
$targetInternalIds = \array_map(
fn (Document $subscriber) => $subscriber->getAttribute('targetInternalId'),
$subscribers
);
$targets = $dbForProject->skipValidation(
fn () => $dbForProject->getAuthorization()->skip(
fn () => $dbForProject->find('targets', [
Query::equal('$sequence', $targetInternalIds),
Query::select(['providerId', 'identifier']),
Query::limit(\count($targetInternalIds)),
])
)
);
yield $this->groupTargetsByProvider($targets, $default);
} while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE);
}
}
if (\count($userIds) > 0) {
$cursor = null;
do {
$queries = [
Query::equal('userId', $userIds),
Query::equal('providerType', [$providerType]),
Query::select(['$sequence', 'providerId', 'identifier']),
Query::orderAsc('$sequence'),
Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE),
];
if ($cursor !== null) {
$queries[] = Query::cursorAfter($cursor);
}
$targets = $dbForProject->find('targets', $queries);
$count = \count($targets);
if ($count === 0) {
break;
}
$cursor = $targets[$count - 1];
yield $this->groupTargetsByProvider($targets, $default);
} while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE);
}
if (\count($targetIds) > 0) {
$cursor = null;
do {
$queries = [
Query::equal('$id', $targetIds),
Query::equal('providerType', [$providerType]),
Query::select(['$sequence', 'providerId', 'identifier']),
Query::orderAsc('$sequence'),
Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE),
];
if ($cursor !== null) {
$queries[] = Query::cursorAfter($cursor);
}
$targets = $dbForProject->find('targets', $queries);
$count = \count($targets);
if ($count === 0) {
break;
}
$cursor = $targets[$count - 1];
yield $this->groupTargetsByProvider($targets, $default);
} while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE);
}
}
/**
* Group a page of target documents by provider id, deduplicating identifiers within the page.
*
* @param array<Document> $targets
* @return array<string, array<string, null>>
*/
private function groupTargetsByProvider(array $targets, Document $default): array
{
/**
* @var array<string, array<string, null>> $identifiers
*/
$identifiers = [];
foreach ($targets as $target) {
$providerId = $target->getAttribute('providerId') ?: $default->getId();
if (!\array_key_exists($providerId, $identifiers)) {
$identifiers[$providerId] = [];
}
// Null values keep identifiers unique without a second lookup structure.
$identifiers[$providerId][$target->getAttribute('identifier')] = null;
}
return $identifiers;
}
/**
* Resolve and cache a provider for the lifetime of a send job, falling back to the default provider.
*
* @param array<string, Document> $providers
*/
private function resolveProvider(
Database $dbForProject,
string $providerId,
array &$providers,
Document $default
): Document {
if (\array_key_exists($providerId, $providers)) {
return $providers[$providerId];
}
$provider = $dbForProject->getDocument('providers', $providerId);
if ($provider->isEmpty() || !$provider->getAttribute('enabled')) {
// Cache the fallback under this id too, so a topic full of targets pointing at the same
// disabled/missing provider does not re-query the providers collection once per page.
$providers[$providerId] = $default;
return $default;
}
$providers[$providerId] = $provider;
return $provider;
}
/**
* Send a single adapter-sized batch and report delivery counts plus a bounded error list.
*
* Wraps the provider call in a backoff/retry loop that reacts to provider rate limiting and transient
* failures (see {@see retrySend()}). Accounting stays exact: only still-failing recipients are ever
* retried, so `delivered` is summed across attempts without double-counting, and `recipients` always
* reports the original batch size so the caller's `failed = recipients - delivered` holds.
*
* @param array<string> $batch
* @return array{delivered: int, recipients: int, errors: array<string>}
*/
private function sendBatch(
array $batch,
Document $message,
Document $provider,
string $providerType,
EmailAdapter|SMSAdapter|PushAdapter $adapter,
Database $dbForProject,
Device $deviceForFiles,
Document $project,
UsagePublisher $publisherForUsage
): array {
$recipients = \count($batch);
[
'delivered' => $delivered,
'errors' => $errors,
] = $this->retrySend($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project);
$failed = $recipients - $delivered;
$usage = new UsageContext();
$usage
->addMetric(METRIC_MESSAGES, $recipients)
->addMetric(METRIC_MESSAGES_SENT, $delivered)
->addMetric(METRIC_MESSAGES_FAILED, $failed)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed);
$publisherForUsage->enqueue(new Usage(
project: $project,
metrics: $usage->getMetrics(),
));
return [
'delivered' => $delivered,
'recipients' => $recipients,
'errors' => $errors,
];
}
/**
* Drive a batch through provider sends with exponential backoff, reacting to throttling and transient
* errors by retrying only the recipients that are still failing.
*
* Partitioning per attempt: the provider's per-recipient results are split into delivered (summed into
* the running total, never retried) and failures. A failure whose error matches {@see isRetryableError()}
* stays pending for the next attempt; any other failure becomes a terminal error immediately. When
* `send()` throws, the throw is retryable only if its message matches; otherwise every still-pending
* recipient is recorded as terminal. The next attempt rebuilds the provider message with only the pending
* recipients in `to`, so an already-delivered recipient is never re-sent. After {@see MESSAGE_SEND_MAX_RETRIES}
* attempts, any recipients still pending are flushed to terminal errors.
*
* Note: provider `Retry-After` hints are not honored — the agnostic adapter {@see \Utopia\Messaging\Response}
* exposes only generic per-recipient strings, never a structured retry delay, so we keep the library
* adapter-agnostic and rely on exponential backoff alone.
*
* @param array<string> $batch
* @return array{delivered: int, errors: array<string>}
*/
private function retrySend(
array $batch,
Document $message,
Document $provider,
string $providerType,
EmailAdapter|SMSAdapter|PushAdapter $adapter,
Database $dbForProject,
Device $deviceForFiles,
Document $project
): array {
$delivered = 0;
$errors = [];
// Recipients still awaiting a successful (or terminal) outcome; shrinks to only the failing ones each attempt.
$pending = $batch;
for ($attempt = 1; $attempt <= MESSAGE_SEND_MAX_RETRIES; $attempt++) {
$hasRetriesLeft = $attempt < MESSAGE_SEND_MAX_RETRIES;
// Rebuild the provider message scoped to only the still-pending recipients so a partially-delivered
// batch never re-sends to recipients that already succeeded on an earlier attempt.
$data = $this->buildMessage($pending, $message, $provider, $providerType, $dbForProject, $deviceForFiles, $project);
$retry = [];
// The try/catch wraps ONLY the provider send. A whole-batch throw is retryable when transient,
// otherwise it records one representative terminal error. The previous behaviour of resetting
// $delivered to 0 on a throw is gone — the retry refactor sums delivered across attempts, and the
// expired-device-token cleanup below is isolated in its own try so a DB hiccup there can never be
// misattributed as a send failure.
try {
$response = $adapter->send($data);
} catch (\Throwable $e) {
if ($hasRetriesLeft && $this->isRetryableError($e->getMessage())) {
$retry = $pending;
} else {
$this->recordError($errors, 'Failed sending to targets with error: ' . $e->getMessage());
}
$response = null;
}
if ($response !== null) {
$delivered += (int) $response['deliveredTo'];
foreach ($response['results'] as $result) {
if ($result['status'] !== 'failure') {
continue;
}
$error = $result['error'] ?? null;
$recipient = $result['recipient'];
// Best-effort: deleting push targets when the token has expired. Isolated so a transient DB
// error here never affects delivery accounting or the retry decision below.
if ($error === 'Expired device token') {
try {
$target = $dbForProject->findOne('targets', [
Query::equal('identifier', [$recipient])
]);
if (!$target->isEmpty()) {
$dbForProject->updateDocument(
'targets',
$target->getId(),
$target->setAttribute('expired', true)
);
}
} catch (\Throwable) {
// Best-effort; must not affect accounting or retries.
}
}
if ($hasRetriesLeft && $this->isRetryableError($error)) {
$retry[] = $recipient;
continue;
}
$this->recordError($errors, "Failed sending to target {$recipient} with error: {$error}");
}
}
$pending = $retry;
if (empty($pending)) {
break;
}
// Exponential backoff with jitter; non-blocking under Swoole so sibling sends keep progressing.
// Skip a non-positive delay: Swoole\Coroutine::sleep() rejects 0/negative values (which tests
// produce by overriding the base delay to 0 for speed), and it would otherwise emit a warning.
$delay = $this->retryDelay() * (2 ** ($attempt - 1));
$delay += $delay * (\random_int(0, 100) / 1000);
if ($delay > 0) {
\Swoole\Coroutine::sleep($delay);
}
}
return [
'delivered' => $delivered,
'errors' => $errors,
];
}
/**
* Append a delivery error while keeping the retained list bounded by {@see MESSAGE_DELIVERY_ERRORS_LIMIT}.
*
* @param array<string> $errors
*/
private function recordError(array &$errors, string $error): void
{
if (\count($errors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) {
return;
}
$errors[] = $error;
}
/**
* Conservatively classify a provider error string as retryable. The agnostic adapter
* {@see \Utopia\Messaging\Response} returns only free-form error strings, so string matching is the only
* provider-agnostic signal available. The pattern list is intentionally narrow — throttling, rate limits,
* quota, service-unavailable and timeout phrasing — so permanent failures (e.g. invalid recipients) are
* never retried.
*/
private function isRetryableError(?string $error): bool
{
if ($error === null || $error === '') {
return false;
}
$patterns = [
'throttl',
'rate exceeded',
'rate limit',
'too many requests',
'429',
'quota',
'service unavailable',
'503',
'timed out',
'timeout',
'temporarily',
];
$needle = \strtolower($error);
foreach ($patterns as $pattern) {
if (\str_contains($needle, $pattern)) {
return true;
}
}
return false;
}
/**
* Base seconds for exponential backoff between send retries. Isolated so tests can override it to keep the
* suite instant without touching the production constant.
*/
protected function retryDelay(): float
{
return MESSAGE_SEND_RETRY_DELAY;
}
/**
* Build the provider-specific message for a set of recipients.
*
* @param array<string> $to
*/
private function buildMessage(
array $to,
Document $message,
Document $provider,
string $providerType,
Database $dbForProject,
Device $deviceForFiles,
Document $project
): Email|SMS|Push {
$messageData = clone $message;
$messageData->setAttribute('to', $to);
return match ($providerType) {
MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider),
MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData),
MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project),
default => throw new \Exception('Provider with the requested ID is of the incorrect type')
};
}
private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void
{
if ($this->adapter === null) {
@@ -450,7 +807,7 @@ class Messaging extends Action
}
private function getSmsAdapter(Document $provider): ?SMSAdapter
protected function getSmsAdapter(Document $provider): ?SMSAdapter
{
$credentials = $provider->getAttribute('credentials');
@@ -493,7 +850,7 @@ class Messaging extends Action
};
}
private function getPushAdapter(Document $provider): ?PushAdapter
protected function getPushAdapter(Document $provider): ?PushAdapter
{
$credentials = $provider->getAttribute('credentials');
$options = $provider->getAttribute('options');
@@ -512,7 +869,7 @@ class Messaging extends Action
};
}
private function getEmailAdapter(Document $provider): ?EmailAdapter
protected function getEmailAdapter(Document $provider): ?EmailAdapter
{
$credentials = $provider->getAttribute('credentials', []);
$options = $provider->getAttribute('options', []);
@@ -0,0 +1,808 @@
<?php
namespace Tests\Unit\Messaging;
use Appwrite\Platform\Workers\Messaging;
use PHPUnit\Framework\TestCase;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Messaging\Adapter\Email as EmailAdapter;
use Utopia\Messaging\Messages\Email as EmailMessage;
use Utopia\Messaging\Response;
use Utopia\Storage\Device\Local;
/**
* Pure unit tests for the messaging worker's fan-out building blocks: the retryable-error classifier, the
* batch retry/backoff loop, and the cursor-paginated recipient stream. Each method is exercised directly
* through reflection with no Swoole coroutines.
*
* The coroutine fan-out integration itself — the bounded-concurrency {@see \Utopia\Lock\Semaphore} driving
* `Swoole\Coroutine\batch()` — is intentionally NOT covered here. Executing real Swoole coroutines inside the
* shared unit process is unsafe: the in-process scheduler destabilises later, unrelated tests in the suite.
* That integration is covered by the e2e Messaging suite instead. See PR #12465.
*
* Private methods are invoked via {@see \ReflectionMethod::invoke()} WITHOUT {@see \ReflectionMethod::setAccessible()}:
* reflection invokes private methods directly on PHP 8.1+, and `setAccessible(true)` is deprecated on PHP 8.5
* (the CI unit-test runtime), where the suite fails on triggered deprecations.
*/
class MessagingFanoutTest extends TestCase
{
private function provider(): Document
{
return new Document([
'$id' => 'provider1',
'$sequence' => '1',
'enabled' => true,
'type' => MESSAGE_TYPE_EMAIL,
'provider' => 'mock',
'name' => 'Mock Email',
'credentials' => [],
'options' => [
'fromName' => 'Appwrite',
'fromEmail' => 'noreply@example.com',
],
]);
}
private function message(): Document
{
return new Document([
'$id' => 'message1',
'$sequence' => '1',
'providerType' => MESSAGE_TYPE_EMAIL,
'topics' => ['topic1'],
'targets' => [],
'users' => [],
'data' => [
'subject' => 'Hello',
'content' => 'World',
'html' => false,
],
]);
}
/**
* Invoke the worker's private {@see Messaging::retrySend()} for an email batch.
*
* @param array<string> $batch
* @return array{delivered: int, errors: array<string>}
*/
private function retrySend(array $batch, EmailAdapter $adapter, Database $database): array
{
$worker = new TestableMessaging();
$method = new \ReflectionMethod(Messaging::class, 'retrySend');
$device = new Local(\sys_get_temp_dir());
$project = new Document(['$id' => 'project1', '$sequence' => '1']);
/** @var array{delivered: int, errors: array<string>} $result */
$result = $method->invoke(
$worker,
$batch,
$this->message(),
$this->provider(),
MESSAGE_TYPE_EMAIL,
$adapter,
$database,
$device,
$project
);
return $result;
}
public function testIsRetryableErrorMatchesTransientFailures(): void
{
$worker = new TestableMessaging();
$method = new \ReflectionMethod(Messaging::class, 'isRetryableError');
$retryable = [
'Throttling',
'rate exceeded',
'Rate limit reached',
'Too Many Requests',
'429 Too Many Requests',
'quota exceeded',
'Service Unavailable',
'503 Service Unavailable',
'Connection timed out',
'Request timeout',
'Service temporarily unavailable',
];
foreach ($retryable as $error) {
$this->assertTrue(
$method->invoke($worker, $error),
"Expected '{$error}' to be classified as retryable"
);
}
$permanent = [
'invalid recipient',
'Mailbox does not exist',
'Authentication failed',
'Expired device token',
];
foreach ($permanent as $error) {
$this->assertFalse(
$method->invoke($worker, $error),
"Expected '{$error}' to be classified as permanent"
);
}
$this->assertFalse($method->invoke($worker, null), 'A null error must never be retryable');
$this->assertFalse($method->invoke($worker, ''), 'An empty error must never be retryable');
}
public function testThrottledBatchIsRetriedThenFullyDelivered(): void
{
$batch = ['user1@example.com', 'user2@example.com', 'user3@example.com', 'user4@example.com', 'user5@example.com'];
// First attempt throttles every recipient; the script then runs out so the second attempt succeeds.
$adapter = new ScriptedEmailAdapter([
static fn (string $recipient): string => 'Rate limit exceeded, please retry',
]);
$result = $this->retrySend($batch, $adapter, new InMemoryDatabase());
// Retried exactly once, then everyone delivered with no double-counting and no terminal errors.
$this->assertSame(2, $adapter->sendCalls, 'Throttled batch must be retried once');
$this->assertSame(\count($batch), $result['delivered']);
$this->assertEmpty($result['errors']);
// Both attempts targeted the full recipient set (none had succeeded on the first attempt).
$this->assertSame($batch, $adapter->recipientsPerCall[0]);
$this->assertSame($batch, $adapter->recipientsPerCall[1]);
}
public function testRetryTargetsOnlyThrottledRecipients(): void
{
$batch = ['user1@example.com', 'user2@example.com', 'user3@example.com', 'user4@example.com', 'user5@example.com'];
// user2@ and user4@ throttle on the first attempt; everyone else succeeds. The second attempt has no
// script step, so the two retried recipients then succeed.
$throttled = ['user2@example.com', 'user4@example.com'];
$adapter = new ScriptedEmailAdapter([
static fn (string $recipient): ?string => \in_array($recipient, $throttled, true)
? 'Too Many Requests'
: null,
]);
$result = $this->retrySend($batch, $adapter, new InMemoryDatabase());
$this->assertSame(2, $adapter->sendCalls, 'A partially throttled batch must be retried once');
// The retry must re-send to EXACTLY the throttled recipients — never the ones already delivered.
$this->assertSame($throttled, $adapter->recipientsPerCall[1]);
// Every recipient is ultimately delivered and counted exactly once.
$this->assertSame(\count($batch), $result['delivered']);
$this->assertEmpty($result['errors']);
}
public function testNonRetryableFailureIsNotRetried(): void
{
$batch = ['user1@example.com', 'user2@example.com', 'user3@example.com'];
// user1@ permanently fails with a non-retryable error; the rest succeed.
$adapter = new ScriptedEmailAdapter([
static fn (string $recipient): ?string => $recipient === 'user1@example.com'
? 'Invalid recipient address'
: null,
]);
$result = $this->retrySend($batch, $adapter, new InMemoryDatabase());
// A permanent failure is recorded immediately, never retried.
$this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry');
$this->assertSame(\count($batch) - 1, $result['delivered']);
$this->assertCount(1, $result['errors']);
$this->assertStringContainsString('user1@example.com', $result['errors'][0]);
$this->assertStringContainsString('Invalid recipient address', $result['errors'][0]);
}
public function testExhaustedRetriesRecordTerminalFailure(): void
{
$batch = ['user1@example.com', 'user2@example.com'];
// Every attempt throttles, for more steps than MESSAGE_SEND_MAX_RETRIES, so retries are exhausted.
$script = \array_fill(
0,
MESSAGE_SEND_MAX_RETRIES + 2,
static fn (string $recipient): string => 'Service unavailable, throttled'
);
$adapter = new ScriptedEmailAdapter($script);
$result = $this->retrySend($batch, $adapter, new InMemoryDatabase());
// The batch is attempted exactly MESSAGE_SEND_MAX_RETRIES times, then the still-throttled recipients
// become terminal failures with nothing delivered.
$this->assertSame(MESSAGE_SEND_MAX_RETRIES, $adapter->sendCalls, 'Attempts must be capped at MESSAGE_SEND_MAX_RETRIES');
$this->assertSame(0, $result['delivered']);
$this->assertCount(\count($batch), $result['errors']);
}
public function testDeliveryErrorsAreCappedWhileDeliveredCountIsExact(): void
{
// More failing recipients than the retained-error cap, plus a known number of successes, so the
// delivered tally stays exact even though the error list is truncated.
$failingCount = MESSAGE_DELIVERY_ERRORS_LIMIT + 20;
$succeedingCount = 100;
$batch = [];
for ($i = 1; $i <= $failingCount; $i++) {
$batch[] = "fail{$i}@example.com";
}
for ($i = 1; $i <= $succeedingCount; $i++) {
$batch[] = "ok{$i}@example.com";
}
// Permanent (non-retryable) failure for every "fail*" recipient; everyone else is delivered.
$adapter = new ScriptedEmailAdapter([
static fn (string $recipient): ?string => \str_starts_with($recipient, 'fail')
? 'Invalid recipient address'
: null,
]);
$result = $this->retrySend($batch, $adapter, new InMemoryDatabase());
// Errors are capped, but the delivered count reflects every success exactly.
$this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $result['errors'], 'Delivery errors must be capped');
$this->assertSame($succeedingCount, $result['delivered'], 'Delivered count stays exact despite the error cap');
$this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry');
}
public function testExpiredTokenCleanupFailureDoesNotCorruptAccounting(): void
{
$batch = ['user1@example.com', 'user2@example.com', 'user3@example.com'];
// The cleanup path always throws, but the worker must isolate it: the send result is what's accounted.
$database = new ExpiredTokenCleanupFailingDatabase();
// user1@ comes back with an expired device token (triggering cleanup, which then throws); the other
// two recipients are delivered. The expired-token error is terminal, never retryable.
$adapter = new ScriptedEmailAdapter([
static fn (string $recipient): ?string => $recipient === 'user1@example.com'
? 'Expired device token'
: null,
]);
$result = $this->retrySend($batch, $adapter, $database);
// The worker reached the cleanup (so the isolation is genuinely exercised) and it threw.
$this->assertGreaterThan(0, $database->targetUpdateAttempts, 'Expired-token cleanup must have been attempted');
// The send was attempted exactly once: a thrown cleanup must not look like a transient send failure
// and must not trigger a retry.
$this->assertSame(1, $adapter->sendCalls, 'Cleanup failure must not trigger a send retry');
// Accounting reflects the send result, not the cleanup throw: two delivered, one terminal failure.
$this->assertSame(\count($batch) - 1, $result['delivered']);
// The recorded error is the provider's expired-token failure for the real recipient — proving the
// database exception was swallowed rather than misattributed as the send error.
$this->assertCount(1, $result['errors']);
$this->assertStringContainsString('user1@example.com', $result['errors'][0]);
$this->assertStringContainsString('Expired device token', $result['errors'][0]);
$this->assertStringNotContainsString('Transient database error', $result['errors'][0]);
}
public function testRecipientsArePaginatedWithoutSubqueryFilter(): void
{
// Always span several pages regardless of the configured page size, so cursor pagination is exercised.
$count = MESSAGE_RECIPIENTS_PAGE_SIZE * 2 + 500;
$data = $this->topicDataset($count);
$database = new RecordingDatabase($data['subscribers'], $data['targets']);
$identifiers = $this->collectStream($database, topicIds: ['topic1']);
$topicReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics'));
$subscriberReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'subscribers'));
// The topic's `targets` attribute (the subQueryTopicTargets filter) is never materialized: topics are
// only read with a $sequence projection, and recipients come from the subscribers collection instead.
$this->assertNotEmpty($topicReads);
foreach ($topicReads as $call) {
$this->assertNotContains('targets', $this->selectFields($call['queries']), 'Topic read must not select the targets subquery attribute');
}
// Subscribers are walked in bounded pages via cursorAfter; no query ever requests a runaway page size.
$expectedPages = (int) \ceil($count / MESSAGE_RECIPIENTS_PAGE_SIZE);
$this->assertCount($expectedPages, $subscriberReads);
foreach ($subscriberReads as $index => $call) {
$this->assertSame(MESSAGE_RECIPIENTS_PAGE_SIZE, $this->limitOf($call['queries']));
$this->assertSame('100', $this->topicInternalIdOf($call['queries']), 'Subscribers must be filtered by the topic internal id');
if ($index === 0) {
$this->assertFalse($this->hasCursor($call['queries']), 'First page must not carry a cursor');
} else {
$this->assertTrue($this->hasCursor($call['queries']), 'Subsequent pages must paginate with cursorAfter');
}
}
// No query in the whole stream issues a runaway (>= 1,000,000) limit.
foreach ($database->findCalls as $call) {
$this->assertLessThan(1_000_000, $this->limitOf($call['queries']), 'No query may request a million-row page');
}
// The duplicate identifier collapses: one row per distinct subscriber identifier resolved.
$this->assertCount($count, $identifiers);
for ($i = 1; $i <= $count; $i++) {
$this->assertArrayHasKey("user{$i}@example.com", $identifiers);
}
}
public function testStreamDeduplicatesIdentifiersWithinPage(): void
{
// Two subscribers in a single page resolve to targets sharing one identifier; the page must collapse
// them to a single recipient entry.
$subscribers = [
new Document(['$id' => 'sub1', '$sequence' => '1', 'topicInternalId' => '100', 'targetInternalId' => '1', 'providerType' => MESSAGE_TYPE_EMAIL]),
new Document(['$id' => 'sub2', '$sequence' => '2', 'topicInternalId' => '100', 'targetInternalId' => '2', 'providerType' => MESSAGE_TYPE_EMAIL]),
];
$targets = [
'1' => new Document(['$id' => 'target1', '$sequence' => '1', 'providerId' => null, 'identifier' => 'dup@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]),
'2' => new Document(['$id' => 'target2', '$sequence' => '2', 'providerId' => null, 'identifier' => 'dup@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]),
];
$database = new RecordingDatabase($subscribers, $targets);
$pages = $this->collectPages($database, topicIds: ['topic1']);
$this->assertCount(1, $pages, 'A single subscriber page yields a single grouped result');
$providerId = $this->provider()->getId();
$this->assertArrayHasKey($providerId, $pages[0]);
$this->assertSame(['dup@example.com'], \array_keys($pages[0][$providerId]), 'Duplicate identifiers must collapse to one');
}
public function testStreamResolvesUsersAndDirectTargets(): void
{
$userTargets = [
new Document(['$id' => 'utarget1', '$sequence' => '10', 'providerId' => null, 'identifier' => 'byuser@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]),
];
$directTargets = [
new Document(['$id' => 'dtarget1', '$sequence' => '20', 'providerId' => null, 'identifier' => 'bytarget@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]),
];
$database = new RecordingDatabase([], [], $userTargets, $directTargets);
$identifiers = $this->collectStream($database, userIds: ['user-a'], targetIds: ['target-b']);
// No topics in play: only the user and direct-target queries run, and both identifiers resolve.
$this->assertSame([], \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics')));
$this->assertArrayHasKey('byuser@example.com', $identifiers);
$this->assertArrayHasKey('bytarget@example.com', $identifiers);
$this->assertCount(2, $identifiers);
}
/**
* Build a topic-backed dataset: $count subscribers, each pointing at one email target. A single identifier
* is duplicated so per-page dedup is exercised when the dataset fits in one page.
*
* @return array{subscribers: array<Document>, targets: array<string, Document>}
*/
private function topicDataset(int $count): array
{
$subscribers = [];
$targets = [];
for ($i = 1; $i <= $count; $i++) {
$sequence = (string) $i;
$subscribers[] = new Document([
'$id' => 'sub' . $i,
'$sequence' => $sequence,
'topicInternalId' => '100',
'targetInternalId' => $sequence,
'providerType' => MESSAGE_TYPE_EMAIL,
]);
$targets[$sequence] = new Document([
'$id' => 'target' . $i,
'$sequence' => $sequence,
'providerId' => null,
'identifier' => "user{$i}@example.com",
'providerType' => MESSAGE_TYPE_EMAIL,
]);
}
return ['subscribers' => $subscribers, 'targets' => $targets];
}
/**
* Drive {@see Messaging::streamRecipients()} and merge every yielded page into a single identifier map.
*
* @param array<string> $topicIds
* @param array<string> $userIds
* @param array<string> $targetIds
* @return array<string, null>
*/
private function collectStream(
RecordingDatabase $database,
array $topicIds = [],
array $userIds = [],
array $targetIds = []
): array {
$identifiers = [];
foreach ($this->collectPages($database, $topicIds, $userIds, $targetIds) as $page) {
foreach ($page as $perProvider) {
foreach ($perProvider as $identifier => $_) {
$identifiers[$identifier] = null;
}
}
}
return $identifiers;
}
/**
* @param array<string> $topicIds
* @param array<string> $userIds
* @param array<string> $targetIds
* @return array<array<string, array<string, null>>>
*/
private function collectPages(
RecordingDatabase $database,
array $topicIds = [],
array $userIds = [],
array $targetIds = []
): array {
$method = new \ReflectionMethod(Messaging::class, 'streamRecipients');
$worker = new TestableMessaging();
/** @var \Generator<array<string, array<string, null>>> $generator */
$generator = $method->invoke(
$worker,
$database,
$topicIds,
$userIds,
$targetIds,
MESSAGE_TYPE_EMAIL,
$this->provider()
);
return \iterator_to_array($generator, false);
}
/**
* @param array<Query> $queries
* @return array<string>
*/
private function selectFields(array $queries): array
{
foreach ($queries as $query) {
if ($query->getMethod() === Query::TYPE_SELECT) {
return $query->getValues();
}
}
return [];
}
/**
* @param array<Query> $queries
*/
private function limitOf(array $queries): int
{
foreach ($queries as $query) {
if ($query->getMethod() === Query::TYPE_LIMIT) {
return (int) ($query->getValues()[0] ?? 0);
}
}
return 0;
}
/**
* @param array<Query> $queries
*/
private function hasCursor(array $queries): bool
{
foreach ($queries as $query) {
if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) {
return true;
}
}
return false;
}
/**
* @param array<Query> $queries
*/
private function topicInternalIdOf(array $queries): ?string
{
foreach ($queries as $query) {
if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'topicInternalId') {
return (string) ($query->getValues()[0] ?? null);
}
}
return null;
}
}
/**
* Worker subclass that zeroes the retry backoff so the throttle/retry tests stay instant. The worker already
* skips a non-positive delay, so no coroutine sleep is ever reached.
*/
class TestableMessaging extends Messaging
{
protected function retryDelay(): float
{
return 0.0;
}
}
/**
* Email adapter double whose per-attempt outcome is scripted. Each {@see process()} call records the exact
* `to` recipients it received and consults the next scripted step, which either throws (simulating a
* connection-level provider error) or returns a per-recipient success/failure verdict. When the script runs
* out, every recipient succeeds — modelling a provider that has stopped throttling.
*/
class ScriptedEmailAdapter extends EmailAdapter
{
public int $sendCalls = 0;
/**
* @var array<array<string>> Recipient email lists captured per {@see process()} call, in call order.
*/
public array $recipientsPerCall = [];
/**
* @param array<\Throwable|(callable(string): ?string)> $script One step per attempt. A Throwable is thrown
* for that attempt; a callable returns an error string for a failing recipient or null for success.
*/
public function __construct(private readonly array $script = [])
{
}
public function getName(): string
{
return 'ScriptedEmail';
}
public function getMaxMessagesPerRequest(): int
{
return 1000;
}
/**
* @param EmailMessage $message
* @return array{deliveredTo: int, type: string, results: array<array<string, mixed>>}
*/
protected function process(EmailMessage $message): array
{
$recipients = \array_map(static fn (array $entry): string => $entry['email'], $message->getTo());
$this->recipientsPerCall[] = $recipients;
$step = $this->script[$this->sendCalls] ?? null;
$this->sendCalls++;
if ($step instanceof \Throwable) {
throw $step;
}
$response = new Response($this->getType());
$delivered = 0;
foreach ($recipients as $recipient) {
$error = \is_callable($step) ? $step($recipient) : null;
if ($error === null) {
$delivered++;
$response->addResult($recipient);
} else {
$response->addResult($recipient, $error);
}
}
$response->setDeliveredTo($delivered);
return $response->toArray();
}
}
/**
* Minimal in-memory {@see Database} double for the retry tests, which only build email messages with no
* attachments and never touch the expired-token cleanup path. Every read resolves empty.
*/
class InMemoryDatabase extends Database
{
private readonly Authorization $authorization;
public function __construct()
{
$this->authorization = new Authorization();
}
public function getAuthorization(): Authorization
{
return $this->authorization;
}
public function skipValidation(callable $callback): mixed
{
return $callback();
}
public function findOne(string $collection, array $queries = []): Document
{
return new Document();
}
public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): Document
{
return new Document();
}
public function updateDocument(string $collection, string $id, Document $document): Document
{
return $document;
}
public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array
{
return [];
}
}
/**
* {@see InMemoryDatabase} variant whose expired-device-token cleanup always fails: `findOne('targets')`
* resolves to a real target so the worker attempts the update, and `updateDocument('targets')` throws a
* transient error. Used to prove that a DB hiccup during best-effort cleanup never leaks into delivery
* accounting or the retry decision.
*/
class ExpiredTokenCleanupFailingDatabase extends InMemoryDatabase
{
public int $targetUpdateAttempts = 0;
public function findOne(string $collection, array $queries = []): Document
{
if ($collection === 'targets') {
return new Document(['$id' => 'target-expired', '$sequence' => '1']);
}
return parent::findOne($collection, $queries);
}
public function updateDocument(string $collection, string $id, Document $document): Document
{
if ($collection === 'targets') {
$this->targetUpdateAttempts++;
throw new \RuntimeException('Transient database error during target cleanup');
}
return parent::updateDocument($collection, $id, $document);
}
}
/**
* In-memory {@see Database} double for the streaming tests. Serves subscribers/targets through cursor
* pagination and records every query issued so the test can assert the streaming contract (cursorAfter +
* bounded page size, topic reads never selecting the 1M-row `targets` subquery, no runaway limits).
*/
class RecordingDatabase extends Database
{
/**
* @var array<array{collection: string, queries: array<Query>}>
*/
public array $findCalls = [];
private readonly Authorization $authorization;
/**
* @param array<Document> $subscribers Subscriber rows (carry $sequence + targetInternalId + providerType).
* @param array<string, Document> $targets Target rows keyed by $sequence (resolved from a subscriber page).
* @param array<Document> $userTargets Targets addressed by userId.
* @param array<Document> $directTargets Targets addressed directly by id.
*/
public function __construct(
private readonly array $subscribers = [],
private readonly array $targets = [],
private readonly array $userTargets = [],
private readonly array $directTargets = []
) {
$this->authorization = new Authorization();
}
public function getAuthorization(): Authorization
{
return $this->authorization;
}
public function skipValidation(callable $callback): mixed
{
return $callback();
}
public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array
{
$this->findCalls[] = ['collection' => $collection, 'queries' => $queries];
return match ($collection) {
'topics' => [new Document(['$id' => 'topic1', '$sequence' => '100'])],
'subscribers' => $this->paginate($this->subscribers, $queries),
'targets' => $this->resolveTargets($queries),
default => [],
};
}
/**
* @param array<Document> $rows
* @param array<Query> $queries
* @return array<Document>
*/
private function paginate(array $rows, array $queries): array
{
$limit = $this->limitOf($queries);
$cursor = $this->cursorOf($queries);
if ($cursor !== null) {
$rows = \array_values(\array_filter(
$rows,
fn (Document $row) => $row->getSequence() > $cursor->getSequence()
));
}
return \array_slice($rows, 0, $limit);
}
/**
* Either resolve a subscriber page's targets by $sequence, or paginate user/direct targets.
*
* @param array<Query> $queries
* @return array<Document>
*/
private function resolveTargets(array $queries): array
{
foreach ($queries as $query) {
if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === '$sequence') {
$sequences = $query->getValues();
return \array_values(\array_map(
fn ($sequence) => $this->targets[$sequence],
\array_filter($sequences, fn ($sequence) => isset($this->targets[$sequence]))
));
}
if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'userId') {
return $this->paginate($this->userTargets, $queries);
}
}
return $this->paginate($this->directTargets, $queries);
}
/**
* @param array<Query> $queries
*/
private function limitOf(array $queries): int
{
foreach ($queries as $query) {
if ($query->getMethod() === Query::TYPE_LIMIT) {
return (int) ($query->getValues()[0] ?? 0);
}
}
return 0;
}
/**
* @param array<Query> $queries
*/
private function cursorOf(array $queries): ?Document
{
foreach ($queries as $query) {
if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) {
$value = $query->getValues()[0] ?? null;
return $value instanceof Document ? $value : null;
}
}
return null;
}
}