diff --git a/app/init/constants.php b/app/init/constants.php index d426e51fb6..4fe7c418a9 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -242,6 +242,12 @@ const RULE_STATUS_VERIFIED = 'verified'; // Message types const MESSAGE_SEND_TYPE_INTERNAL = 'internal'; const MESSAGE_SEND_TYPE_EXTERNAL = 'external'; +// External message fan-out tuning +const MESSAGE_RECIPIENTS_PAGE_SIZE = 5000; // Recipients resolved from the database per page while streaming a send; aligned with APP_LIMIT_COUNT to minimise round-trips on large topics +const MESSAGE_SEND_CONCURRENCY = 10; // Maximum adapter send requests running concurrently within a single send job +const MESSAGE_DELIVERY_ERRORS_LIMIT = 100; // Maximum number of per-recipient delivery errors retained on a message +const MESSAGE_SEND_MAX_RETRIES = 5; // Maximum number of attempts to deliver a batch when the provider throttles or fails transiently +const MESSAGE_SEND_RETRY_DELAY = 1.0; // Base seconds for exponential backoff between batch send retries; overridable in tests for an instant suite // Mail Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 82548b83a7..ddd7edf641 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -16,6 +16,7 @@ use Utopia\Database\Document; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\DSN\DSN; +use Utopia\Lock\Semaphore; use Utopia\Logger\Log; use Utopia\Messaging\Adapter\Email as EmailAdapter; use Utopia\Messaging\Adapter\Email\Mailgun; @@ -129,69 +130,19 @@ class Messaging extends Action Document $project, UsagePublisher $publisherForUsage ): void { + $status = $message->getAttribute('status'); + + // Idempotency guard: queue redelivery can hand us a message that already finished. + if (\in_array($status, [MessageStatus::SENT, MessageStatus::FAILED], true)) { + Span::add('message.skipped', 'already_processed'); + return; + } + $topicIds = $message->getAttribute('topics', []); $targetIds = $message->getAttribute('targets', []); $userIds = $message->getAttribute('users', []); $providerType = $message->getAttribute('providerType'); - /** - * @var array $allTargets - */ - $allTargets = []; - - if (\count($topicIds) > 0) { - $topics = $dbForProject->find('topics', [ - Query::equal('$id', $topicIds), - Query::limit(\count($topicIds)), - ]); - foreach ($topics as $topic) { - $targets = \array_filter($topic->getAttribute('targets'), function (Document $target) use ($providerType) { - return $target->getAttribute('providerType') === $providerType; - }); - - \array_push($allTargets, ...$targets); - } - } - - if (\count($userIds) > 0) { - $limit = 1000; - $offset = 0; - - do { - $targets = $dbForProject->find('targets', [ - Query::equal('userId', $userIds), - Query::select(['providerId', 'identifier']), - Query::equal('providerType', [$providerType]), - Query::limit($limit), - Query::offset($offset), - ]); - - \array_push($allTargets, ...$targets); - $offset += \count($targets); - } while (\count($targets) === $limit); - } - - if (\count($targetIds) > 0) { - $targets = $dbForProject->find('targets', [ - Query::select(['providerId', 'identifier']), - Query::equal('$id', $targetIds), - Query::equal('providerType', [$providerType]), - Query::limit(\count($targetIds)), - ]); - - \array_push($allTargets, ...$targets); - } - - if (empty($allTargets)) { - $dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([ - 'status' => MessageStatus::FAILED, - 'deliveryErrors' => ['No valid recipients found.'] - ])); - - Span::add('message.skipped', 'no_valid_recipients'); - return; - } - $default = $dbForProject->findOne('providers', [ Query::equal('enabled', [true]), Query::equal('type', [$providerType]), @@ -208,55 +159,33 @@ class Messaging extends Action } /** - * @var array> $identifiers - */ - $identifiers = []; - - /** - * @var array $providers + * Resolved providers cached for the lifetime of this job, keyed by provider id. + * Seeded with the default provider so most sends never touch the providers collection. + * + * @var array $providers */ $providers = [ - $default->getId() => $default + $default->getId() => $default, ]; - foreach ($allTargets as $target) { - $providerId = $target->getAttribute('providerId'); + $semaphore = new Semaphore(MESSAGE_SEND_CONCURRENCY); - if (!$providerId) { - $providerId = $default->getId(); - } + $deliveredTotal = 0; + $failedTotal = 0; + $deliveryErrors = []; + $hasRecipients = false; - if ($providerId) { - if (!\array_key_exists($providerId, $identifiers)) { - $identifiers[$providerId] = []; - } - // Use null as value to avoid duplicate keys - $identifiers[$providerId][$target->getAttribute('identifier')] = null; - } - } + foreach ($this->streamRecipients($dbForProject, $topicIds, $userIds, $targetIds, $providerType, $default) as $page) { + /** + * @var array $tasks + */ + $tasks = []; - /** - * @var array $results - */ - $results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - if (\array_key_exists($providerId, $providers)) { - $provider = $providers[$providerId]; - } else { - $provider = $dbForProject->getDocument('providers', $providerId); + foreach ($page as $providerId => $identifiers) { + $provider = $this->resolveProvider($dbForProject, $providerId, $providers, $default); + $resolvedProviderType = $provider->getAttribute('type'); - if ($provider->isEmpty() || !$provider->getAttribute('enabled')) { - $provider = $default; - } else { - $providers[$providerId] = $provider; - } - } - - $identifiersForProvider = $identifiers[$providerId]; - - $providerType = $provider->getAttribute('type'); - - $adapter = match ($providerType) { + $adapter = match ($resolvedProviderType) { MESSAGE_TYPE_SMS => $this->getSmsAdapter($provider), MESSAGE_TYPE_PUSH => $this->getPushAdapter($provider), MESSAGE_TYPE_EMAIL => $this->getEmailAdapter($provider), @@ -264,101 +193,74 @@ class Messaging extends Action }; $batches = \array_chunk( - \array_keys($identifiersForProvider), + \array_keys($identifiers), $adapter->getMaxMessagesPerRequest() ); - return batch(\array_map(function ($batch) use ($message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - return function () use ($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - $deliveredTotal = 0; - $deliveryErrors = []; - $messageData = clone $message; - $messageData->setAttribute('to', $batch); + foreach ($batches as $batch) { + $tasks[] = fn (): array => $semaphore->withLock( + fn (): array => $this->sendBatch( + $batch, + $message, + $provider, + $resolvedProviderType, + $adapter, + $dbForProject, + $deviceForFiles, + $project, + $publisherForUsage + ) + ); + } + } - $data = match ($providerType) { - MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider), - MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData), - MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project), - }; + // A page can resolve to zero identifiers (e.g. subscribers whose targetInternalId matches no targets + // row), so an empty task list must not count as recipients or run batch() for nothing. + if (empty($tasks)) { + continue; + } - try { - $response = $adapter->send($data); - $deliveredTotal += (int) $response['deliveredTo']; - foreach ($response['results'] as $result) { - if ($result['status'] === 'failure') { - $deliveryErrors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}"; - } + $hasRecipients = true; - // Deleting push targets when token has expired. - if (($result['error'] ?? '') === 'Expired device token') { - $target = $dbForProject->findOne('targets', [ - Query::equal('identifier', [$result['recipient']]) - ]); + /** + * @var array}> $results + */ + $results = batch($tasks); - if (!$target->isEmpty()) { - $dbForProject->updateDocument( - 'targets', - $target->getId(), - $target->setAttribute('expired', true) - ); - } - } - } - } catch (\Throwable $e) { - $deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage(); - } finally { - $errorTotal = \count($deliveryErrors); - $usage = new UsageContext(); - $usage - ->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal)) - ->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal) - ->addMetric(METRIC_MESSAGES_FAILED, $errorTotal) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), ($deliveredTotal + $errorTotal)) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $deliveredTotal) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $errorTotal) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), ($deliveredTotal + $errorTotal)) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $deliveredTotal) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal); + foreach ($results as $result) { + $deliveredTotal += $result['delivered']; + $failedTotal += $result['recipients'] - $result['delivered']; - $publisherForUsage->enqueue(new Usage( - project: $project, - metrics: $usage->getMetrics(), - )); + foreach ($result['errors'] as $error) { + if (\count($deliveryErrors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) { + break; + } - return [ - 'deliveredTotal' => $deliveredTotal, - 'deliveryErrors' => $deliveryErrors, - ]; - } - }; - }, $batches)); - }; - }, \array_keys($identifiers))); + $deliveryErrors[] = $error; + } + } + } - $results = \array_merge(...$results); + if (!$hasRecipients) { + $dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([ + 'status' => MessageStatus::FAILED, + 'deliveryErrors' => ['No valid recipients found.'] + ])); - $deliveredTotal = 0; - $deliveryErrors = []; - - foreach ($results as $result) { - $deliveredTotal += $result['deliveredTotal']; - $deliveryErrors = \array_merge($deliveryErrors, $result['deliveryErrors']); + Span::add('message.skipped', 'no_valid_recipients'); + return; } if (empty($deliveryErrors) && $deliveredTotal === 0) { $deliveryErrors[] = 'Unknown error'; } + $hasFailures = $failedTotal > 0 || \count($deliveryErrors) > 0; + $message->setAttribute('status', $hasFailures ? MessageStatus::FAILED : MessageStatus::SENT); $message->setAttribute('deliveryErrors', $deliveryErrors); - if (\count($message->getAttribute('deliveryErrors')) > 0) { - $message->setAttribute('status', MessageStatus::FAILED); - } else { - $message->setAttribute('status', MessageStatus::SENT); - } - Span::add('message.delivered_total', $deliveredTotal); - Span::add('message.errors_total', \count($deliveryErrors)); + Span::add('message.errors_total', $failedTotal); $message->removeAttribute('to'); @@ -409,6 +311,461 @@ class Messaging extends Action } } + /** + * Stream a message's recipients in bounded pages, grouped by provider and deduplicated by identifier within each page. + * + * Peak memory is O(MESSAGE_RECIPIENTS_PAGE_SIZE), never O(topic size): topics are walked through the + * subscribers collection with cursor pagination rather than reading the topic's `targets` attribute, which + * triggers the subQueryTopicTargets filter and loads up to APP_LIMIT_SUBSCRIBERS_SUBQUERY rows at once. + * + * @param array $topicIds + * @param array $userIds + * @param array $targetIds + * @return \Generator>> + * @throws \Exception + */ + private function streamRecipients( + Database $dbForProject, + array $topicIds, + array $userIds, + array $targetIds, + string $providerType, + Document $default + ): \Generator { + if (\count($topicIds) > 0) { + $topics = $dbForProject->find('topics', [ + Query::select(['$sequence']), + Query::equal('$id', $topicIds), + Query::limit(\count($topicIds)), + ]); + + foreach ($topics as $topic) { + $cursor = null; + + do { + $queries = [ + Query::equal('topicInternalId', [$topic->getSequence()]), + Query::equal('providerType', [$providerType]), + Query::select(['$sequence', 'targetInternalId']), + Query::orderAsc('$sequence'), + Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE), + ]; + + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + + $subscribers = $dbForProject->getAuthorization()->skip( + fn () => $dbForProject->find('subscribers', $queries) + ); + + $count = \count($subscribers); + + if ($count === 0) { + break; + } + + $cursor = $subscribers[$count - 1]; + + $targetInternalIds = \array_map( + fn (Document $subscriber) => $subscriber->getAttribute('targetInternalId'), + $subscribers + ); + + $targets = $dbForProject->skipValidation( + fn () => $dbForProject->getAuthorization()->skip( + fn () => $dbForProject->find('targets', [ + Query::equal('$sequence', $targetInternalIds), + Query::select(['providerId', 'identifier']), + Query::limit(\count($targetInternalIds)), + ]) + ) + ); + + yield $this->groupTargetsByProvider($targets, $default); + } while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE); + } + } + + if (\count($userIds) > 0) { + $cursor = null; + + do { + $queries = [ + Query::equal('userId', $userIds), + Query::equal('providerType', [$providerType]), + Query::select(['$sequence', 'providerId', 'identifier']), + Query::orderAsc('$sequence'), + Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE), + ]; + + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + + $targets = $dbForProject->find('targets', $queries); + $count = \count($targets); + + if ($count === 0) { + break; + } + + $cursor = $targets[$count - 1]; + + yield $this->groupTargetsByProvider($targets, $default); + } while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE); + } + + if (\count($targetIds) > 0) { + $cursor = null; + + do { + $queries = [ + Query::equal('$id', $targetIds), + Query::equal('providerType', [$providerType]), + Query::select(['$sequence', 'providerId', 'identifier']), + Query::orderAsc('$sequence'), + Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE), + ]; + + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + + $targets = $dbForProject->find('targets', $queries); + $count = \count($targets); + + if ($count === 0) { + break; + } + + $cursor = $targets[$count - 1]; + + yield $this->groupTargetsByProvider($targets, $default); + } while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE); + } + } + + /** + * Group a page of target documents by provider id, deduplicating identifiers within the page. + * + * @param array $targets + * @return array> + */ + private function groupTargetsByProvider(array $targets, Document $default): array + { + /** + * @var array> $identifiers + */ + $identifiers = []; + + foreach ($targets as $target) { + $providerId = $target->getAttribute('providerId') ?: $default->getId(); + + if (!\array_key_exists($providerId, $identifiers)) { + $identifiers[$providerId] = []; + } + + // Null values keep identifiers unique without a second lookup structure. + $identifiers[$providerId][$target->getAttribute('identifier')] = null; + } + + return $identifiers; + } + + /** + * Resolve and cache a provider for the lifetime of a send job, falling back to the default provider. + * + * @param array $providers + */ + private function resolveProvider( + Database $dbForProject, + string $providerId, + array &$providers, + Document $default + ): Document { + if (\array_key_exists($providerId, $providers)) { + return $providers[$providerId]; + } + + $provider = $dbForProject->getDocument('providers', $providerId); + + if ($provider->isEmpty() || !$provider->getAttribute('enabled')) { + // Cache the fallback under this id too, so a topic full of targets pointing at the same + // disabled/missing provider does not re-query the providers collection once per page. + $providers[$providerId] = $default; + + return $default; + } + + $providers[$providerId] = $provider; + + return $provider; + } + + /** + * Send a single adapter-sized batch and report delivery counts plus a bounded error list. + * + * Wraps the provider call in a backoff/retry loop that reacts to provider rate limiting and transient + * failures (see {@see retrySend()}). Accounting stays exact: only still-failing recipients are ever + * retried, so `delivered` is summed across attempts without double-counting, and `recipients` always + * reports the original batch size so the caller's `failed = recipients - delivered` holds. + * + * @param array $batch + * @return array{delivered: int, recipients: int, errors: array} + */ + private function sendBatch( + array $batch, + Document $message, + Document $provider, + string $providerType, + EmailAdapter|SMSAdapter|PushAdapter $adapter, + Database $dbForProject, + Device $deviceForFiles, + Document $project, + UsagePublisher $publisherForUsage + ): array { + $recipients = \count($batch); + + [ + 'delivered' => $delivered, + 'errors' => $errors, + ] = $this->retrySend($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project); + + $failed = $recipients - $delivered; + $usage = new UsageContext(); + $usage + ->addMetric(METRIC_MESSAGES, $recipients) + ->addMetric(METRIC_MESSAGES_SENT, $delivered) + ->addMetric(METRIC_MESSAGES_FAILED, $failed) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed); + + $publisherForUsage->enqueue(new Usage( + project: $project, + metrics: $usage->getMetrics(), + )); + + return [ + 'delivered' => $delivered, + 'recipients' => $recipients, + 'errors' => $errors, + ]; + } + + /** + * Drive a batch through provider sends with exponential backoff, reacting to throttling and transient + * errors by retrying only the recipients that are still failing. + * + * Partitioning per attempt: the provider's per-recipient results are split into delivered (summed into + * the running total, never retried) and failures. A failure whose error matches {@see isRetryableError()} + * stays pending for the next attempt; any other failure becomes a terminal error immediately. When + * `send()` throws, the throw is retryable only if its message matches; otherwise every still-pending + * recipient is recorded as terminal. The next attempt rebuilds the provider message with only the pending + * recipients in `to`, so an already-delivered recipient is never re-sent. After {@see MESSAGE_SEND_MAX_RETRIES} + * attempts, any recipients still pending are flushed to terminal errors. + * + * Note: provider `Retry-After` hints are not honored — the agnostic adapter {@see \Utopia\Messaging\Response} + * exposes only generic per-recipient strings, never a structured retry delay, so we keep the library + * adapter-agnostic and rely on exponential backoff alone. + * + * @param array $batch + * @return array{delivered: int, errors: array} + */ + private function retrySend( + array $batch, + Document $message, + Document $provider, + string $providerType, + EmailAdapter|SMSAdapter|PushAdapter $adapter, + Database $dbForProject, + Device $deviceForFiles, + Document $project + ): array { + $delivered = 0; + $errors = []; + + // Recipients still awaiting a successful (or terminal) outcome; shrinks to only the failing ones each attempt. + $pending = $batch; + + for ($attempt = 1; $attempt <= MESSAGE_SEND_MAX_RETRIES; $attempt++) { + $hasRetriesLeft = $attempt < MESSAGE_SEND_MAX_RETRIES; + + // Rebuild the provider message scoped to only the still-pending recipients so a partially-delivered + // batch never re-sends to recipients that already succeeded on an earlier attempt. + $data = $this->buildMessage($pending, $message, $provider, $providerType, $dbForProject, $deviceForFiles, $project); + + $retry = []; + + // The try/catch wraps ONLY the provider send. A whole-batch throw is retryable when transient, + // otherwise it records one representative terminal error. The previous behaviour of resetting + // $delivered to 0 on a throw is gone — the retry refactor sums delivered across attempts, and the + // expired-device-token cleanup below is isolated in its own try so a DB hiccup there can never be + // misattributed as a send failure. + try { + $response = $adapter->send($data); + } catch (\Throwable $e) { + if ($hasRetriesLeft && $this->isRetryableError($e->getMessage())) { + $retry = $pending; + } else { + $this->recordError($errors, 'Failed sending to targets with error: ' . $e->getMessage()); + } + + $response = null; + } + + if ($response !== null) { + $delivered += (int) $response['deliveredTo']; + + foreach ($response['results'] as $result) { + if ($result['status'] !== 'failure') { + continue; + } + + $error = $result['error'] ?? null; + $recipient = $result['recipient']; + + // Best-effort: deleting push targets when the token has expired. Isolated so a transient DB + // error here never affects delivery accounting or the retry decision below. + if ($error === 'Expired device token') { + try { + $target = $dbForProject->findOne('targets', [ + Query::equal('identifier', [$recipient]) + ]); + + if (!$target->isEmpty()) { + $dbForProject->updateDocument( + 'targets', + $target->getId(), + $target->setAttribute('expired', true) + ); + } + } catch (\Throwable) { + // Best-effort; must not affect accounting or retries. + } + } + + if ($hasRetriesLeft && $this->isRetryableError($error)) { + $retry[] = $recipient; + continue; + } + + $this->recordError($errors, "Failed sending to target {$recipient} with error: {$error}"); + } + } + + $pending = $retry; + + if (empty($pending)) { + break; + } + + // Exponential backoff with jitter; non-blocking under Swoole so sibling sends keep progressing. + // Skip a non-positive delay: Swoole\Coroutine::sleep() rejects 0/negative values (which tests + // produce by overriding the base delay to 0 for speed), and it would otherwise emit a warning. + $delay = $this->retryDelay() * (2 ** ($attempt - 1)); + $delay += $delay * (\random_int(0, 100) / 1000); + if ($delay > 0) { + \Swoole\Coroutine::sleep($delay); + } + } + + return [ + 'delivered' => $delivered, + 'errors' => $errors, + ]; + } + + /** + * Append a delivery error while keeping the retained list bounded by {@see MESSAGE_DELIVERY_ERRORS_LIMIT}. + * + * @param array $errors + */ + private function recordError(array &$errors, string $error): void + { + if (\count($errors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) { + return; + } + + $errors[] = $error; + } + + /** + * Conservatively classify a provider error string as retryable. The agnostic adapter + * {@see \Utopia\Messaging\Response} returns only free-form error strings, so string matching is the only + * provider-agnostic signal available. The pattern list is intentionally narrow — throttling, rate limits, + * quota, service-unavailable and timeout phrasing — so permanent failures (e.g. invalid recipients) are + * never retried. + */ + private function isRetryableError(?string $error): bool + { + if ($error === null || $error === '') { + return false; + } + + $patterns = [ + 'throttl', + 'rate exceeded', + 'rate limit', + 'too many requests', + '429', + 'quota', + 'service unavailable', + '503', + 'timed out', + 'timeout', + 'temporarily', + ]; + + $needle = \strtolower($error); + + foreach ($patterns as $pattern) { + if (\str_contains($needle, $pattern)) { + return true; + } + } + + return false; + } + + /** + * Base seconds for exponential backoff between send retries. Isolated so tests can override it to keep the + * suite instant without touching the production constant. + */ + protected function retryDelay(): float + { + return MESSAGE_SEND_RETRY_DELAY; + } + + /** + * Build the provider-specific message for a set of recipients. + * + * @param array $to + */ + private function buildMessage( + array $to, + Document $message, + Document $provider, + string $providerType, + Database $dbForProject, + Device $deviceForFiles, + Document $project + ): Email|SMS|Push { + $messageData = clone $message; + $messageData->setAttribute('to', $to); + + return match ($providerType) { + MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider), + MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData), + MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project), + default => throw new \Exception('Provider with the requested ID is of the incorrect type') + }; + } + private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void { if ($this->adapter === null) { @@ -450,7 +807,7 @@ class Messaging extends Action } - private function getSmsAdapter(Document $provider): ?SMSAdapter + protected function getSmsAdapter(Document $provider): ?SMSAdapter { $credentials = $provider->getAttribute('credentials'); @@ -493,7 +850,7 @@ class Messaging extends Action }; } - private function getPushAdapter(Document $provider): ?PushAdapter + protected function getPushAdapter(Document $provider): ?PushAdapter { $credentials = $provider->getAttribute('credentials'); $options = $provider->getAttribute('options'); @@ -512,7 +869,7 @@ class Messaging extends Action }; } - private function getEmailAdapter(Document $provider): ?EmailAdapter + protected function getEmailAdapter(Document $provider): ?EmailAdapter { $credentials = $provider->getAttribute('credentials', []); $options = $provider->getAttribute('options', []); diff --git a/tests/unit/Messaging/MessagingFanoutTest.php b/tests/unit/Messaging/MessagingFanoutTest.php new file mode 100644 index 0000000000..02b1ee55d4 --- /dev/null +++ b/tests/unit/Messaging/MessagingFanoutTest.php @@ -0,0 +1,808 @@ + 'provider1', + '$sequence' => '1', + 'enabled' => true, + 'type' => MESSAGE_TYPE_EMAIL, + 'provider' => 'mock', + 'name' => 'Mock Email', + 'credentials' => [], + 'options' => [ + 'fromName' => 'Appwrite', + 'fromEmail' => 'noreply@example.com', + ], + ]); + } + + private function message(): Document + { + return new Document([ + '$id' => 'message1', + '$sequence' => '1', + 'providerType' => MESSAGE_TYPE_EMAIL, + 'topics' => ['topic1'], + 'targets' => [], + 'users' => [], + 'data' => [ + 'subject' => 'Hello', + 'content' => 'World', + 'html' => false, + ], + ]); + } + + /** + * Invoke the worker's private {@see Messaging::retrySend()} for an email batch. + * + * @param array $batch + * @return array{delivered: int, errors: array} + */ + private function retrySend(array $batch, EmailAdapter $adapter, Database $database): array + { + $worker = new TestableMessaging(); + $method = new \ReflectionMethod(Messaging::class, 'retrySend'); + + $device = new Local(\sys_get_temp_dir()); + $project = new Document(['$id' => 'project1', '$sequence' => '1']); + + /** @var array{delivered: int, errors: array} $result */ + $result = $method->invoke( + $worker, + $batch, + $this->message(), + $this->provider(), + MESSAGE_TYPE_EMAIL, + $adapter, + $database, + $device, + $project + ); + + return $result; + } + + public function testIsRetryableErrorMatchesTransientFailures(): void + { + $worker = new TestableMessaging(); + $method = new \ReflectionMethod(Messaging::class, 'isRetryableError'); + + $retryable = [ + 'Throttling', + 'rate exceeded', + 'Rate limit reached', + 'Too Many Requests', + '429 Too Many Requests', + 'quota exceeded', + 'Service Unavailable', + '503 Service Unavailable', + 'Connection timed out', + 'Request timeout', + 'Service temporarily unavailable', + ]; + + foreach ($retryable as $error) { + $this->assertTrue( + $method->invoke($worker, $error), + "Expected '{$error}' to be classified as retryable" + ); + } + + $permanent = [ + 'invalid recipient', + 'Mailbox does not exist', + 'Authentication failed', + 'Expired device token', + ]; + + foreach ($permanent as $error) { + $this->assertFalse( + $method->invoke($worker, $error), + "Expected '{$error}' to be classified as permanent" + ); + } + + $this->assertFalse($method->invoke($worker, null), 'A null error must never be retryable'); + $this->assertFalse($method->invoke($worker, ''), 'An empty error must never be retryable'); + } + + public function testThrottledBatchIsRetriedThenFullyDelivered(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com', 'user4@example.com', 'user5@example.com']; + + // First attempt throttles every recipient; the script then runs out so the second attempt succeeds. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): string => 'Rate limit exceeded, please retry', + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // Retried exactly once, then everyone delivered with no double-counting and no terminal errors. + $this->assertSame(2, $adapter->sendCalls, 'Throttled batch must be retried once'); + $this->assertSame(\count($batch), $result['delivered']); + $this->assertEmpty($result['errors']); + + // Both attempts targeted the full recipient set (none had succeeded on the first attempt). + $this->assertSame($batch, $adapter->recipientsPerCall[0]); + $this->assertSame($batch, $adapter->recipientsPerCall[1]); + } + + public function testRetryTargetsOnlyThrottledRecipients(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com', 'user4@example.com', 'user5@example.com']; + + // user2@ and user4@ throttle on the first attempt; everyone else succeeds. The second attempt has no + // script step, so the two retried recipients then succeed. + $throttled = ['user2@example.com', 'user4@example.com']; + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => \in_array($recipient, $throttled, true) + ? 'Too Many Requests' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + $this->assertSame(2, $adapter->sendCalls, 'A partially throttled batch must be retried once'); + + // The retry must re-send to EXACTLY the throttled recipients — never the ones already delivered. + $this->assertSame($throttled, $adapter->recipientsPerCall[1]); + + // Every recipient is ultimately delivered and counted exactly once. + $this->assertSame(\count($batch), $result['delivered']); + $this->assertEmpty($result['errors']); + } + + public function testNonRetryableFailureIsNotRetried(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com']; + + // user1@ permanently fails with a non-retryable error; the rest succeed. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => $recipient === 'user1@example.com' + ? 'Invalid recipient address' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // A permanent failure is recorded immediately, never retried. + $this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry'); + $this->assertSame(\count($batch) - 1, $result['delivered']); + $this->assertCount(1, $result['errors']); + $this->assertStringContainsString('user1@example.com', $result['errors'][0]); + $this->assertStringContainsString('Invalid recipient address', $result['errors'][0]); + } + + public function testExhaustedRetriesRecordTerminalFailure(): void + { + $batch = ['user1@example.com', 'user2@example.com']; + + // Every attempt throttles, for more steps than MESSAGE_SEND_MAX_RETRIES, so retries are exhausted. + $script = \array_fill( + 0, + MESSAGE_SEND_MAX_RETRIES + 2, + static fn (string $recipient): string => 'Service unavailable, throttled' + ); + $adapter = new ScriptedEmailAdapter($script); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // The batch is attempted exactly MESSAGE_SEND_MAX_RETRIES times, then the still-throttled recipients + // become terminal failures with nothing delivered. + $this->assertSame(MESSAGE_SEND_MAX_RETRIES, $adapter->sendCalls, 'Attempts must be capped at MESSAGE_SEND_MAX_RETRIES'); + $this->assertSame(0, $result['delivered']); + $this->assertCount(\count($batch), $result['errors']); + } + + public function testDeliveryErrorsAreCappedWhileDeliveredCountIsExact(): void + { + // More failing recipients than the retained-error cap, plus a known number of successes, so the + // delivered tally stays exact even though the error list is truncated. + $failingCount = MESSAGE_DELIVERY_ERRORS_LIMIT + 20; + $succeedingCount = 100; + + $batch = []; + for ($i = 1; $i <= $failingCount; $i++) { + $batch[] = "fail{$i}@example.com"; + } + for ($i = 1; $i <= $succeedingCount; $i++) { + $batch[] = "ok{$i}@example.com"; + } + + // Permanent (non-retryable) failure for every "fail*" recipient; everyone else is delivered. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => \str_starts_with($recipient, 'fail') + ? 'Invalid recipient address' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // Errors are capped, but the delivered count reflects every success exactly. + $this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $result['errors'], 'Delivery errors must be capped'); + $this->assertSame($succeedingCount, $result['delivered'], 'Delivered count stays exact despite the error cap'); + $this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry'); + } + + public function testExpiredTokenCleanupFailureDoesNotCorruptAccounting(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com']; + + // The cleanup path always throws, but the worker must isolate it: the send result is what's accounted. + $database = new ExpiredTokenCleanupFailingDatabase(); + + // user1@ comes back with an expired device token (triggering cleanup, which then throws); the other + // two recipients are delivered. The expired-token error is terminal, never retryable. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => $recipient === 'user1@example.com' + ? 'Expired device token' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, $database); + + // The worker reached the cleanup (so the isolation is genuinely exercised) and it threw. + $this->assertGreaterThan(0, $database->targetUpdateAttempts, 'Expired-token cleanup must have been attempted'); + + // The send was attempted exactly once: a thrown cleanup must not look like a transient send failure + // and must not trigger a retry. + $this->assertSame(1, $adapter->sendCalls, 'Cleanup failure must not trigger a send retry'); + + // Accounting reflects the send result, not the cleanup throw: two delivered, one terminal failure. + $this->assertSame(\count($batch) - 1, $result['delivered']); + + // The recorded error is the provider's expired-token failure for the real recipient — proving the + // database exception was swallowed rather than misattributed as the send error. + $this->assertCount(1, $result['errors']); + $this->assertStringContainsString('user1@example.com', $result['errors'][0]); + $this->assertStringContainsString('Expired device token', $result['errors'][0]); + $this->assertStringNotContainsString('Transient database error', $result['errors'][0]); + } + + public function testRecipientsArePaginatedWithoutSubqueryFilter(): void + { + // Always span several pages regardless of the configured page size, so cursor pagination is exercised. + $count = MESSAGE_RECIPIENTS_PAGE_SIZE * 2 + 500; + $data = $this->topicDataset($count); + $database = new RecordingDatabase($data['subscribers'], $data['targets']); + + $identifiers = $this->collectStream($database, topicIds: ['topic1']); + + $topicReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics')); + $subscriberReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'subscribers')); + + // The topic's `targets` attribute (the subQueryTopicTargets filter) is never materialized: topics are + // only read with a $sequence projection, and recipients come from the subscribers collection instead. + $this->assertNotEmpty($topicReads); + foreach ($topicReads as $call) { + $this->assertNotContains('targets', $this->selectFields($call['queries']), 'Topic read must not select the targets subquery attribute'); + } + + // Subscribers are walked in bounded pages via cursorAfter; no query ever requests a runaway page size. + $expectedPages = (int) \ceil($count / MESSAGE_RECIPIENTS_PAGE_SIZE); + $this->assertCount($expectedPages, $subscriberReads); + + foreach ($subscriberReads as $index => $call) { + $this->assertSame(MESSAGE_RECIPIENTS_PAGE_SIZE, $this->limitOf($call['queries'])); + $this->assertSame('100', $this->topicInternalIdOf($call['queries']), 'Subscribers must be filtered by the topic internal id'); + + if ($index === 0) { + $this->assertFalse($this->hasCursor($call['queries']), 'First page must not carry a cursor'); + } else { + $this->assertTrue($this->hasCursor($call['queries']), 'Subsequent pages must paginate with cursorAfter'); + } + } + + // No query in the whole stream issues a runaway (>= 1,000,000) limit. + foreach ($database->findCalls as $call) { + $this->assertLessThan(1_000_000, $this->limitOf($call['queries']), 'No query may request a million-row page'); + } + + // The duplicate identifier collapses: one row per distinct subscriber identifier resolved. + $this->assertCount($count, $identifiers); + for ($i = 1; $i <= $count; $i++) { + $this->assertArrayHasKey("user{$i}@example.com", $identifiers); + } + } + + public function testStreamDeduplicatesIdentifiersWithinPage(): void + { + // Two subscribers in a single page resolve to targets sharing one identifier; the page must collapse + // them to a single recipient entry. + $subscribers = [ + new Document(['$id' => 'sub1', '$sequence' => '1', 'topicInternalId' => '100', 'targetInternalId' => '1', 'providerType' => MESSAGE_TYPE_EMAIL]), + new Document(['$id' => 'sub2', '$sequence' => '2', 'topicInternalId' => '100', 'targetInternalId' => '2', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $targets = [ + '1' => new Document(['$id' => 'target1', '$sequence' => '1', 'providerId' => null, 'identifier' => 'dup@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + '2' => new Document(['$id' => 'target2', '$sequence' => '2', 'providerId' => null, 'identifier' => 'dup@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $database = new RecordingDatabase($subscribers, $targets); + + $pages = $this->collectPages($database, topicIds: ['topic1']); + + $this->assertCount(1, $pages, 'A single subscriber page yields a single grouped result'); + $providerId = $this->provider()->getId(); + $this->assertArrayHasKey($providerId, $pages[0]); + $this->assertSame(['dup@example.com'], \array_keys($pages[0][$providerId]), 'Duplicate identifiers must collapse to one'); + } + + public function testStreamResolvesUsersAndDirectTargets(): void + { + $userTargets = [ + new Document(['$id' => 'utarget1', '$sequence' => '10', 'providerId' => null, 'identifier' => 'byuser@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $directTargets = [ + new Document(['$id' => 'dtarget1', '$sequence' => '20', 'providerId' => null, 'identifier' => 'bytarget@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $database = new RecordingDatabase([], [], $userTargets, $directTargets); + + $identifiers = $this->collectStream($database, userIds: ['user-a'], targetIds: ['target-b']); + + // No topics in play: only the user and direct-target queries run, and both identifiers resolve. + $this->assertSame([], \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics'))); + $this->assertArrayHasKey('byuser@example.com', $identifiers); + $this->assertArrayHasKey('bytarget@example.com', $identifiers); + $this->assertCount(2, $identifiers); + } + + /** + * Build a topic-backed dataset: $count subscribers, each pointing at one email target. A single identifier + * is duplicated so per-page dedup is exercised when the dataset fits in one page. + * + * @return array{subscribers: array, targets: array} + */ + private function topicDataset(int $count): array + { + $subscribers = []; + $targets = []; + + for ($i = 1; $i <= $count; $i++) { + $sequence = (string) $i; + $subscribers[] = new Document([ + '$id' => 'sub' . $i, + '$sequence' => $sequence, + 'topicInternalId' => '100', + 'targetInternalId' => $sequence, + 'providerType' => MESSAGE_TYPE_EMAIL, + ]); + $targets[$sequence] = new Document([ + '$id' => 'target' . $i, + '$sequence' => $sequence, + 'providerId' => null, + 'identifier' => "user{$i}@example.com", + 'providerType' => MESSAGE_TYPE_EMAIL, + ]); + } + + return ['subscribers' => $subscribers, 'targets' => $targets]; + } + + /** + * Drive {@see Messaging::streamRecipients()} and merge every yielded page into a single identifier map. + * + * @param array $topicIds + * @param array $userIds + * @param array $targetIds + * @return array + */ + private function collectStream( + RecordingDatabase $database, + array $topicIds = [], + array $userIds = [], + array $targetIds = [] + ): array { + $identifiers = []; + + foreach ($this->collectPages($database, $topicIds, $userIds, $targetIds) as $page) { + foreach ($page as $perProvider) { + foreach ($perProvider as $identifier => $_) { + $identifiers[$identifier] = null; + } + } + } + + return $identifiers; + } + + /** + * @param array $topicIds + * @param array $userIds + * @param array $targetIds + * @return array>> + */ + private function collectPages( + RecordingDatabase $database, + array $topicIds = [], + array $userIds = [], + array $targetIds = [] + ): array { + $method = new \ReflectionMethod(Messaging::class, 'streamRecipients'); + $worker = new TestableMessaging(); + + /** @var \Generator>> $generator */ + $generator = $method->invoke( + $worker, + $database, + $topicIds, + $userIds, + $targetIds, + MESSAGE_TYPE_EMAIL, + $this->provider() + ); + + return \iterator_to_array($generator, false); + } + + /** + * @param array $queries + * @return array + */ + private function selectFields(array $queries): array + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_SELECT) { + return $query->getValues(); + } + } + + return []; + } + + /** + * @param array $queries + */ + private function limitOf(array $queries): int + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_LIMIT) { + return (int) ($query->getValues()[0] ?? 0); + } + } + + return 0; + } + + /** + * @param array $queries + */ + private function hasCursor(array $queries): bool + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) { + return true; + } + } + + return false; + } + + /** + * @param array $queries + */ + private function topicInternalIdOf(array $queries): ?string + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'topicInternalId') { + return (string) ($query->getValues()[0] ?? null); + } + } + + return null; + } +} + +/** + * Worker subclass that zeroes the retry backoff so the throttle/retry tests stay instant. The worker already + * skips a non-positive delay, so no coroutine sleep is ever reached. + */ +class TestableMessaging extends Messaging +{ + protected function retryDelay(): float + { + return 0.0; + } +} + +/** + * Email adapter double whose per-attempt outcome is scripted. Each {@see process()} call records the exact + * `to` recipients it received and consults the next scripted step, which either throws (simulating a + * connection-level provider error) or returns a per-recipient success/failure verdict. When the script runs + * out, every recipient succeeds — modelling a provider that has stopped throttling. + */ +class ScriptedEmailAdapter extends EmailAdapter +{ + public int $sendCalls = 0; + + /** + * @var array> Recipient email lists captured per {@see process()} call, in call order. + */ + public array $recipientsPerCall = []; + + /** + * @param array<\Throwable|(callable(string): ?string)> $script One step per attempt. A Throwable is thrown + * for that attempt; a callable returns an error string for a failing recipient or null for success. + */ + public function __construct(private readonly array $script = []) + { + } + + public function getName(): string + { + return 'ScriptedEmail'; + } + + public function getMaxMessagesPerRequest(): int + { + return 1000; + } + + /** + * @param EmailMessage $message + * @return array{deliveredTo: int, type: string, results: array>} + */ + protected function process(EmailMessage $message): array + { + $recipients = \array_map(static fn (array $entry): string => $entry['email'], $message->getTo()); + $this->recipientsPerCall[] = $recipients; + + $step = $this->script[$this->sendCalls] ?? null; + $this->sendCalls++; + + if ($step instanceof \Throwable) { + throw $step; + } + + $response = new Response($this->getType()); + $delivered = 0; + + foreach ($recipients as $recipient) { + $error = \is_callable($step) ? $step($recipient) : null; + + if ($error === null) { + $delivered++; + $response->addResult($recipient); + } else { + $response->addResult($recipient, $error); + } + } + + $response->setDeliveredTo($delivered); + + return $response->toArray(); + } +} + +/** + * Minimal in-memory {@see Database} double for the retry tests, which only build email messages with no + * attachments and never touch the expired-token cleanup path. Every read resolves empty. + */ +class InMemoryDatabase extends Database +{ + private readonly Authorization $authorization; + + public function __construct() + { + $this->authorization = new Authorization(); + } + + public function getAuthorization(): Authorization + { + return $this->authorization; + } + + public function skipValidation(callable $callback): mixed + { + return $callback(); + } + + public function findOne(string $collection, array $queries = []): Document + { + return new Document(); + } + + public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): Document + { + return new Document(); + } + + public function updateDocument(string $collection, string $id, Document $document): Document + { + return $document; + } + + public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array + { + return []; + } +} + +/** + * {@see InMemoryDatabase} variant whose expired-device-token cleanup always fails: `findOne('targets')` + * resolves to a real target so the worker attempts the update, and `updateDocument('targets')` throws a + * transient error. Used to prove that a DB hiccup during best-effort cleanup never leaks into delivery + * accounting or the retry decision. + */ +class ExpiredTokenCleanupFailingDatabase extends InMemoryDatabase +{ + public int $targetUpdateAttempts = 0; + + public function findOne(string $collection, array $queries = []): Document + { + if ($collection === 'targets') { + return new Document(['$id' => 'target-expired', '$sequence' => '1']); + } + + return parent::findOne($collection, $queries); + } + + public function updateDocument(string $collection, string $id, Document $document): Document + { + if ($collection === 'targets') { + $this->targetUpdateAttempts++; + + throw new \RuntimeException('Transient database error during target cleanup'); + } + + return parent::updateDocument($collection, $id, $document); + } +} + +/** + * In-memory {@see Database} double for the streaming tests. Serves subscribers/targets through cursor + * pagination and records every query issued so the test can assert the streaming contract (cursorAfter + + * bounded page size, topic reads never selecting the 1M-row `targets` subquery, no runaway limits). + */ +class RecordingDatabase extends Database +{ + /** + * @var array}> + */ + public array $findCalls = []; + + private readonly Authorization $authorization; + + /** + * @param array $subscribers Subscriber rows (carry $sequence + targetInternalId + providerType). + * @param array $targets Target rows keyed by $sequence (resolved from a subscriber page). + * @param array $userTargets Targets addressed by userId. + * @param array $directTargets Targets addressed directly by id. + */ + public function __construct( + private readonly array $subscribers = [], + private readonly array $targets = [], + private readonly array $userTargets = [], + private readonly array $directTargets = [] + ) { + $this->authorization = new Authorization(); + } + + public function getAuthorization(): Authorization + { + return $this->authorization; + } + + public function skipValidation(callable $callback): mixed + { + return $callback(); + } + + public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array + { + $this->findCalls[] = ['collection' => $collection, 'queries' => $queries]; + + return match ($collection) { + 'topics' => [new Document(['$id' => 'topic1', '$sequence' => '100'])], + 'subscribers' => $this->paginate($this->subscribers, $queries), + 'targets' => $this->resolveTargets($queries), + default => [], + }; + } + + /** + * @param array $rows + * @param array $queries + * @return array + */ + private function paginate(array $rows, array $queries): array + { + $limit = $this->limitOf($queries); + $cursor = $this->cursorOf($queries); + + if ($cursor !== null) { + $rows = \array_values(\array_filter( + $rows, + fn (Document $row) => $row->getSequence() > $cursor->getSequence() + )); + } + + return \array_slice($rows, 0, $limit); + } + + /** + * Either resolve a subscriber page's targets by $sequence, or paginate user/direct targets. + * + * @param array $queries + * @return array + */ + private function resolveTargets(array $queries): array + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === '$sequence') { + $sequences = $query->getValues(); + + return \array_values(\array_map( + fn ($sequence) => $this->targets[$sequence], + \array_filter($sequences, fn ($sequence) => isset($this->targets[$sequence])) + )); + } + + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'userId') { + return $this->paginate($this->userTargets, $queries); + } + } + + return $this->paginate($this->directTargets, $queries); + } + + /** + * @param array $queries + */ + private function limitOf(array $queries): int + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_LIMIT) { + return (int) ($query->getValues()[0] ?? 0); + } + } + + return 0; + } + + /** + * @param array $queries + */ + private function cursorOf(array $queries): ?Document + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) { + $value = $query->getValues()[0] ?? null; + + return $value instanceof Document ? $value : null; + } + } + + return null; + } +}