From da061ba7d299e097668a3cfeab304b3656d5f3f0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Jun 2026 02:10:47 +1200 Subject: [PATCH] fix(messaging): stream large-topic sends and bound send concurrency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Large external sends (e.g. a 500K-recipient topic email) OOM'd the messaging worker: sendExternalMessage read $topic->getAttribute('targets'), triggering the subQueryTopicTargets filter that loads up to APP_LIMIT_SUBSCRIBERS_SUBQUERY (1M) subscribers plus a giant targets IN() into memory, then fanned every adapter-sized chunk out through a single unbounded Swoole\Coroutine\batch() — spawning unbounded concurrent sends. Rework the external send path into a single-job streaming model: - Recipients are streamed in bounded pages (MESSAGE_RECIPIENTS_PAGE_SIZE) via cursor pagination. Topics walk the subscribers collection by topicInternalId and resolve each page's targets by $sequence, so the topic's targets subquery is never materialized. Users/direct targets paginate the targets collection. Peak memory is O(page), not O(topic). - Send concurrency is bounded with Utopia\Lock\Semaphore (MESSAGE_SEND_CONCURRENCY permits): each adapter batch send is wrapped in $semaphore->withLock() and awaited per page with Swoole's native batch(). Excess coroutines block on acquire instead of all running at once. - Results accumulate across pages: an exact deliveredTotal/failedTotal partition (failed = recipients - delivered) and a delivery-errors list capped at MESSAGE_DELIVERY_ERRORS_LIMIT, so a 500K send never collects one error string per recipient. - An idempotency guard skips messages already in a terminal status, so queue redelivery does not reprocess a completed send. Only existing message fields (status, deliveredTotal, deliveryErrors) are written — no schema, migration, model, or queue-payload changes. Internal SMS and other transactional paths are untouched. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/init/constants.php | 4 + src/Appwrite/Platform/Workers/Messaging.php | 526 +++++++++++------ tests/unit/Messaging/MessagingFanoutTest.php | 528 ++++++++++++++++++ .../Messaging/SwooleCoroutinePolyfill.php | 160 ++++++ 4 files changed, 1040 insertions(+), 178 deletions(-) create mode 100644 tests/unit/Messaging/MessagingFanoutTest.php create mode 100644 tests/unit/Messaging/SwooleCoroutinePolyfill.php diff --git a/app/init/constants.php b/app/init/constants.php index d426e51fb6..2aa0d347f1 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -242,6 +242,10 @@ const RULE_STATUS_VERIFIED = 'verified'; // Message types const MESSAGE_SEND_TYPE_INTERNAL = 'internal'; const MESSAGE_SEND_TYPE_EXTERNAL = 'external'; +// External message fan-out tuning +const MESSAGE_RECIPIENTS_PAGE_SIZE = 1000; // Recipients resolved from the database per page while streaming a send +const MESSAGE_SEND_CONCURRENCY = 10; // Maximum adapter send requests running concurrently within a single send job +const MESSAGE_DELIVERY_ERRORS_LIMIT = 100; // Maximum number of per-recipient delivery errors retained on a message // Mail Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 7af079e914..a80110681b 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -16,6 +16,7 @@ use Utopia\Database\Document; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\DSN\DSN; +use Utopia\Lock\Semaphore; use Utopia\Logger\Log; use Utopia\Messaging\Adapter\Email as EmailAdapter; use Utopia\Messaging\Adapter\Email\Mailgun; @@ -130,69 +131,19 @@ class Messaging extends Action Document $project, UsagePublisher $publisherForUsage ): void { + $status = $message->getAttribute('status'); + + // Idempotency guard: queue redelivery can hand us a message that already finished. + if (\in_array($status, [MessageStatus::SENT, MessageStatus::FAILED], true)) { + Span::add('message.skipped', 'already_processed'); + return; + } + $topicIds = $message->getAttribute('topics', []); $targetIds = $message->getAttribute('targets', []); $userIds = $message->getAttribute('users', []); $providerType = $message->getAttribute('providerType'); - /** - * @var array $allTargets - */ - $allTargets = []; - - if (\count($topicIds) > 0) { - $topics = $dbForProject->find('topics', [ - Query::equal('$id', $topicIds), - Query::limit(\count($topicIds)), - ]); - foreach ($topics as $topic) { - $targets = \array_filter($topic->getAttribute('targets'), function (Document $target) use ($providerType) { - return $target->getAttribute('providerType') === $providerType; - }); - - \array_push($allTargets, ...$targets); - } - } - - if (\count($userIds) > 0) { - $limit = 1000; - $offset = 0; - - do { - $targets = $dbForProject->find('targets', [ - Query::equal('userId', $userIds), - Query::select(['providerId', 'identifier']), - Query::equal('providerType', [$providerType]), - Query::limit($limit), - Query::offset($offset), - ]); - - \array_push($allTargets, ...$targets); - $offset += \count($targets); - } while (\count($targets) === $limit); - } - - if (\count($targetIds) > 0) { - $targets = $dbForProject->find('targets', [ - Query::select(['providerId', 'identifier']), - Query::equal('$id', $targetIds), - Query::equal('providerType', [$providerType]), - Query::limit(\count($targetIds)), - ]); - - \array_push($allTargets, ...$targets); - } - - if (empty($allTargets)) { - $dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([ - 'status' => MessageStatus::FAILED, - 'deliveryErrors' => ['No valid recipients found.'] - ])); - - Span::add('message.skipped', 'no_valid_recipients'); - return; - } - $default = $dbForProject->findOne('providers', [ Query::equal('enabled', [true]), Query::equal('type', [$providerType]), @@ -209,55 +160,35 @@ class Messaging extends Action } /** - * @var array> $identifiers - */ - $identifiers = []; - - /** - * @var array $providers + * Resolved providers cached for the lifetime of this job, keyed by provider id. + * Seeded with the default provider so most sends never touch the providers collection. + * + * @var array $providers */ $providers = [ - $default->getId() => $default + $default->getId() => $default, ]; - foreach ($allTargets as $target) { - $providerId = $target->getAttribute('providerId'); + $semaphore = new Semaphore(MESSAGE_SEND_CONCURRENCY); - if (!$providerId) { - $providerId = $default->getId(); - } + $deliveredTotal = 0; + $failedTotal = 0; + $deliveryErrors = []; + $hasRecipients = false; - if ($providerId) { - if (!\array_key_exists($providerId, $identifiers)) { - $identifiers[$providerId] = []; - } - // Use null as value to avoid duplicate keys - $identifiers[$providerId][$target->getAttribute('identifier')] = null; - } - } + foreach ($this->streamRecipients($dbForProject, $topicIds, $userIds, $targetIds, $providerType, $default) as $page) { + $hasRecipients = true; - /** - * @var array $results - */ - $results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - if (\array_key_exists($providerId, $providers)) { - $provider = $providers[$providerId]; - } else { - $provider = $dbForProject->getDocument('providers', $providerId); + /** + * @var array $tasks + */ + $tasks = []; - if ($provider->isEmpty() || !$provider->getAttribute('enabled')) { - $provider = $default; - } else { - $providers[$providerId] = $provider; - } - } + foreach ($page as $providerId => $identifiers) { + $provider = $this->resolveProvider($dbForProject, $providerId, $providers, $default); + $resolvedProviderType = $provider->getAttribute('type'); - $identifiersForProvider = $identifiers[$providerId]; - - $providerType = $provider->getAttribute('type'); - - $adapter = match ($providerType) { + $adapter = match ($resolvedProviderType) { MESSAGE_TYPE_SMS => $this->getSmsAdapter($provider), MESSAGE_TYPE_PUSH => $this->getPushAdapter($provider), MESSAGE_TYPE_EMAIL => $this->getEmailAdapter($provider), @@ -265,101 +196,66 @@ class Messaging extends Action }; $batches = \array_chunk( - \array_keys($identifiersForProvider), + \array_keys($identifiers), $adapter->getMaxMessagesPerRequest() ); - return batch(\array_map(function ($batch) use ($message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - return function () use ($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - $deliveredTotal = 0; - $deliveryErrors = []; - $messageData = clone $message; - $messageData->setAttribute('to', $batch); + foreach ($batches as $batch) { + $tasks[] = fn (): array => $semaphore->withLock( + fn (): array => $this->sendBatch( + $batch, + $message, + $provider, + $resolvedProviderType, + $adapter, + $dbForProject, + $deviceForFiles, + $project, + $publisherForUsage + ) + ); + } + } - $data = match ($providerType) { - MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider), - MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData), - MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project), - }; + /** + * @var array}> $results + */ + $results = batch($tasks); - try { - $response = $adapter->send($data); - $deliveredTotal += (int) $response['deliveredTo']; - foreach ($response['results'] as $result) { - if ($result['status'] === 'failure') { - $deliveryErrors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}"; - } + foreach ($results as $result) { + $deliveredTotal += $result['delivered']; + $failedTotal += $result['recipients'] - $result['delivered']; - // Deleting push targets when token has expired. - if (($result['error'] ?? '') === 'Expired device token') { - $target = $dbForProject->findOne('targets', [ - Query::equal('identifier', [$result['recipient']]) - ]); + foreach ($result['errors'] as $error) { + if (\count($deliveryErrors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) { + break; + } - if (!$target->isEmpty()) { - $dbForProject->updateDocument( - 'targets', - $target->getId(), - $target->setAttribute('expired', true) - ); - } - } - } - } catch (\Throwable $e) { - $deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage(); - } finally { - $errorTotal = \count($deliveryErrors); - $usage = new UsageContext(); - $usage - ->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal)) - ->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal) - ->addMetric(METRIC_MESSAGES_FAILED, $errorTotal) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), ($deliveredTotal + $errorTotal)) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $deliveredTotal) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $errorTotal) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), ($deliveredTotal + $errorTotal)) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $deliveredTotal) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal); + $deliveryErrors[] = $error; + } + } + } - $publisherForUsage->enqueue(new Usage( - project: $project, - metrics: $usage->getMetrics(), - )); + if (!$hasRecipients) { + $dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([ + 'status' => MessageStatus::FAILED, + 'deliveryErrors' => ['No valid recipients found.'] + ])); - return [ - 'deliveredTotal' => $deliveredTotal, - 'deliveryErrors' => $deliveryErrors, - ]; - } - }; - }, $batches)); - }; - }, \array_keys($identifiers))); - - $results = \array_merge(...$results); - - $deliveredTotal = 0; - $deliveryErrors = []; - - foreach ($results as $result) { - $deliveredTotal += $result['deliveredTotal']; - $deliveryErrors = \array_merge($deliveryErrors, $result['deliveryErrors']); + Span::add('message.skipped', 'no_valid_recipients'); + return; } if (empty($deliveryErrors) && $deliveredTotal === 0) { $deliveryErrors[] = 'Unknown error'; } + $hasFailures = $failedTotal > 0 || \count($deliveryErrors) > 0; + $message->setAttribute('status', $hasFailures ? MessageStatus::FAILED : MessageStatus::SENT); $message->setAttribute('deliveryErrors', $deliveryErrors); - if (\count($message->getAttribute('deliveryErrors')) > 0) { - $message->setAttribute('status', MessageStatus::FAILED); - } else { - $message->setAttribute('status', MessageStatus::SENT); - } - Span::add('message.delivered_total', $deliveredTotal); - Span::add('message.errors_total', \count($deliveryErrors)); + Span::add('message.errors_total', $failedTotal); $message->removeAttribute('to'); @@ -410,6 +306,280 @@ class Messaging extends Action } } + /** + * Stream a message's recipients in bounded pages, grouped by provider and deduplicated by identifier within each page. + * + * Peak memory is O(MESSAGE_RECIPIENTS_PAGE_SIZE), never O(topic size): topics are walked through the + * subscribers collection with cursor pagination rather than reading the topic's `targets` attribute, which + * triggers the subQueryTopicTargets filter and loads up to APP_LIMIT_SUBSCRIBERS_SUBQUERY rows at once. + * + * @param array $topicIds + * @param array $userIds + * @param array $targetIds + * @return \Generator>> + * @throws \Exception + */ + private function streamRecipients( + Database $dbForProject, + array $topicIds, + array $userIds, + array $targetIds, + string $providerType, + Document $default + ): \Generator { + if (\count($topicIds) > 0) { + $topics = $dbForProject->find('topics', [ + Query::select(['$sequence']), + Query::equal('$id', $topicIds), + Query::limit(\count($topicIds)), + ]); + + foreach ($topics as $topic) { + $cursor = null; + + do { + $queries = [ + Query::equal('topicInternalId', [$topic->getSequence()]), + Query::equal('providerType', [$providerType]), + Query::select(['$sequence', 'targetInternalId']), + Query::orderAsc('$sequence'), + Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE), + ]; + + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + + $subscribers = $dbForProject->getAuthorization()->skip( + fn () => $dbForProject->find('subscribers', $queries) + ); + + $count = \count($subscribers); + + if ($count === 0) { + break; + } + + $cursor = $subscribers[$count - 1]; + + $targetInternalIds = \array_map( + fn (Document $subscriber) => $subscriber->getAttribute('targetInternalId'), + $subscribers + ); + + $targets = $dbForProject->skipValidation( + fn () => $dbForProject->getAuthorization()->skip( + fn () => $dbForProject->find('targets', [ + Query::equal('$sequence', $targetInternalIds), + Query::select(['providerId', 'identifier']), + Query::limit(\count($targetInternalIds)), + ]) + ) + ); + + yield $this->groupTargetsByProvider($targets, $default); + } while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE); + } + } + + if (\count($userIds) > 0) { + $cursor = null; + + do { + $queries = [ + Query::equal('userId', $userIds), + Query::equal('providerType', [$providerType]), + Query::select(['$sequence', 'providerId', 'identifier']), + Query::orderAsc('$sequence'), + Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE), + ]; + + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + + $targets = $dbForProject->find('targets', $queries); + $count = \count($targets); + + if ($count === 0) { + break; + } + + $cursor = $targets[$count - 1]; + + yield $this->groupTargetsByProvider($targets, $default); + } while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE); + } + + if (\count($targetIds) > 0) { + $cursor = null; + + do { + $queries = [ + Query::equal('$id', $targetIds), + Query::equal('providerType', [$providerType]), + Query::select(['$sequence', 'providerId', 'identifier']), + Query::orderAsc('$sequence'), + Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE), + ]; + + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + + $targets = $dbForProject->find('targets', $queries); + $count = \count($targets); + + if ($count === 0) { + break; + } + + $cursor = $targets[$count - 1]; + + yield $this->groupTargetsByProvider($targets, $default); + } while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE); + } + } + + /** + * Group a page of target documents by provider id, deduplicating identifiers within the page. + * + * @param array $targets + * @return array> + */ + private function groupTargetsByProvider(array $targets, Document $default): array + { + /** + * @var array> $identifiers + */ + $identifiers = []; + + foreach ($targets as $target) { + $providerId = $target->getAttribute('providerId') ?: $default->getId(); + + if (!\array_key_exists($providerId, $identifiers)) { + $identifiers[$providerId] = []; + } + + // Null values keep identifiers unique without a second lookup structure. + $identifiers[$providerId][$target->getAttribute('identifier')] = null; + } + + return $identifiers; + } + + /** + * Resolve and cache a provider for the lifetime of a send job, falling back to the default provider. + * + * @param array $providers + */ + private function resolveProvider( + Database $dbForProject, + string $providerId, + array &$providers, + Document $default + ): Document { + if (\array_key_exists($providerId, $providers)) { + return $providers[$providerId]; + } + + $provider = $dbForProject->getDocument('providers', $providerId); + + if ($provider->isEmpty() || !$provider->getAttribute('enabled')) { + return $default; + } + + $providers[$providerId] = $provider; + + return $provider; + } + + /** + * Send a single adapter-sized batch and report delivery counts plus a bounded error list. + * + * @param array $batch + * @return array{delivered: int, recipients: int, errors: array} + */ + private function sendBatch( + array $batch, + Document $message, + Document $provider, + string $providerType, + EmailAdapter|SMSAdapter|PushAdapter $adapter, + Database $dbForProject, + Device $deviceForFiles, + Document $project, + UsagePublisher $publisherForUsage + ): array { + $recipients = \count($batch); + $delivered = 0; + $errors = []; + + $messageData = clone $message; + $messageData->setAttribute('to', $batch); + + $data = match ($providerType) { + MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider), + MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData), + MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project), + default => throw new \Exception('Provider with the requested ID is of the incorrect type') + }; + + try { + $response = $adapter->send($data); + $delivered = (int) $response['deliveredTo']; + + foreach ($response['results'] as $result) { + if ($result['status'] === 'failure' && \count($errors) < MESSAGE_DELIVERY_ERRORS_LIMIT) { + $errors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}"; + } + + // Deleting push targets when token has expired. + if (($result['error'] ?? '') === 'Expired device token') { + $target = $dbForProject->findOne('targets', [ + Query::equal('identifier', [$result['recipient']]) + ]); + + if (!$target->isEmpty()) { + $dbForProject->updateDocument( + 'targets', + $target->getId(), + $target->setAttribute('expired', true) + ); + } + } + } + } catch (\Throwable $e) { + // Whole-batch failure: record one representative error and count every recipient as failed. + $delivered = 0; + $errors = ['Failed sending to targets with error: ' . $e->getMessage()]; + } finally { + $failed = $recipients - $delivered; + $usage = new UsageContext(); + $usage + ->addMetric(METRIC_MESSAGES, $recipients) + ->addMetric(METRIC_MESSAGES_SENT, $delivered) + ->addMetric(METRIC_MESSAGES_FAILED, $failed) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed); + + $publisherForUsage->enqueue(new Usage( + project: $project, + metrics: $usage->getMetrics(), + )); + } + + return [ + 'delivered' => $delivered, + 'recipients' => $recipients, + 'errors' => $errors, + ]; + } + private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void { if ($this->adapter === null) { @@ -451,7 +621,7 @@ class Messaging extends Action } - private function getSmsAdapter(Document $provider): ?SMSAdapter + protected function getSmsAdapter(Document $provider): ?SMSAdapter { $credentials = $provider->getAttribute('credentials'); @@ -494,7 +664,7 @@ class Messaging extends Action }; } - private function getPushAdapter(Document $provider): ?PushAdapter + protected function getPushAdapter(Document $provider): ?PushAdapter { $credentials = $provider->getAttribute('credentials'); $options = $provider->getAttribute('options'); @@ -513,7 +683,7 @@ class Messaging extends Action }; } - private function getEmailAdapter(Document $provider): ?EmailAdapter + protected function getEmailAdapter(Document $provider): ?EmailAdapter { $credentials = $provider->getAttribute('credentials', []); $options = $provider->getAttribute('options', []); diff --git a/tests/unit/Messaging/MessagingFanoutTest.php b/tests/unit/Messaging/MessagingFanoutTest.php new file mode 100644 index 0000000000..024cdbf43e --- /dev/null +++ b/tests/unit/Messaging/MessagingFanoutTest.php @@ -0,0 +1,528 @@ +maxPerRequest; + } + + /** + * @param EmailMessage $message + * @return array{deliveredTo: int, type: string, results: array>} + */ + protected function process(EmailMessage $message): array + { + $this->sendCalls++; + $this->inFlight++; + $this->peakConcurrency = \max($this->peakConcurrency, $this->inFlight); + + // Yield so other coroutines run while this "request" is in flight. + \Swoole\Coroutine::sleep(0.001); + + $response = new Response($this->getType()); + $delivered = 0; + + foreach ($message->getTo() as $recipient) { + // Email recipients are normalized to ['email' => ..., 'name' => ...] arrays by the message. + $email = $recipient['email']; + $fails = \is_callable($this->failureFor) && ($this->failureFor)($email); + + if ($fails) { + $response->addResult($email, 'Simulated delivery failure'); + } else { + $delivered++; + $response->addResult($email); + } + } + + $response->setDeliveredTo($delivered); + + $this->inFlight--; + + return $response->toArray(); + } +} + +/** + * Worker subclass that injects the instrumented email adapter instead of a network-backed provider adapter. + */ +class TestableMessaging extends Messaging +{ + public function __construct(private readonly EmailAdapter $emailAdapter) + { + } + + protected function getEmailAdapter(Document $provider): ?EmailAdapter + { + return $this->emailAdapter; + } +} + +/** + * In-memory {@see Database} double. Serves subscribers/targets through cursor pagination and records every + * query issued so the test can assert the streaming contract (cursorAfter + bounded page size, and that the + * 1M-row subQueryTopicTargets filter is never used). + */ +class RecordingDatabase extends Database +{ + /** + * @var array}> + */ + public array $findCalls = []; + + public ?Document $updatedMessage = null; + + private readonly Authorization $authorization; + + /** + * @param array $subscribers Subscriber rows (carry $sequence + targetInternalId + providerType). + * @param array $targets Target rows keyed by $sequence. + * @param array $directTargets Targets addressed directly by id or userId. + * @param Document $provider The single enabled provider. + */ + public function __construct( + private readonly array $subscribers = [], + private readonly array $targets = [], + private readonly array $directTargets = [], + private readonly ?Document $provider = null + ) { + $this->authorization = new Authorization(); + } + + public function getAuthorization(): Authorization + { + return $this->authorization; + } + + public function skipValidation(callable $callback): mixed + { + return $callback(); + } + + public function findOne(string $collection, array $queries = []): Document + { + if ($collection === 'providers') { + return $this->provider ?? new Document(); + } + + return new Document(); + } + + public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): Document + { + return new Document(); + } + + public function updateDocument(string $collection, string $id, Document $document): Document + { + if ($collection === 'messages') { + $this->updatedMessage = $document; + } + + return $document; + } + + public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array + { + $this->findCalls[] = ['collection' => $collection, 'queries' => $queries]; + + return match ($collection) { + 'topics' => [new Document(['$id' => 'topic1', '$sequence' => '100'])], + 'subscribers' => $this->paginate($this->subscribers, $queries), + 'targets' => $this->resolveTargets($queries), + default => [], + }; + } + + /** + * @param array $rows + * @param array $queries + * @return array + */ + private function paginate(array $rows, array $queries): array + { + $limit = $this->limitOf($queries); + $cursor = $this->cursorOf($queries); + + if ($cursor !== null) { + $rows = \array_values(\array_filter( + $rows, + fn (Document $row) => $row->getSequence() > $cursor->getSequence() + )); + } + + return \array_slice($rows, 0, $limit); + } + + /** + * Either resolve a subscriber page's targets by $sequence, or paginate direct/user targets. + * + * @param array $queries + * @return array + */ + private function resolveTargets(array $queries): array + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === '$sequence') { + $sequences = $query->getValues(); + + return \array_values(\array_map( + fn ($sequence) => $this->targets[$sequence], + \array_filter($sequences, fn ($sequence) => isset($this->targets[$sequence])) + )); + } + } + + return $this->paginate($this->directTargets, $queries); + } + + /** + * @param array $queries + */ + private function limitOf(array $queries): int + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_LIMIT) { + return (int) ($query->getValues()[0] ?? 0); + } + } + + return 0; + } + + /** + * @param array $queries + */ + private function cursorOf(array $queries): ?Document + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) { + $value = $query->getValues()[0] ?? null; + + return $value instanceof Document ? $value : null; + } + } + + return null; + } +} + +class MessagingFanoutTest extends TestCase +{ + private const RECIPIENT_COUNT = 4500; + + private function provider(): Document + { + return new Document([ + '$id' => 'provider1', + '$sequence' => '1', + 'enabled' => true, + 'type' => MESSAGE_TYPE_EMAIL, + 'provider' => 'mock', + 'name' => 'Mock Email', + 'credentials' => [], + 'options' => [ + 'fromName' => 'Appwrite', + 'fromEmail' => 'noreply@example.com', + ], + ]); + } + + private function message(): Document + { + return new Document([ + '$id' => 'message1', + '$sequence' => '1', + 'providerType' => MESSAGE_TYPE_EMAIL, + 'topics' => ['topic1'], + 'targets' => [], + 'users' => [], + 'status' => MessageStatus::PROCESSING, + 'search' => 'message1', + 'data' => [ + 'subject' => 'Hello', + 'content' => 'World', + 'html' => false, + ], + ]); + } + + /** + * Build a topic-backed dataset: RECIPIENT_COUNT subscribers, each pointing at one email target. + * + * @return array{subscribers: array, targets: array} + */ + private function topicDataset(int $count): array + { + $subscribers = []; + $targets = []; + + for ($i = 1; $i <= $count; $i++) { + $sequence = (string) $i; + $subscribers[] = new Document([ + '$id' => 'sub' . $i, + '$sequence' => $sequence, + 'topicInternalId' => '100', + 'targetInternalId' => $sequence, + 'providerType' => MESSAGE_TYPE_EMAIL, + ]); + $targets[$sequence] = new Document([ + '$id' => 'target' . $i, + '$sequence' => $sequence, + 'providerId' => null, + 'identifier' => "user{$i}@example.com", + 'providerType' => MESSAGE_TYPE_EMAIL, + ]); + } + + return ['subscribers' => $subscribers, 'targets' => $targets]; + } + + private function sendTopic( + RecordingDatabase $database, + ConcurrencyProbeAdapter $adapter + ): void { + $worker = new TestableMessaging($adapter); + $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); + + $publisher = $this->createMock(UsagePublisher::class); + $publisher->method('enqueue')->willReturn(true); + + $device = new Local(\sys_get_temp_dir()); + $project = new Document(['$id' => 'project1', '$sequence' => '1']); + + run(function () use ($worker, $method, $database, $device, $project, $publisher) { + $method->invoke($worker, $database, $this->message(), $device, $project, $publisher); + }); + } + + public function testRecipientsArePaginatedWithoutSubqueryFilter(): void + { + $data = $this->topicDataset(self::RECIPIENT_COUNT); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + $adapter = new ConcurrencyProbeAdapter(); + + $this->sendTopic($database, $adapter); + + $topicReads = \array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics'); + $subscriberReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'subscribers')); + + // The topic's `targets` attribute (the subQueryTopicTargets filter) is never materialized: topics are + // only read with a $sequence projection, and recipients come from the subscribers collection instead. + $this->assertNotEmpty($topicReads); + foreach ($topicReads as $call) { + $selected = $this->selectFields($call['queries']); + $this->assertNotContains('targets', $selected, 'Topic read must not select the targets subquery attribute'); + } + + // Subscribers are walked in bounded pages via cursorAfter. 4500 recipients over a 1000 page size is + // five pages; every page after the first carries a cursor and none exceeds the page size. + $expectedPages = (int) \ceil(self::RECIPIENT_COUNT / MESSAGE_RECIPIENTS_PAGE_SIZE); + $this->assertCount($expectedPages, $subscriberReads); + + foreach ($subscriberReads as $index => $call) { + $this->assertSame(MESSAGE_RECIPIENTS_PAGE_SIZE, $this->limitOf($call['queries'])); + + if ($index === 0) { + $this->assertFalse($this->hasCursor($call['queries']), 'First page must not carry a cursor'); + } else { + $this->assertTrue($this->hasCursor($call['queries']), 'Subsequent pages must paginate with cursorAfter'); + } + } + } + + public function testConcurrentSendsNeverExceedConfiguredLimit(): void + { + // One recipient per batch (maxPerRequest = 1) maximises the number of concurrent in-flight sends so + // the Semaphore's bound is actually exercised: 1500 batches contend for MESSAGE_SEND_CONCURRENCY permits. + $count = 1500; + $data = $this->topicDataset($count); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + $adapter = new ConcurrencyProbeAdapter(maxPerRequest: 1); + + $this->sendTopic($database, $adapter); + + $this->assertSame($count, $adapter->sendCalls, 'Every recipient is sent exactly once'); + $this->assertGreaterThan(1, $adapter->peakConcurrency, 'Sends should actually run concurrently'); + $this->assertLessThanOrEqual( + MESSAGE_SEND_CONCURRENCY, + $adapter->peakConcurrency, + 'Concurrent sends must never exceed MESSAGE_SEND_CONCURRENCY' + ); + } + + public function testDeliveryErrorsAreCappedWhileFailedCountIsComplete(): void + { + $count = self::RECIPIENT_COUNT; + $data = $this->topicDataset($count); + // Every recipient fails: 4500 individual failures, but only MESSAGE_DELIVERY_ERRORS_LIMIT are retained. + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + $adapter = new ConcurrencyProbeAdapter(failureFor: fn (string $recipient) => true); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + + $errors = $message->getAttribute('deliveryErrors'); + $this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $errors, 'Delivery errors must be capped'); + + // Failed recipients are fully accounted for even though only 100 error strings are kept: every one of + // the 4500 recipients failed, so deliveredTotal is zero and the message is marked FAILED. + $this->assertSame(0, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::FAILED, $message->getAttribute('status')); + } + + public function testDeliveredTotalAndStatusReflectPartialSuccess(): void + { + $count = self::RECIPIENT_COUNT; + $data = $this->topicDataset($count); + // Fail exactly the recipients whose local-part index is divisible by 10. + $failing = static function (string $recipient): bool { + \preg_match('/user(\d+)@/', $recipient, $matches); + + return isset($matches[1]) && ((int) $matches[1]) % 10 === 0; + }; + $expectedFailures = \intdiv($count, 10); + + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + $adapter = new ConcurrencyProbeAdapter(failureFor: $failing); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + + // deliveredTotal is exact across all pages; failures partition the remainder exactly. + $this->assertSame($count - $expectedFailures, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::FAILED, $message->getAttribute('status')); + $this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $message->getAttribute('deliveryErrors')); + } + + public function testFullSuccessMarksMessageSent(): void + { + $data = $this->topicDataset(self::RECIPIENT_COUNT); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + $adapter = new ConcurrencyProbeAdapter(); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + $this->assertSame(self::RECIPIENT_COUNT, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::SENT, $message->getAttribute('status')); + $this->assertEmpty($message->getAttribute('deliveryErrors')); + } + + public function testTerminalStatusIsSkipped(): void + { + $data = $this->topicDataset(10); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + $adapter = new ConcurrencyProbeAdapter(); + + $worker = new TestableMessaging($adapter); + $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); + $publisher = $this->createMock(UsagePublisher::class); + $device = new Local(\sys_get_temp_dir()); + $project = new Document(['$id' => 'project1', '$sequence' => '1']); + + $message = $this->message(); + $message->setAttribute('status', MessageStatus::SENT); + + run(function () use ($worker, $method, $database, $device, $project, $publisher, $message) { + $method->invoke($worker, $database, $message, $device, $project, $publisher); + }); + + $this->assertSame(0, $adapter->sendCalls, 'Already-sent messages must not be reprocessed'); + $this->assertNull($database->updatedMessage, 'Terminal messages must not be rewritten'); + } + + /** + * @param array $queries + * @return array + */ + private function selectFields(array $queries): array + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_SELECT) { + return $query->getValues(); + } + } + + return []; + } + + /** + * @param array $queries + */ + private function limitOf(array $queries): int + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_LIMIT) { + return (int) ($query->getValues()[0] ?? 0); + } + } + + return 0; + } + + /** + * @param array $queries + */ + private function hasCursor(array $queries): bool + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) { + return true; + } + } + + return false; + } +} diff --git a/tests/unit/Messaging/SwooleCoroutinePolyfill.php b/tests/unit/Messaging/SwooleCoroutinePolyfill.php new file mode 100644 index 0000000000..b509d84bec --- /dev/null +++ b/tests/unit/Messaging/SwooleCoroutinePolyfill.php @@ -0,0 +1,160 @@ + + */ + private array $buffer = []; + + public function __construct(private readonly int $capacity = 1) + { + } + + public function push(mixed $data, float $timeout = 0): bool + { + $this->buffer[] = $data; + + return true; + } + + public function pop(float $timeout = 0): mixed + { + return \array_shift($this->buffer); + } + + public function isFull(): bool + { + return \count($this->buffer) >= $this->capacity; + } + + public function isEmpty(): bool + { + return \count($this->buffer) === 0; + } + } + } +} + +namespace Swoole\Coroutine { + use Swoole\Coroutine; + use Utopia\Lock\Exception\Contention; + + if (! \function_exists('Swoole\Coroutine\run')) { + function run(callable $callback): void + { + $fiber = new \Fiber($callback); + $previous = Coroutine::$current; + Coroutine::$current = $fiber; + $fiber->start(); + + while (! $fiber->isTerminated()) { + $fiber->resume(); + } + + Coroutine::$current = $previous; + } + } + + if (! \function_exists('Swoole\Coroutine\batch')) { + /** + * @param array $tasks + * @return array + */ + function batch(array $tasks, float $timeout = -1): array + { + $results = []; + $done = []; + $fibers = []; + + foreach ($tasks as $key => $task) { + $fibers[$key] = new \Fiber($task); + } + + $previous = Coroutine::$current; + + while (\count($done) < \count($tasks)) { + $progressed = false; + + foreach ($fibers as $key => $fiber) { + if (isset($done[$key])) { + continue; + } + + Coroutine::$current = $fiber; + + try { + if (! $fiber->isStarted()) { + $fiber->start(); + } elseif (! $fiber->isTerminated()) { + $fiber->resume(); + } + + // Any successful start/resume advances a fiber (toward a yield or termination), which + // counts as progress; fibers advance monotonically so this cannot spin forever. + $progressed = true; + + if ($fiber->isTerminated()) { + $results[$key] = $fiber->getReturn(); + $done[$key] = true; + } + } catch (Contention) { + // All permits held: model a blocking acquire by retrying the task on a later tick. + $fibers[$key] = new \Fiber($tasks[$key]); + $progressed = true; + } + } + + if (! $progressed) { + break; + } + } + + Coroutine::$current = $previous; + \ksort($results); + + return $results; + } + } +}