From da061ba7d299e097668a3cfeab304b3656d5f3f0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Jun 2026 02:10:47 +1200 Subject: [PATCH 1/9] fix(messaging): stream large-topic sends and bound send concurrency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Large external sends (e.g. a 500K-recipient topic email) OOM'd the messaging worker: sendExternalMessage read $topic->getAttribute('targets'), triggering the subQueryTopicTargets filter that loads up to APP_LIMIT_SUBSCRIBERS_SUBQUERY (1M) subscribers plus a giant targets IN() into memory, then fanned every adapter-sized chunk out through a single unbounded Swoole\Coroutine\batch() — spawning unbounded concurrent sends. Rework the external send path into a single-job streaming model: - Recipients are streamed in bounded pages (MESSAGE_RECIPIENTS_PAGE_SIZE) via cursor pagination. Topics walk the subscribers collection by topicInternalId and resolve each page's targets by $sequence, so the topic's targets subquery is never materialized. Users/direct targets paginate the targets collection. Peak memory is O(page), not O(topic). - Send concurrency is bounded with Utopia\Lock\Semaphore (MESSAGE_SEND_CONCURRENCY permits): each adapter batch send is wrapped in $semaphore->withLock() and awaited per page with Swoole's native batch(). Excess coroutines block on acquire instead of all running at once. - Results accumulate across pages: an exact deliveredTotal/failedTotal partition (failed = recipients - delivered) and a delivery-errors list capped at MESSAGE_DELIVERY_ERRORS_LIMIT, so a 500K send never collects one error string per recipient. - An idempotency guard skips messages already in a terminal status, so queue redelivery does not reprocess a completed send. Only existing message fields (status, deliveredTotal, deliveryErrors) are written — no schema, migration, model, or queue-payload changes. Internal SMS and other transactional paths are untouched. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/init/constants.php | 4 + src/Appwrite/Platform/Workers/Messaging.php | 526 +++++++++++------ tests/unit/Messaging/MessagingFanoutTest.php | 528 ++++++++++++++++++ .../Messaging/SwooleCoroutinePolyfill.php | 160 ++++++ 4 files changed, 1040 insertions(+), 178 deletions(-) create mode 100644 tests/unit/Messaging/MessagingFanoutTest.php create mode 100644 tests/unit/Messaging/SwooleCoroutinePolyfill.php diff --git a/app/init/constants.php b/app/init/constants.php index d426e51fb6..2aa0d347f1 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -242,6 +242,10 @@ const RULE_STATUS_VERIFIED = 'verified'; // Message types const MESSAGE_SEND_TYPE_INTERNAL = 'internal'; const MESSAGE_SEND_TYPE_EXTERNAL = 'external'; +// External message fan-out tuning +const MESSAGE_RECIPIENTS_PAGE_SIZE = 1000; // Recipients resolved from the database per page while streaming a send +const MESSAGE_SEND_CONCURRENCY = 10; // Maximum adapter send requests running concurrently within a single send job +const MESSAGE_DELIVERY_ERRORS_LIMIT = 100; // Maximum number of per-recipient delivery errors retained on a message // Mail Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 7af079e914..a80110681b 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -16,6 +16,7 @@ use Utopia\Database\Document; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\DSN\DSN; +use Utopia\Lock\Semaphore; use Utopia\Logger\Log; use Utopia\Messaging\Adapter\Email as EmailAdapter; use Utopia\Messaging\Adapter\Email\Mailgun; @@ -130,69 +131,19 @@ class Messaging extends Action Document $project, UsagePublisher $publisherForUsage ): void { + $status = $message->getAttribute('status'); + + // Idempotency guard: queue redelivery can hand us a message that already finished. + if (\in_array($status, [MessageStatus::SENT, MessageStatus::FAILED], true)) { + Span::add('message.skipped', 'already_processed'); + return; + } + $topicIds = $message->getAttribute('topics', []); $targetIds = $message->getAttribute('targets', []); $userIds = $message->getAttribute('users', []); $providerType = $message->getAttribute('providerType'); - /** - * @var array $allTargets - */ - $allTargets = []; - - if (\count($topicIds) > 0) { - $topics = $dbForProject->find('topics', [ - Query::equal('$id', $topicIds), - Query::limit(\count($topicIds)), - ]); - foreach ($topics as $topic) { - $targets = \array_filter($topic->getAttribute('targets'), function (Document $target) use ($providerType) { - return $target->getAttribute('providerType') === $providerType; - }); - - \array_push($allTargets, ...$targets); - } - } - - if (\count($userIds) > 0) { - $limit = 1000; - $offset = 0; - - do { - $targets = $dbForProject->find('targets', [ - Query::equal('userId', $userIds), - Query::select(['providerId', 'identifier']), - Query::equal('providerType', [$providerType]), - Query::limit($limit), - Query::offset($offset), - ]); - - \array_push($allTargets, ...$targets); - $offset += \count($targets); - } while (\count($targets) === $limit); - } - - if (\count($targetIds) > 0) { - $targets = $dbForProject->find('targets', [ - Query::select(['providerId', 'identifier']), - Query::equal('$id', $targetIds), - Query::equal('providerType', [$providerType]), - Query::limit(\count($targetIds)), - ]); - - \array_push($allTargets, ...$targets); - } - - if (empty($allTargets)) { - $dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([ - 'status' => MessageStatus::FAILED, - 'deliveryErrors' => ['No valid recipients found.'] - ])); - - Span::add('message.skipped', 'no_valid_recipients'); - return; - } - $default = $dbForProject->findOne('providers', [ Query::equal('enabled', [true]), Query::equal('type', [$providerType]), @@ -209,55 +160,35 @@ class Messaging extends Action } /** - * @var array> $identifiers - */ - $identifiers = []; - - /** - * @var array $providers + * Resolved providers cached for the lifetime of this job, keyed by provider id. + * Seeded with the default provider so most sends never touch the providers collection. + * + * @var array $providers */ $providers = [ - $default->getId() => $default + $default->getId() => $default, ]; - foreach ($allTargets as $target) { - $providerId = $target->getAttribute('providerId'); + $semaphore = new Semaphore(MESSAGE_SEND_CONCURRENCY); - if (!$providerId) { - $providerId = $default->getId(); - } + $deliveredTotal = 0; + $failedTotal = 0; + $deliveryErrors = []; + $hasRecipients = false; - if ($providerId) { - if (!\array_key_exists($providerId, $identifiers)) { - $identifiers[$providerId] = []; - } - // Use null as value to avoid duplicate keys - $identifiers[$providerId][$target->getAttribute('identifier')] = null; - } - } + foreach ($this->streamRecipients($dbForProject, $topicIds, $userIds, $targetIds, $providerType, $default) as $page) { + $hasRecipients = true; - /** - * @var array $results - */ - $results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - if (\array_key_exists($providerId, $providers)) { - $provider = $providers[$providerId]; - } else { - $provider = $dbForProject->getDocument('providers', $providerId); + /** + * @var array $tasks + */ + $tasks = []; - if ($provider->isEmpty() || !$provider->getAttribute('enabled')) { - $provider = $default; - } else { - $providers[$providerId] = $provider; - } - } + foreach ($page as $providerId => $identifiers) { + $provider = $this->resolveProvider($dbForProject, $providerId, $providers, $default); + $resolvedProviderType = $provider->getAttribute('type'); - $identifiersForProvider = $identifiers[$providerId]; - - $providerType = $provider->getAttribute('type'); - - $adapter = match ($providerType) { + $adapter = match ($resolvedProviderType) { MESSAGE_TYPE_SMS => $this->getSmsAdapter($provider), MESSAGE_TYPE_PUSH => $this->getPushAdapter($provider), MESSAGE_TYPE_EMAIL => $this->getEmailAdapter($provider), @@ -265,101 +196,66 @@ class Messaging extends Action }; $batches = \array_chunk( - \array_keys($identifiersForProvider), + \array_keys($identifiers), $adapter->getMaxMessagesPerRequest() ); - return batch(\array_map(function ($batch) use ($message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - return function () use ($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - $deliveredTotal = 0; - $deliveryErrors = []; - $messageData = clone $message; - $messageData->setAttribute('to', $batch); + foreach ($batches as $batch) { + $tasks[] = fn (): array => $semaphore->withLock( + fn (): array => $this->sendBatch( + $batch, + $message, + $provider, + $resolvedProviderType, + $adapter, + $dbForProject, + $deviceForFiles, + $project, + $publisherForUsage + ) + ); + } + } - $data = match ($providerType) { - MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider), - MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData), - MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project), - }; + /** + * @var array}> $results + */ + $results = batch($tasks); - try { - $response = $adapter->send($data); - $deliveredTotal += (int) $response['deliveredTo']; - foreach ($response['results'] as $result) { - if ($result['status'] === 'failure') { - $deliveryErrors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}"; - } + foreach ($results as $result) { + $deliveredTotal += $result['delivered']; + $failedTotal += $result['recipients'] - $result['delivered']; - // Deleting push targets when token has expired. - if (($result['error'] ?? '') === 'Expired device token') { - $target = $dbForProject->findOne('targets', [ - Query::equal('identifier', [$result['recipient']]) - ]); + foreach ($result['errors'] as $error) { + if (\count($deliveryErrors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) { + break; + } - if (!$target->isEmpty()) { - $dbForProject->updateDocument( - 'targets', - $target->getId(), - $target->setAttribute('expired', true) - ); - } - } - } - } catch (\Throwable $e) { - $deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage(); - } finally { - $errorTotal = \count($deliveryErrors); - $usage = new UsageContext(); - $usage - ->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal)) - ->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal) - ->addMetric(METRIC_MESSAGES_FAILED, $errorTotal) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), ($deliveredTotal + $errorTotal)) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $deliveredTotal) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $errorTotal) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), ($deliveredTotal + $errorTotal)) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $deliveredTotal) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal); + $deliveryErrors[] = $error; + } + } + } - $publisherForUsage->enqueue(new Usage( - project: $project, - metrics: $usage->getMetrics(), - )); + if (!$hasRecipients) { + $dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([ + 'status' => MessageStatus::FAILED, + 'deliveryErrors' => ['No valid recipients found.'] + ])); - return [ - 'deliveredTotal' => $deliveredTotal, - 'deliveryErrors' => $deliveryErrors, - ]; - } - }; - }, $batches)); - }; - }, \array_keys($identifiers))); - - $results = \array_merge(...$results); - - $deliveredTotal = 0; - $deliveryErrors = []; - - foreach ($results as $result) { - $deliveredTotal += $result['deliveredTotal']; - $deliveryErrors = \array_merge($deliveryErrors, $result['deliveryErrors']); + Span::add('message.skipped', 'no_valid_recipients'); + return; } if (empty($deliveryErrors) && $deliveredTotal === 0) { $deliveryErrors[] = 'Unknown error'; } + $hasFailures = $failedTotal > 0 || \count($deliveryErrors) > 0; + $message->setAttribute('status', $hasFailures ? MessageStatus::FAILED : MessageStatus::SENT); $message->setAttribute('deliveryErrors', $deliveryErrors); - if (\count($message->getAttribute('deliveryErrors')) > 0) { - $message->setAttribute('status', MessageStatus::FAILED); - } else { - $message->setAttribute('status', MessageStatus::SENT); - } - Span::add('message.delivered_total', $deliveredTotal); - Span::add('message.errors_total', \count($deliveryErrors)); + Span::add('message.errors_total', $failedTotal); $message->removeAttribute('to'); @@ -410,6 +306,280 @@ class Messaging extends Action } } + /** + * Stream a message's recipients in bounded pages, grouped by provider and deduplicated by identifier within each page. + * + * Peak memory is O(MESSAGE_RECIPIENTS_PAGE_SIZE), never O(topic size): topics are walked through the + * subscribers collection with cursor pagination rather than reading the topic's `targets` attribute, which + * triggers the subQueryTopicTargets filter and loads up to APP_LIMIT_SUBSCRIBERS_SUBQUERY rows at once. + * + * @param array $topicIds + * @param array $userIds + * @param array $targetIds + * @return \Generator>> + * @throws \Exception + */ + private function streamRecipients( + Database $dbForProject, + array $topicIds, + array $userIds, + array $targetIds, + string $providerType, + Document $default + ): \Generator { + if (\count($topicIds) > 0) { + $topics = $dbForProject->find('topics', [ + Query::select(['$sequence']), + Query::equal('$id', $topicIds), + Query::limit(\count($topicIds)), + ]); + + foreach ($topics as $topic) { + $cursor = null; + + do { + $queries = [ + Query::equal('topicInternalId', [$topic->getSequence()]), + Query::equal('providerType', [$providerType]), + Query::select(['$sequence', 'targetInternalId']), + Query::orderAsc('$sequence'), + Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE), + ]; + + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + + $subscribers = $dbForProject->getAuthorization()->skip( + fn () => $dbForProject->find('subscribers', $queries) + ); + + $count = \count($subscribers); + + if ($count === 0) { + break; + } + + $cursor = $subscribers[$count - 1]; + + $targetInternalIds = \array_map( + fn (Document $subscriber) => $subscriber->getAttribute('targetInternalId'), + $subscribers + ); + + $targets = $dbForProject->skipValidation( + fn () => $dbForProject->getAuthorization()->skip( + fn () => $dbForProject->find('targets', [ + Query::equal('$sequence', $targetInternalIds), + Query::select(['providerId', 'identifier']), + Query::limit(\count($targetInternalIds)), + ]) + ) + ); + + yield $this->groupTargetsByProvider($targets, $default); + } while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE); + } + } + + if (\count($userIds) > 0) { + $cursor = null; + + do { + $queries = [ + Query::equal('userId', $userIds), + Query::equal('providerType', [$providerType]), + Query::select(['$sequence', 'providerId', 'identifier']), + Query::orderAsc('$sequence'), + Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE), + ]; + + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + + $targets = $dbForProject->find('targets', $queries); + $count = \count($targets); + + if ($count === 0) { + break; + } + + $cursor = $targets[$count - 1]; + + yield $this->groupTargetsByProvider($targets, $default); + } while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE); + } + + if (\count($targetIds) > 0) { + $cursor = null; + + do { + $queries = [ + Query::equal('$id', $targetIds), + Query::equal('providerType', [$providerType]), + Query::select(['$sequence', 'providerId', 'identifier']), + Query::orderAsc('$sequence'), + Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE), + ]; + + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + + $targets = $dbForProject->find('targets', $queries); + $count = \count($targets); + + if ($count === 0) { + break; + } + + $cursor = $targets[$count - 1]; + + yield $this->groupTargetsByProvider($targets, $default); + } while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE); + } + } + + /** + * Group a page of target documents by provider id, deduplicating identifiers within the page. + * + * @param array $targets + * @return array> + */ + private function groupTargetsByProvider(array $targets, Document $default): array + { + /** + * @var array> $identifiers + */ + $identifiers = []; + + foreach ($targets as $target) { + $providerId = $target->getAttribute('providerId') ?: $default->getId(); + + if (!\array_key_exists($providerId, $identifiers)) { + $identifiers[$providerId] = []; + } + + // Null values keep identifiers unique without a second lookup structure. + $identifiers[$providerId][$target->getAttribute('identifier')] = null; + } + + return $identifiers; + } + + /** + * Resolve and cache a provider for the lifetime of a send job, falling back to the default provider. + * + * @param array $providers + */ + private function resolveProvider( + Database $dbForProject, + string $providerId, + array &$providers, + Document $default + ): Document { + if (\array_key_exists($providerId, $providers)) { + return $providers[$providerId]; + } + + $provider = $dbForProject->getDocument('providers', $providerId); + + if ($provider->isEmpty() || !$provider->getAttribute('enabled')) { + return $default; + } + + $providers[$providerId] = $provider; + + return $provider; + } + + /** + * Send a single adapter-sized batch and report delivery counts plus a bounded error list. + * + * @param array $batch + * @return array{delivered: int, recipients: int, errors: array} + */ + private function sendBatch( + array $batch, + Document $message, + Document $provider, + string $providerType, + EmailAdapter|SMSAdapter|PushAdapter $adapter, + Database $dbForProject, + Device $deviceForFiles, + Document $project, + UsagePublisher $publisherForUsage + ): array { + $recipients = \count($batch); + $delivered = 0; + $errors = []; + + $messageData = clone $message; + $messageData->setAttribute('to', $batch); + + $data = match ($providerType) { + MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider), + MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData), + MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project), + default => throw new \Exception('Provider with the requested ID is of the incorrect type') + }; + + try { + $response = $adapter->send($data); + $delivered = (int) $response['deliveredTo']; + + foreach ($response['results'] as $result) { + if ($result['status'] === 'failure' && \count($errors) < MESSAGE_DELIVERY_ERRORS_LIMIT) { + $errors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}"; + } + + // Deleting push targets when token has expired. + if (($result['error'] ?? '') === 'Expired device token') { + $target = $dbForProject->findOne('targets', [ + Query::equal('identifier', [$result['recipient']]) + ]); + + if (!$target->isEmpty()) { + $dbForProject->updateDocument( + 'targets', + $target->getId(), + $target->setAttribute('expired', true) + ); + } + } + } + } catch (\Throwable $e) { + // Whole-batch failure: record one representative error and count every recipient as failed. + $delivered = 0; + $errors = ['Failed sending to targets with error: ' . $e->getMessage()]; + } finally { + $failed = $recipients - $delivered; + $usage = new UsageContext(); + $usage + ->addMetric(METRIC_MESSAGES, $recipients) + ->addMetric(METRIC_MESSAGES_SENT, $delivered) + ->addMetric(METRIC_MESSAGES_FAILED, $failed) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed); + + $publisherForUsage->enqueue(new Usage( + project: $project, + metrics: $usage->getMetrics(), + )); + } + + return [ + 'delivered' => $delivered, + 'recipients' => $recipients, + 'errors' => $errors, + ]; + } + private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void { if ($this->adapter === null) { @@ -451,7 +621,7 @@ class Messaging extends Action } - private function getSmsAdapter(Document $provider): ?SMSAdapter + protected function getSmsAdapter(Document $provider): ?SMSAdapter { $credentials = $provider->getAttribute('credentials'); @@ -494,7 +664,7 @@ class Messaging extends Action }; } - private function getPushAdapter(Document $provider): ?PushAdapter + protected function getPushAdapter(Document $provider): ?PushAdapter { $credentials = $provider->getAttribute('credentials'); $options = $provider->getAttribute('options'); @@ -513,7 +683,7 @@ class Messaging extends Action }; } - private function getEmailAdapter(Document $provider): ?EmailAdapter + protected function getEmailAdapter(Document $provider): ?EmailAdapter { $credentials = $provider->getAttribute('credentials', []); $options = $provider->getAttribute('options', []); diff --git a/tests/unit/Messaging/MessagingFanoutTest.php b/tests/unit/Messaging/MessagingFanoutTest.php new file mode 100644 index 0000000000..024cdbf43e --- /dev/null +++ b/tests/unit/Messaging/MessagingFanoutTest.php @@ -0,0 +1,528 @@ +maxPerRequest; + } + + /** + * @param EmailMessage $message + * @return array{deliveredTo: int, type: string, results: array>} + */ + protected function process(EmailMessage $message): array + { + $this->sendCalls++; + $this->inFlight++; + $this->peakConcurrency = \max($this->peakConcurrency, $this->inFlight); + + // Yield so other coroutines run while this "request" is in flight. + \Swoole\Coroutine::sleep(0.001); + + $response = new Response($this->getType()); + $delivered = 0; + + foreach ($message->getTo() as $recipient) { + // Email recipients are normalized to ['email' => ..., 'name' => ...] arrays by the message. + $email = $recipient['email']; + $fails = \is_callable($this->failureFor) && ($this->failureFor)($email); + + if ($fails) { + $response->addResult($email, 'Simulated delivery failure'); + } else { + $delivered++; + $response->addResult($email); + } + } + + $response->setDeliveredTo($delivered); + + $this->inFlight--; + + return $response->toArray(); + } +} + +/** + * Worker subclass that injects the instrumented email adapter instead of a network-backed provider adapter. + */ +class TestableMessaging extends Messaging +{ + public function __construct(private readonly EmailAdapter $emailAdapter) + { + } + + protected function getEmailAdapter(Document $provider): ?EmailAdapter + { + return $this->emailAdapter; + } +} + +/** + * In-memory {@see Database} double. Serves subscribers/targets through cursor pagination and records every + * query issued so the test can assert the streaming contract (cursorAfter + bounded page size, and that the + * 1M-row subQueryTopicTargets filter is never used). + */ +class RecordingDatabase extends Database +{ + /** + * @var array}> + */ + public array $findCalls = []; + + public ?Document $updatedMessage = null; + + private readonly Authorization $authorization; + + /** + * @param array $subscribers Subscriber rows (carry $sequence + targetInternalId + providerType). + * @param array $targets Target rows keyed by $sequence. + * @param array $directTargets Targets addressed directly by id or userId. + * @param Document $provider The single enabled provider. + */ + public function __construct( + private readonly array $subscribers = [], + private readonly array $targets = [], + private readonly array $directTargets = [], + private readonly ?Document $provider = null + ) { + $this->authorization = new Authorization(); + } + + public function getAuthorization(): Authorization + { + return $this->authorization; + } + + public function skipValidation(callable $callback): mixed + { + return $callback(); + } + + public function findOne(string $collection, array $queries = []): Document + { + if ($collection === 'providers') { + return $this->provider ?? new Document(); + } + + return new Document(); + } + + public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): Document + { + return new Document(); + } + + public function updateDocument(string $collection, string $id, Document $document): Document + { + if ($collection === 'messages') { + $this->updatedMessage = $document; + } + + return $document; + } + + public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array + { + $this->findCalls[] = ['collection' => $collection, 'queries' => $queries]; + + return match ($collection) { + 'topics' => [new Document(['$id' => 'topic1', '$sequence' => '100'])], + 'subscribers' => $this->paginate($this->subscribers, $queries), + 'targets' => $this->resolveTargets($queries), + default => [], + }; + } + + /** + * @param array $rows + * @param array $queries + * @return array + */ + private function paginate(array $rows, array $queries): array + { + $limit = $this->limitOf($queries); + $cursor = $this->cursorOf($queries); + + if ($cursor !== null) { + $rows = \array_values(\array_filter( + $rows, + fn (Document $row) => $row->getSequence() > $cursor->getSequence() + )); + } + + return \array_slice($rows, 0, $limit); + } + + /** + * Either resolve a subscriber page's targets by $sequence, or paginate direct/user targets. + * + * @param array $queries + * @return array + */ + private function resolveTargets(array $queries): array + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === '$sequence') { + $sequences = $query->getValues(); + + return \array_values(\array_map( + fn ($sequence) => $this->targets[$sequence], + \array_filter($sequences, fn ($sequence) => isset($this->targets[$sequence])) + )); + } + } + + return $this->paginate($this->directTargets, $queries); + } + + /** + * @param array $queries + */ + private function limitOf(array $queries): int + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_LIMIT) { + return (int) ($query->getValues()[0] ?? 0); + } + } + + return 0; + } + + /** + * @param array $queries + */ + private function cursorOf(array $queries): ?Document + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) { + $value = $query->getValues()[0] ?? null; + + return $value instanceof Document ? $value : null; + } + } + + return null; + } +} + +class MessagingFanoutTest extends TestCase +{ + private const RECIPIENT_COUNT = 4500; + + private function provider(): Document + { + return new Document([ + '$id' => 'provider1', + '$sequence' => '1', + 'enabled' => true, + 'type' => MESSAGE_TYPE_EMAIL, + 'provider' => 'mock', + 'name' => 'Mock Email', + 'credentials' => [], + 'options' => [ + 'fromName' => 'Appwrite', + 'fromEmail' => 'noreply@example.com', + ], + ]); + } + + private function message(): Document + { + return new Document([ + '$id' => 'message1', + '$sequence' => '1', + 'providerType' => MESSAGE_TYPE_EMAIL, + 'topics' => ['topic1'], + 'targets' => [], + 'users' => [], + 'status' => MessageStatus::PROCESSING, + 'search' => 'message1', + 'data' => [ + 'subject' => 'Hello', + 'content' => 'World', + 'html' => false, + ], + ]); + } + + /** + * Build a topic-backed dataset: RECIPIENT_COUNT subscribers, each pointing at one email target. + * + * @return array{subscribers: array, targets: array} + */ + private function topicDataset(int $count): array + { + $subscribers = []; + $targets = []; + + for ($i = 1; $i <= $count; $i++) { + $sequence = (string) $i; + $subscribers[] = new Document([ + '$id' => 'sub' . $i, + '$sequence' => $sequence, + 'topicInternalId' => '100', + 'targetInternalId' => $sequence, + 'providerType' => MESSAGE_TYPE_EMAIL, + ]); + $targets[$sequence] = new Document([ + '$id' => 'target' . $i, + '$sequence' => $sequence, + 'providerId' => null, + 'identifier' => "user{$i}@example.com", + 'providerType' => MESSAGE_TYPE_EMAIL, + ]); + } + + return ['subscribers' => $subscribers, 'targets' => $targets]; + } + + private function sendTopic( + RecordingDatabase $database, + ConcurrencyProbeAdapter $adapter + ): void { + $worker = new TestableMessaging($adapter); + $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); + + $publisher = $this->createMock(UsagePublisher::class); + $publisher->method('enqueue')->willReturn(true); + + $device = new Local(\sys_get_temp_dir()); + $project = new Document(['$id' => 'project1', '$sequence' => '1']); + + run(function () use ($worker, $method, $database, $device, $project, $publisher) { + $method->invoke($worker, $database, $this->message(), $device, $project, $publisher); + }); + } + + public function testRecipientsArePaginatedWithoutSubqueryFilter(): void + { + $data = $this->topicDataset(self::RECIPIENT_COUNT); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + $adapter = new ConcurrencyProbeAdapter(); + + $this->sendTopic($database, $adapter); + + $topicReads = \array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics'); + $subscriberReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'subscribers')); + + // The topic's `targets` attribute (the subQueryTopicTargets filter) is never materialized: topics are + // only read with a $sequence projection, and recipients come from the subscribers collection instead. + $this->assertNotEmpty($topicReads); + foreach ($topicReads as $call) { + $selected = $this->selectFields($call['queries']); + $this->assertNotContains('targets', $selected, 'Topic read must not select the targets subquery attribute'); + } + + // Subscribers are walked in bounded pages via cursorAfter. 4500 recipients over a 1000 page size is + // five pages; every page after the first carries a cursor and none exceeds the page size. + $expectedPages = (int) \ceil(self::RECIPIENT_COUNT / MESSAGE_RECIPIENTS_PAGE_SIZE); + $this->assertCount($expectedPages, $subscriberReads); + + foreach ($subscriberReads as $index => $call) { + $this->assertSame(MESSAGE_RECIPIENTS_PAGE_SIZE, $this->limitOf($call['queries'])); + + if ($index === 0) { + $this->assertFalse($this->hasCursor($call['queries']), 'First page must not carry a cursor'); + } else { + $this->assertTrue($this->hasCursor($call['queries']), 'Subsequent pages must paginate with cursorAfter'); + } + } + } + + public function testConcurrentSendsNeverExceedConfiguredLimit(): void + { + // One recipient per batch (maxPerRequest = 1) maximises the number of concurrent in-flight sends so + // the Semaphore's bound is actually exercised: 1500 batches contend for MESSAGE_SEND_CONCURRENCY permits. + $count = 1500; + $data = $this->topicDataset($count); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + $adapter = new ConcurrencyProbeAdapter(maxPerRequest: 1); + + $this->sendTopic($database, $adapter); + + $this->assertSame($count, $adapter->sendCalls, 'Every recipient is sent exactly once'); + $this->assertGreaterThan(1, $adapter->peakConcurrency, 'Sends should actually run concurrently'); + $this->assertLessThanOrEqual( + MESSAGE_SEND_CONCURRENCY, + $adapter->peakConcurrency, + 'Concurrent sends must never exceed MESSAGE_SEND_CONCURRENCY' + ); + } + + public function testDeliveryErrorsAreCappedWhileFailedCountIsComplete(): void + { + $count = self::RECIPIENT_COUNT; + $data = $this->topicDataset($count); + // Every recipient fails: 4500 individual failures, but only MESSAGE_DELIVERY_ERRORS_LIMIT are retained. + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + $adapter = new ConcurrencyProbeAdapter(failureFor: fn (string $recipient) => true); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + + $errors = $message->getAttribute('deliveryErrors'); + $this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $errors, 'Delivery errors must be capped'); + + // Failed recipients are fully accounted for even though only 100 error strings are kept: every one of + // the 4500 recipients failed, so deliveredTotal is zero and the message is marked FAILED. + $this->assertSame(0, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::FAILED, $message->getAttribute('status')); + } + + public function testDeliveredTotalAndStatusReflectPartialSuccess(): void + { + $count = self::RECIPIENT_COUNT; + $data = $this->topicDataset($count); + // Fail exactly the recipients whose local-part index is divisible by 10. + $failing = static function (string $recipient): bool { + \preg_match('/user(\d+)@/', $recipient, $matches); + + return isset($matches[1]) && ((int) $matches[1]) % 10 === 0; + }; + $expectedFailures = \intdiv($count, 10); + + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + $adapter = new ConcurrencyProbeAdapter(failureFor: $failing); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + + // deliveredTotal is exact across all pages; failures partition the remainder exactly. + $this->assertSame($count - $expectedFailures, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::FAILED, $message->getAttribute('status')); + $this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $message->getAttribute('deliveryErrors')); + } + + public function testFullSuccessMarksMessageSent(): void + { + $data = $this->topicDataset(self::RECIPIENT_COUNT); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + $adapter = new ConcurrencyProbeAdapter(); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + $this->assertSame(self::RECIPIENT_COUNT, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::SENT, $message->getAttribute('status')); + $this->assertEmpty($message->getAttribute('deliveryErrors')); + } + + public function testTerminalStatusIsSkipped(): void + { + $data = $this->topicDataset(10); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + $adapter = new ConcurrencyProbeAdapter(); + + $worker = new TestableMessaging($adapter); + $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); + $publisher = $this->createMock(UsagePublisher::class); + $device = new Local(\sys_get_temp_dir()); + $project = new Document(['$id' => 'project1', '$sequence' => '1']); + + $message = $this->message(); + $message->setAttribute('status', MessageStatus::SENT); + + run(function () use ($worker, $method, $database, $device, $project, $publisher, $message) { + $method->invoke($worker, $database, $message, $device, $project, $publisher); + }); + + $this->assertSame(0, $adapter->sendCalls, 'Already-sent messages must not be reprocessed'); + $this->assertNull($database->updatedMessage, 'Terminal messages must not be rewritten'); + } + + /** + * @param array $queries + * @return array + */ + private function selectFields(array $queries): array + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_SELECT) { + return $query->getValues(); + } + } + + return []; + } + + /** + * @param array $queries + */ + private function limitOf(array $queries): int + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_LIMIT) { + return (int) ($query->getValues()[0] ?? 0); + } + } + + return 0; + } + + /** + * @param array $queries + */ + private function hasCursor(array $queries): bool + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) { + return true; + } + } + + return false; + } +} diff --git a/tests/unit/Messaging/SwooleCoroutinePolyfill.php b/tests/unit/Messaging/SwooleCoroutinePolyfill.php new file mode 100644 index 0000000000..b509d84bec --- /dev/null +++ b/tests/unit/Messaging/SwooleCoroutinePolyfill.php @@ -0,0 +1,160 @@ + + */ + private array $buffer = []; + + public function __construct(private readonly int $capacity = 1) + { + } + + public function push(mixed $data, float $timeout = 0): bool + { + $this->buffer[] = $data; + + return true; + } + + public function pop(float $timeout = 0): mixed + { + return \array_shift($this->buffer); + } + + public function isFull(): bool + { + return \count($this->buffer) >= $this->capacity; + } + + public function isEmpty(): bool + { + return \count($this->buffer) === 0; + } + } + } +} + +namespace Swoole\Coroutine { + use Swoole\Coroutine; + use Utopia\Lock\Exception\Contention; + + if (! \function_exists('Swoole\Coroutine\run')) { + function run(callable $callback): void + { + $fiber = new \Fiber($callback); + $previous = Coroutine::$current; + Coroutine::$current = $fiber; + $fiber->start(); + + while (! $fiber->isTerminated()) { + $fiber->resume(); + } + + Coroutine::$current = $previous; + } + } + + if (! \function_exists('Swoole\Coroutine\batch')) { + /** + * @param array $tasks + * @return array + */ + function batch(array $tasks, float $timeout = -1): array + { + $results = []; + $done = []; + $fibers = []; + + foreach ($tasks as $key => $task) { + $fibers[$key] = new \Fiber($task); + } + + $previous = Coroutine::$current; + + while (\count($done) < \count($tasks)) { + $progressed = false; + + foreach ($fibers as $key => $fiber) { + if (isset($done[$key])) { + continue; + } + + Coroutine::$current = $fiber; + + try { + if (! $fiber->isStarted()) { + $fiber->start(); + } elseif (! $fiber->isTerminated()) { + $fiber->resume(); + } + + // Any successful start/resume advances a fiber (toward a yield or termination), which + // counts as progress; fibers advance monotonically so this cannot spin forever. + $progressed = true; + + if ($fiber->isTerminated()) { + $results[$key] = $fiber->getReturn(); + $done[$key] = true; + } + } catch (Contention) { + // All permits held: model a blocking acquire by retrying the task on a later tick. + $fibers[$key] = new \Fiber($tasks[$key]); + $progressed = true; + } + } + + if (! $progressed) { + break; + } + } + + Coroutine::$current = $previous; + \ksort($results); + + return $results; + } + } +} From 2cd1abc2732c4b3d8246426abbd4333ee079e406 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Jun 2026 03:57:39 +1200 Subject: [PATCH 2/9] feat(messaging): retry throttled recipients with backoff in send path Provider rate limiting and transient errors previously failed a whole batch outright. Wrap the adapter send in a backoff/retry loop that partitions per-recipient results: delivered recipients are summed and never retried, retryable failures (throttle/rate-limit/quota/503/timeout phrasing) are re-sent with only the still-failing identifiers in `to`, and permanent failures stay terminal. Exponential backoff with jitter runs on the non-blocking Swoole coroutine sleep so sibling sends keep progressing; Retry-After is intentionally not honored to keep the messaging library adapter-agnostic. Accounting stays exact: recipients is the original batch size and delivered is summed across attempts without double-counting. --- app/init/constants.php | 2 + src/Appwrite/Platform/Workers/Messaging.php | 274 +++++++++++++++---- tests/unit/Messaging/MessagingFanoutTest.php | 191 ++++++++++++- 3 files changed, 409 insertions(+), 58 deletions(-) diff --git a/app/init/constants.php b/app/init/constants.php index 2aa0d347f1..2b72f8814f 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -246,6 +246,8 @@ const MESSAGE_SEND_TYPE_EXTERNAL = 'external'; const MESSAGE_RECIPIENTS_PAGE_SIZE = 1000; // Recipients resolved from the database per page while streaming a send const MESSAGE_SEND_CONCURRENCY = 10; // Maximum adapter send requests running concurrently within a single send job const MESSAGE_DELIVERY_ERRORS_LIMIT = 100; // Maximum number of per-recipient delivery errors retained on a message +const MESSAGE_SEND_MAX_RETRIES = 5; // Maximum number of attempts to deliver a batch when the provider throttles or fails transiently +const MESSAGE_SEND_RETRY_DELAY = 1.0; // Base seconds for exponential backoff between batch send retries; overridable in tests for an instant suite // Mail Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index a80110681b..b239a69c75 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -497,6 +497,11 @@ class Messaging extends Action /** * Send a single adapter-sized batch and report delivery counts plus a bounded error list. * + * Wraps the provider call in a backoff/retry loop that reacts to provider rate limiting and transient + * failures (see {@see retrySend()}). Accounting stays exact: only still-failing recipients are ever + * retried, so `delivered` is summed across attempts without double-counting, and `recipients` always + * reports the original batch size so the caller's `failed = recipients - delivered` holds. + * * @param array $batch * @return array{delivered: int, recipients: int, errors: array} */ @@ -512,66 +517,29 @@ class Messaging extends Action UsagePublisher $publisherForUsage ): array { $recipients = \count($batch); - $delivered = 0; - $errors = []; - $messageData = clone $message; - $messageData->setAttribute('to', $batch); + [ + 'delivered' => $delivered, + 'errors' => $errors, + ] = $this->retrySend($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project); - $data = match ($providerType) { - MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider), - MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData), - MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project), - default => throw new \Exception('Provider with the requested ID is of the incorrect type') - }; + $failed = $recipients - $delivered; + $usage = new UsageContext(); + $usage + ->addMetric(METRIC_MESSAGES, $recipients) + ->addMetric(METRIC_MESSAGES_SENT, $delivered) + ->addMetric(METRIC_MESSAGES_FAILED, $failed) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed); - try { - $response = $adapter->send($data); - $delivered = (int) $response['deliveredTo']; - - foreach ($response['results'] as $result) { - if ($result['status'] === 'failure' && \count($errors) < MESSAGE_DELIVERY_ERRORS_LIMIT) { - $errors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}"; - } - - // Deleting push targets when token has expired. - if (($result['error'] ?? '') === 'Expired device token') { - $target = $dbForProject->findOne('targets', [ - Query::equal('identifier', [$result['recipient']]) - ]); - - if (!$target->isEmpty()) { - $dbForProject->updateDocument( - 'targets', - $target->getId(), - $target->setAttribute('expired', true) - ); - } - } - } - } catch (\Throwable $e) { - // Whole-batch failure: record one representative error and count every recipient as failed. - $delivered = 0; - $errors = ['Failed sending to targets with error: ' . $e->getMessage()]; - } finally { - $failed = $recipients - $delivered; - $usage = new UsageContext(); - $usage - ->addMetric(METRIC_MESSAGES, $recipients) - ->addMetric(METRIC_MESSAGES_SENT, $delivered) - ->addMetric(METRIC_MESSAGES_FAILED, $failed) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed); - - $publisherForUsage->enqueue(new Usage( - project: $project, - metrics: $usage->getMetrics(), - )); - } + $publisherForUsage->enqueue(new Usage( + project: $project, + metrics: $usage->getMetrics(), + )); return [ 'delivered' => $delivered, @@ -580,6 +548,198 @@ class Messaging extends Action ]; } + /** + * Drive a batch through provider sends with exponential backoff, reacting to throttling and transient + * errors by retrying only the recipients that are still failing. + * + * Partitioning per attempt: the provider's per-recipient results are split into delivered (summed into + * the running total, never retried) and failures. A failure whose error matches {@see isRetryableError()} + * stays pending for the next attempt; any other failure becomes a terminal error immediately. When + * `send()` throws, the throw is retryable only if its message matches; otherwise every still-pending + * recipient is recorded as terminal. The next attempt rebuilds the provider message with only the pending + * recipients in `to`, so an already-delivered recipient is never re-sent. After {@see MESSAGE_SEND_MAX_RETRIES} + * attempts, any recipients still pending are flushed to terminal errors. + * + * Note: provider `Retry-After` hints are not honored — the agnostic adapter {@see \Utopia\Messaging\Response} + * exposes only generic per-recipient strings, never a structured retry delay, so we keep the library + * adapter-agnostic and rely on exponential backoff alone. + * + * @param array $batch + * @return array{delivered: int, errors: array} + */ + private function retrySend( + array $batch, + Document $message, + Document $provider, + string $providerType, + EmailAdapter|SMSAdapter|PushAdapter $adapter, + Database $dbForProject, + Device $deviceForFiles, + Document $project + ): array { + $delivered = 0; + $errors = []; + + // Recipients still awaiting a successful (or terminal) outcome; shrinks to only the failing ones each attempt. + $pending = $batch; + + for ($attempt = 1; $attempt <= MESSAGE_SEND_MAX_RETRIES; $attempt++) { + $hasRetriesLeft = $attempt < MESSAGE_SEND_MAX_RETRIES; + + // Rebuild the provider message scoped to only the still-pending recipients so a partially-delivered + // batch never re-sends to recipients that already succeeded on an earlier attempt. + $data = $this->buildMessage($pending, $message, $provider, $providerType, $dbForProject, $deviceForFiles, $project); + + $retry = []; + + try { + $response = $adapter->send($data); + $delivered += (int) $response['deliveredTo']; + + foreach ($response['results'] as $result) { + if ($result['status'] !== 'failure') { + continue; + } + + $error = $result['error'] ?? null; + $recipient = $result['recipient']; + + // Deleting push targets when token has expired. + if ($error === 'Expired device token') { + $target = $dbForProject->findOne('targets', [ + Query::equal('identifier', [$recipient]) + ]); + + if (!$target->isEmpty()) { + $dbForProject->updateDocument( + 'targets', + $target->getId(), + $target->setAttribute('expired', true) + ); + } + } + + if ($hasRetriesLeft && $this->isRetryableError($error)) { + $retry[] = $recipient; + continue; + } + + $this->recordError($errors, "Failed sending to target {$recipient} with error: {$error}"); + } + } catch (\Throwable $e) { + // Whole-batch failure: retry every still-pending recipient when the throw looks transient, + // otherwise record one representative terminal error covering them all. + if ($hasRetriesLeft && $this->isRetryableError($e->getMessage())) { + $retry = $pending; + } else { + $this->recordError($errors, 'Failed sending to targets with error: ' . $e->getMessage()); + } + } + + $pending = $retry; + + if (empty($pending)) { + break; + } + + // Exponential backoff with jitter; non-blocking under Swoole so sibling sends keep progressing. + $delay = $this->retryDelay() * (2 ** ($attempt - 1)); + $delay += $delay * (\random_int(0, 100) / 1000); + \Swoole\Coroutine::sleep($delay); + } + + return [ + 'delivered' => $delivered, + 'errors' => $errors, + ]; + } + + /** + * Append a delivery error while keeping the retained list bounded by {@see MESSAGE_DELIVERY_ERRORS_LIMIT}. + * + * @param array $errors + */ + private function recordError(array &$errors, string $error): void + { + if (\count($errors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) { + return; + } + + $errors[] = $error; + } + + /** + * Conservatively classify a provider error string as retryable. The agnostic adapter + * {@see \Utopia\Messaging\Response} returns only free-form error strings, so string matching is the only + * provider-agnostic signal available. The pattern list is intentionally narrow — throttling, rate limits, + * quota, service-unavailable and timeout phrasing — so permanent failures (e.g. invalid recipients) are + * never retried. + */ + private function isRetryableError(?string $error): bool + { + if ($error === null || $error === '') { + return false; + } + + $patterns = [ + 'throttl', + 'rate exceeded', + 'rate limit', + 'too many requests', + '429', + 'quota', + 'service unavailable', + '503', + 'timed out', + 'timeout', + 'temporarily', + ]; + + $needle = \strtolower($error); + + foreach ($patterns as $pattern) { + if (\str_contains($needle, $pattern)) { + return true; + } + } + + return false; + } + + /** + * Base seconds for exponential backoff between send retries. Isolated so tests can override it to keep the + * suite instant without touching the production constant. + */ + protected function retryDelay(): float + { + return MESSAGE_SEND_RETRY_DELAY; + } + + /** + * Build the provider-specific message for a set of recipients. + * + * @param array $to + */ + private function buildMessage( + array $to, + Document $message, + Document $provider, + string $providerType, + Database $dbForProject, + Device $deviceForFiles, + Document $project + ): Email|SMS|Push { + $messageData = clone $message; + $messageData->setAttribute('to', $to); + + return match ($providerType) { + MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider), + MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData), + MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project), + default => throw new \Exception('Provider with the requested ID is of the incorrect type') + }; + } + private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void { if ($this->adapter === null) { diff --git a/tests/unit/Messaging/MessagingFanoutTest.php b/tests/unit/Messaging/MessagingFanoutTest.php index 024cdbf43e..f2779b9a85 100644 --- a/tests/unit/Messaging/MessagingFanoutTest.php +++ b/tests/unit/Messaging/MessagingFanoutTest.php @@ -91,6 +91,7 @@ class ConcurrencyProbeAdapter extends EmailAdapter /** * Worker subclass that injects the instrumented email adapter instead of a network-backed provider adapter. + * Retry backoff is zeroed so the throttle/retry tests stay instant. */ class TestableMessaging extends Messaging { @@ -102,6 +103,80 @@ class TestableMessaging extends Messaging { return $this->emailAdapter; } + + protected function retryDelay(): float + { + return 0.0; + } +} + +/** + * Email adapter double whose per-attempt outcome is scripted. Each {@see process()} call records the exact + * `to` recipients it received and consults the next scripted step, which either throws (simulating a + * connection-level provider error) or returns a per-recipient success/failure verdict. When the script runs + * out, every recipient succeeds — modelling a provider that has stopped throttling. + */ +class ScriptedEmailAdapter extends EmailAdapter +{ + public int $sendCalls = 0; + + /** + * @var array> Recipient email lists captured per {@see process()} call, in call order. + */ + public array $recipientsPerCall = []; + + /** + * @param array<\Throwable|(callable(string): ?string)> $script One step per attempt. A Throwable is thrown + * for that attempt; a callable returns an error string for a failing recipient or null for success. + */ + public function __construct(private readonly array $script = []) + { + } + + public function getName(): string + { + return 'ScriptedEmail'; + } + + public function getMaxMessagesPerRequest(): int + { + return 1000; + } + + /** + * @param EmailMessage $message + * @return array{deliveredTo: int, type: string, results: array>} + */ + protected function process(EmailMessage $message): array + { + $recipients = \array_map(static fn (array $entry): string => $entry['email'], $message->getTo()); + $this->recipientsPerCall[] = $recipients; + + $step = $this->script[$this->sendCalls] ?? null; + $this->sendCalls++; + + if ($step instanceof \Throwable) { + throw $step; + } + + $response = new Response($this->getType()); + $delivered = 0; + + foreach ($recipients as $recipient) { + $error = \is_callable($step) ? $step($recipient) : null; + + if ($error === null) { + $delivered++; + $response->addResult($recipient); + } else { + $response->addResult($recipient, $error); + } + } + + $response->setDeliveredTo($delivered); + + return $response->toArray(); + } } /** @@ -326,7 +401,7 @@ class MessagingFanoutTest extends TestCase private function sendTopic( RecordingDatabase $database, - ConcurrencyProbeAdapter $adapter + EmailAdapter $adapter ): void { $worker = new TestableMessaging($adapter); $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); @@ -483,6 +558,120 @@ class MessagingFanoutTest extends TestCase $this->assertNull($database->updatedMessage, 'Terminal messages must not be rewritten'); } + public function testThrottledBatchIsRetriedThenFullyDelivered(): void + { + $count = 5; + $data = $this->topicDataset($count); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + + // First attempt throttles every recipient; the script then runs out so the second attempt succeeds. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): string => 'Rate limit exceeded, please retry', + ]); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + + // The batch was retried exactly once and then fully delivered with no double-counting. + $this->assertSame(2, $adapter->sendCalls, 'Throttled batch must be retried once'); + $this->assertSame($count, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::SENT, $message->getAttribute('status')); + $this->assertEmpty($message->getAttribute('deliveryErrors')); + + // Both attempts targeted the full recipient set (none had succeeded on the first attempt). + $this->assertCount($count, $adapter->recipientsPerCall[0]); + $this->assertCount($count, $adapter->recipientsPerCall[1]); + } + + public function testRetryTargetsOnlyThrottledRecipients(): void + { + $count = 5; + $data = $this->topicDataset($count); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + + // user2@ and user4@ throttle on the first attempt; everyone else succeeds. The second attempt has no + // script step, so the two retried recipients then succeed. + $throttled = ['user2@example.com', 'user4@example.com']; + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => \in_array($recipient, $throttled, true) + ? 'Too Many Requests' + : null, + ]); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + + $this->assertSame(2, $adapter->sendCalls, 'A partially throttled batch must be retried once'); + + // The retry must re-send to ONLY the throttled recipients — never the ones already delivered. + \sort($adapter->recipientsPerCall[1]); + $this->assertSame($throttled, $adapter->recipientsPerCall[1]); + + // Every recipient is ultimately delivered and counted exactly once. + $this->assertSame($count, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::SENT, $message->getAttribute('status')); + $this->assertEmpty($message->getAttribute('deliveryErrors')); + } + + public function testNonRetryableFailureIsNotRetried(): void + { + $count = 3; + $data = $this->topicDataset($count); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + + // user1@ permanently fails with a non-retryable error; the rest succeed. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => $recipient === 'user1@example.com' + ? 'Invalid recipient address' + : null, + ]); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + + // A permanent failure is recorded immediately, never retried. + $this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry'); + $this->assertSame($count - 1, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::FAILED, $message->getAttribute('status')); + $errors = $message->getAttribute('deliveryErrors'); + $this->assertCount(1, $errors); + $this->assertStringContainsString('user1@example.com', $errors[0]); + $this->assertStringContainsString('Invalid recipient address', $errors[0]); + } + + public function testExhaustedRetriesRecordTerminalFailure(): void + { + $count = 2; + $data = $this->topicDataset($count); + $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + + // Every attempt throttles, for more steps than MESSAGE_SEND_MAX_RETRIES, so retries are exhausted. + $script = \array_fill( + 0, + MESSAGE_SEND_MAX_RETRIES + 2, + static fn (string $recipient): string => 'Service unavailable, throttled' + ); + $adapter = new ScriptedEmailAdapter($script); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + + // The batch is attempted exactly MESSAGE_SEND_MAX_RETRIES times, then the still-throttled recipients + // become terminal failures with nothing delivered. + $this->assertSame(MESSAGE_SEND_MAX_RETRIES, $adapter->sendCalls, 'Attempts must be capped at MESSAGE_SEND_MAX_RETRIES'); + $this->assertSame(0, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::FAILED, $message->getAttribute('status')); + $this->assertCount($count, $message->getAttribute('deliveryErrors')); + } + /** * @param array $queries * @return array From 7c8d7dcb80f36f271e996e93f9a8a1498c824f15 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Jun 2026 04:13:49 +1200 Subject: [PATCH 3/9] fix(messaging): address fanout scale review comments Apply five reviewer fixes to the large-topic send path without touching the adapter-agnostic messaging library: - Empty recipient pages no longer count as recipients: a page that resolves to zero identifiers is skipped instead of running batch([]) and masking the "No valid recipients found." outcome. - Best-effort expired-device-token cleanup is isolated from the provider send: the send try/catch wraps only adapter->send(), results are processed outside it, and the findOne/updateDocument cleanup has its own try/catch so a transient DB error can never be misattributed as a send failure or trigger a spurious retry. Delivery accounting stays exact across attempts. - resolveProvider caches the fallback provider under the requested id so a topic full of targets pointing at one disabled/missing provider does not re-query the providers collection once per page. - The test Swoole channel polyfill enforces capacity (push returns false when full) so a saturated semaphore surfaces Contention and the scheduler defers the task, making the concurrency cap genuinely bounded under the polyfill. - ReflectionMethod::setAccessible(true) added for explicitness/back-compat. Adds a regression test proving a thrown expired-token cleanup is swallowed and the real provider failure is recorded instead. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/Appwrite/Platform/Workers/Messaging.php | 63 +++++++++++----- tests/unit/Messaging/MessagingFanoutTest.php | 73 +++++++++++++++++++ .../Messaging/SwooleCoroutinePolyfill.php | 8 ++ 3 files changed, 124 insertions(+), 20 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index b239a69c75..0154716021 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -177,8 +177,6 @@ class Messaging extends Action $hasRecipients = false; foreach ($this->streamRecipients($dbForProject, $topicIds, $userIds, $targetIds, $providerType, $default) as $page) { - $hasRecipients = true; - /** * @var array $tasks */ @@ -217,6 +215,14 @@ class Messaging extends Action } } + // A page can resolve to zero identifiers (e.g. subscribers whose targetInternalId matches no targets + // row), so an empty task list must not count as recipients or run batch() for nothing. + if (empty($tasks)) { + continue; + } + + $hasRecipients = true; + /** * @var array}> $results */ @@ -486,6 +492,10 @@ class Messaging extends Action $provider = $dbForProject->getDocument('providers', $providerId); if ($provider->isEmpty() || !$provider->getAttribute('enabled')) { + // Cache the fallback under this id too, so a topic full of targets pointing at the same + // disabled/missing provider does not re-query the providers collection once per page. + $providers[$providerId] = $default; + return $default; } @@ -592,8 +602,24 @@ class Messaging extends Action $retry = []; + // The try/catch wraps ONLY the provider send. A whole-batch throw is retryable when transient, + // otherwise it records one representative terminal error. The previous behaviour of resetting + // $delivered to 0 on a throw is gone — the retry refactor sums delivered across attempts, and the + // expired-device-token cleanup below is isolated in its own try so a DB hiccup there can never be + // misattributed as a send failure. try { $response = $adapter->send($data); + } catch (\Throwable $e) { + if ($hasRetriesLeft && $this->isRetryableError($e->getMessage())) { + $retry = $pending; + } else { + $this->recordError($errors, 'Failed sending to targets with error: ' . $e->getMessage()); + } + + $response = null; + } + + if ($response !== null) { $delivered += (int) $response['deliveredTo']; foreach ($response['results'] as $result) { @@ -604,18 +630,23 @@ class Messaging extends Action $error = $result['error'] ?? null; $recipient = $result['recipient']; - // Deleting push targets when token has expired. + // Best-effort: deleting push targets when the token has expired. Isolated so a transient DB + // error here never affects delivery accounting or the retry decision below. if ($error === 'Expired device token') { - $target = $dbForProject->findOne('targets', [ - Query::equal('identifier', [$recipient]) - ]); + try { + $target = $dbForProject->findOne('targets', [ + Query::equal('identifier', [$recipient]) + ]); - if (!$target->isEmpty()) { - $dbForProject->updateDocument( - 'targets', - $target->getId(), - $target->setAttribute('expired', true) - ); + if (!$target->isEmpty()) { + $dbForProject->updateDocument( + 'targets', + $target->getId(), + $target->setAttribute('expired', true) + ); + } + } catch (\Throwable) { + // Best-effort; must not affect accounting or retries. } } @@ -626,14 +657,6 @@ class Messaging extends Action $this->recordError($errors, "Failed sending to target {$recipient} with error: {$error}"); } - } catch (\Throwable $e) { - // Whole-batch failure: retry every still-pending recipient when the throw looks transient, - // otherwise record one representative terminal error covering them all. - if ($hasRetriesLeft && $this->isRetryableError($e->getMessage())) { - $retry = $pending; - } else { - $this->recordError($errors, 'Failed sending to targets with error: ' . $e->getMessage()); - } } $pending = $retry; diff --git a/tests/unit/Messaging/MessagingFanoutTest.php b/tests/unit/Messaging/MessagingFanoutTest.php index f2779b9a85..b7529433c5 100644 --- a/tests/unit/Messaging/MessagingFanoutTest.php +++ b/tests/unit/Messaging/MessagingFanoutTest.php @@ -328,6 +328,37 @@ class RecordingDatabase extends Database } } +/** + * {@see RecordingDatabase} variant whose expired-device-token cleanup always fails: `findOne('targets')` + * resolves to a real target so the worker attempts the update, and `updateDocument('targets')` throws a + * transient error. Used to prove that a DB hiccup during best-effort cleanup never leaks into delivery + * accounting or the retry decision. + */ +class ExpiredTokenCleanupFailingDatabase extends RecordingDatabase +{ + public int $targetUpdateAttempts = 0; + + public function findOne(string $collection, array $queries = []): Document + { + if ($collection === 'targets') { + return new Document(['$id' => 'target-expired', '$sequence' => '1']); + } + + return parent::findOne($collection, $queries); + } + + public function updateDocument(string $collection, string $id, Document $document): Document + { + if ($collection === 'targets') { + $this->targetUpdateAttempts++; + + throw new \RuntimeException('Transient database error during target cleanup'); + } + + return parent::updateDocument($collection, $id, $document); + } +} + class MessagingFanoutTest extends TestCase { private const RECIPIENT_COUNT = 4500; @@ -405,6 +436,7 @@ class MessagingFanoutTest extends TestCase ): void { $worker = new TestableMessaging($adapter); $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); + $method->setAccessible(true); $publisher = $this->createMock(UsagePublisher::class); $publisher->method('enqueue')->willReturn(true); @@ -543,6 +575,7 @@ class MessagingFanoutTest extends TestCase $worker = new TestableMessaging($adapter); $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); + $method->setAccessible(true); $publisher = $this->createMock(UsagePublisher::class); $device = new Local(\sys_get_temp_dir()); $project = new Document(['$id' => 'project1', '$sequence' => '1']); @@ -672,6 +705,46 @@ class MessagingFanoutTest extends TestCase $this->assertCount($count, $message->getAttribute('deliveryErrors')); } + public function testExpiredTokenCleanupFailureDoesNotCorruptAccounting(): void + { + $count = 3; + $data = $this->topicDataset($count); + // The cleanup path always throws, but the worker must isolate it: the send result is what's accounted. + $database = new ExpiredTokenCleanupFailingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); + + // user1@ comes back with an expired device token (triggering cleanup, which then throws); the other + // two recipients are delivered. The expired-token error is terminal, never retryable. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => $recipient === 'user1@example.com' + ? 'Expired device token' + : null, + ]); + + $this->sendTopic($database, $adapter); + + $message = $database->updatedMessage; + $this->assertNotNull($message); + + // The worker reached the cleanup (so the isolation is genuinely exercised) and it threw. + $this->assertGreaterThan(0, $database->targetUpdateAttempts, 'Expired-token cleanup must have been attempted'); + + // The send was attempted exactly once: a thrown cleanup must not look like a transient send failure + // and must not trigger a retry. + $this->assertSame(1, $adapter->sendCalls, 'Cleanup failure must not trigger a send retry'); + + // Accounting reflects the send result, not the cleanup throw: two delivered, one terminal failure. + $this->assertSame($count - 1, $message->getAttribute('deliveredTotal')); + $this->assertSame(MessageStatus::FAILED, $message->getAttribute('status')); + + // The recorded error is the provider's expired-token failure for the real recipient — proving the + // database exception was swallowed rather than misattributed as the send error. + $errors = $message->getAttribute('deliveryErrors'); + $this->assertCount(1, $errors); + $this->assertStringContainsString('user1@example.com', $errors[0]); + $this->assertStringContainsString('Expired device token', $errors[0]); + $this->assertStringNotContainsString('Transient database error', $errors[0]); + } + /** * @param array $queries * @return array diff --git a/tests/unit/Messaging/SwooleCoroutinePolyfill.php b/tests/unit/Messaging/SwooleCoroutinePolyfill.php index b509d84bec..fcf520a911 100644 --- a/tests/unit/Messaging/SwooleCoroutinePolyfill.php +++ b/tests/unit/Messaging/SwooleCoroutinePolyfill.php @@ -54,6 +54,14 @@ namespace Swoole\Coroutine { public function push(mixed $data, float $timeout = 0): bool { + // Enforce capacity so a full channel reports back-pressure exactly like a real Swoole channel. + // A full Semaphore permit channel then fails to acquire, which surfaces as a + // {@see \Utopia\Lock\Exception\Contention} that the batch() scheduler defers to a later tick — + // making the concurrency bound genuinely enforced rather than silently unbounded. + if (\count($this->buffer) >= $this->capacity) { + return false; + } + $this->buffer[] = $data; return true; From 7247b0effd00457a4dcd38b7424516c7fa57a9cf Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Jun 2026 04:22:36 +1200 Subject: [PATCH 4/9] fix(messaging): skip non-positive retry backoff sleep Swoole\Coroutine::sleep() rejects 0/negative values (CI runs unit tests with the real Swoole extension, where the Fiber test polyfill is inert). The retry tests override the base delay to 0 for speed, which made sleep(0.0) abort the process under real Swoole. Guard the call; production delays are always >= 1s. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/Appwrite/Platform/Workers/Messaging.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 0154716021..911db304dd 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -666,9 +666,13 @@ class Messaging extends Action } // Exponential backoff with jitter; non-blocking under Swoole so sibling sends keep progressing. + // Skip a non-positive delay: Swoole\Coroutine::sleep() rejects 0/negative values (which tests + // produce by overriding the base delay to 0 for speed), and it would otherwise emit a warning. $delay = $this->retryDelay() * (2 ** ($attempt - 1)); $delay += $delay * (\random_int(0, 100) / 1000); - \Swoole\Coroutine::sleep($delay); + if ($delay > 0) { + \Swoole\Coroutine::sleep($delay); + } } return [ From e2278a73701dda4f2d161bb2c50436cd0a17ddfb Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Jun 2026 04:35:12 +1200 Subject: [PATCH 5/9] test(messaging): drop deprecated ReflectionMethod::setAccessible call The CI unit-test job runs inside the Appwrite container on PHP 8.5, where ReflectionMethod::setAccessible() is deprecated, and the suite fails on triggered deprecations. It is also a no-op on PHP 8.1+ (reflection invokes private methods without it), so it was both unnecessary and the cause of the red Tests/Unit check. Reverts the setAccessible addition from 7c8d7dcb80. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/unit/Messaging/MessagingFanoutTest.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/Messaging/MessagingFanoutTest.php b/tests/unit/Messaging/MessagingFanoutTest.php index b7529433c5..ec6dbe87f4 100644 --- a/tests/unit/Messaging/MessagingFanoutTest.php +++ b/tests/unit/Messaging/MessagingFanoutTest.php @@ -435,8 +435,9 @@ class MessagingFanoutTest extends TestCase EmailAdapter $adapter ): void { $worker = new TestableMessaging($adapter); + // No setAccessible(true): reflection invokes private methods without it on PHP 8.1+, and the call is + // deprecated on PHP 8.5 (the CI unit-test runtime), where the suite fails on triggered deprecations. $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); - $method->setAccessible(true); $publisher = $this->createMock(UsagePublisher::class); $publisher->method('enqueue')->willReturn(true); @@ -574,8 +575,9 @@ class MessagingFanoutTest extends TestCase $adapter = new ConcurrencyProbeAdapter(); $worker = new TestableMessaging($adapter); + // No setAccessible(true): reflection invokes private methods without it on PHP 8.1+, and the call is + // deprecated on PHP 8.5 (the CI unit-test runtime), where the suite fails on triggered deprecations. $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); - $method->setAccessible(true); $publisher = $this->createMock(UsagePublisher::class); $device = new Local(\sys_get_temp_dir()); $project = new Document(['$id' => 'project1', '$sequence' => '1']); From c5551258fedf8c1989708da9331c088d66a0d99c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Jun 2026 04:45:32 +1200 Subject: [PATCH 6/9] test(messaging): run fan-out coroutine tests only without native Swoole Tests/Unit runs in the Appwrite container (PHP 8.5) with the native Swoole extension, where these tests' real Co\run()/batch() scheduler runs in-process and destabilises later coroutine-sensitive tests in the shared unit suite (the run aborted mid-suite at ~126/644 inside the Network/Fetcher tests). Skip them under native Swoole; they still run locally via the Fiber polyfill, and the real worker path is covered by the e2e Messaging suite. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/unit/Messaging/MessagingFanoutTest.php | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/Messaging/MessagingFanoutTest.php b/tests/unit/Messaging/MessagingFanoutTest.php index ec6dbe87f4..b42733c615 100644 --- a/tests/unit/Messaging/MessagingFanoutTest.php +++ b/tests/unit/Messaging/MessagingFanoutTest.php @@ -363,6 +363,18 @@ class MessagingFanoutTest extends TestCase { private const RECIPIENT_COUNT = 4500; + protected function setUp(): void + { + // These tests drive the worker through real Swoole coroutines (Co\run/batch). Under the native + // extension that scheduler runs in-process and destabilises later tests in the shared unit suite + // (the run aborts mid-suite inside unrelated coroutine-sensitive tests). They run locally against + // the Fiber polyfill in SwooleCoroutinePolyfill.php; the real worker path is exercised on CI by the + // e2e Messaging suite. See PR #12465. + if (\extension_loaded('swoole')) { + $this->markTestSkipped('Fan-out coroutine tests run only without the native Swoole extension; the worker path is covered on CI by the e2e Messaging suite.'); + } + } + private function provider(): Document { return new Document([ From 977cbd41da4db00722a18bdc4ca4585f693fce85 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Jun 2026 04:58:52 +1200 Subject: [PATCH 7/9] chore(messaging): exclude Swoole coroutine polyfill from PHPStan The test polyfill declares stand-in Swoole\Coroutine and Swoole\Coroutine\Channel classes behind a runtime extension check. PHPStan ignores the guard and parsed those minimal declarations, shadowing the real swoole/ide-helper stubs and reporting false 'undefined static method' errors codebase-wide (e.g. getContext, join, create, Channel::close in app/realtime.php). Exclude the polyfill from analysis so the ide-helper stubs are used. Verified: Checks/Analyze clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- phpstan.neon | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/phpstan.neon b/phpstan.neon index 0b8761c19e..e28de76d2d 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -12,3 +12,7 @@ parameters: - vendor/swoole/ide-helper excludePaths: - tests/resources + # Declares stand-in Swoole\Coroutine and Swoole\Coroutine\Channel classes guarded by a runtime + # extension check. PHPStan ignores that guard, so analysing this file would shadow the real Swoole + # ide-helper stubs and report false "undefined method" errors across the codebase (e.g. app/realtime.php). + - tests/unit/Messaging/SwooleCoroutinePolyfill.php From 58d7ae919470e531a8963f1b75644a1184d050a8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Jun 2026 12:42:32 +1200 Subject: [PATCH 8/9] test(messaging): rewrite fan-out tests as coroutine-free unit tests The Fiber-based SwooleCoroutinePolyfill shadowed real Swoole symbols and poisoned PHPStan, and driving the worker through real Swoole coroutines (Co\run/batch) inside the shared unit process contaminated and crashed later, unrelated tests. The native extension is now available, so delete the polyfill (and its PHPStan exclude) and test the worker's logic by invoking its component methods directly with zero coroutines. Reflection invokes the private methods without setAccessible (deprecated on PHP 8.5), and a TestableMessaging subclass zeroes retryDelay so the backoff never sleeps. Coverage: isRetryableError classification, retrySend (throttle-then-success, partial-throttle scoping, non-retryable terminal, exhausted retries, error-cap with exact delivered accounting, isolated expired-token cleanup failure) and streamRecipients (cursor pagination by topicInternalId, no targets-subquery read, no runaway limit, per-page dedup, user/direct-target resolution). The bounded- concurrency Semaphore + batch() integration stays covered by the e2e Messaging suite, where executing real coroutines is safe. Co-Authored-By: Claude Opus 4.8 (1M context) --- phpstan.neon | 4 - tests/unit/Messaging/MessagingFanoutTest.php | 1143 +++++++++-------- .../Messaging/SwooleCoroutinePolyfill.php | 168 --- 3 files changed, 574 insertions(+), 741 deletions(-) delete mode 100644 tests/unit/Messaging/SwooleCoroutinePolyfill.php diff --git a/phpstan.neon b/phpstan.neon index e28de76d2d..0b8761c19e 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -12,7 +12,3 @@ parameters: - vendor/swoole/ide-helper excludePaths: - tests/resources - # Declares stand-in Swoole\Coroutine and Swoole\Coroutine\Channel classes guarded by a runtime - # extension check. PHPStan ignores that guard, so analysing this file would shadow the real Swoole - # ide-helper stubs and report false "undefined method" errors across the codebase (e.g. app/realtime.php). - - tests/unit/Messaging/SwooleCoroutinePolyfill.php diff --git a/tests/unit/Messaging/MessagingFanoutTest.php b/tests/unit/Messaging/MessagingFanoutTest.php index b42733c615..97e888e228 100644 --- a/tests/unit/Messaging/MessagingFanoutTest.php +++ b/tests/unit/Messaging/MessagingFanoutTest.php @@ -2,8 +2,6 @@ namespace Tests\Unit\Messaging; -use Appwrite\Event\Publisher\Usage as UsagePublisher; -use Appwrite\Messaging\Status as MessageStatus; use Appwrite\Platform\Workers\Messaging; use PHPUnit\Framework\TestCase; use Utopia\Database\Database; @@ -15,95 +13,523 @@ use Utopia\Messaging\Messages\Email as EmailMessage; use Utopia\Messaging\Response; use Utopia\Storage\Device\Local; -use function Swoole\Coroutine\run; - -require_once __DIR__ . '/SwooleCoroutinePolyfill.php'; - /** - * Email adapter double that records peak in-flight {@see process()} calls and applies a configurable - * per-recipient failure predicate. It cooperatively yields inside the send so fibers scheduled by the - * polyfilled (or real) `batch()` interleave, making concurrent sends observable. + * Pure unit tests for the messaging worker's fan-out building blocks: the retryable-error classifier, the + * batch retry/backoff loop, and the cursor-paginated recipient stream. Each method is exercised directly + * through reflection with no Swoole coroutines. + * + * The coroutine fan-out integration itself — the bounded-concurrency {@see \Utopia\Lock\Semaphore} driving + * `Swoole\Coroutine\batch()` — is intentionally NOT covered here. Executing real Swoole coroutines inside the + * shared unit process is unsafe: the in-process scheduler destabilises later, unrelated tests in the suite. + * That integration is covered by the e2e Messaging suite instead. See PR #12465. + * + * Private methods are invoked via {@see \ReflectionMethod::invoke()} WITHOUT {@see \ReflectionMethod::setAccessible()}: + * reflection invokes private methods directly on PHP 8.1+, and `setAccessible(true)` is deprecated on PHP 8.5 + * (the CI unit-test runtime), where the suite fails on triggered deprecations. */ -class ConcurrencyProbeAdapter extends EmailAdapter +class MessagingFanoutTest extends TestCase { - public int $peakConcurrency = 0; - - public int $sendCalls = 0; - - private int $inFlight = 0; - - /** - * @param int $maxPerRequest - * @param (callable(string): bool)|null $failureFor Returns true when the given recipient should fail. - */ - public function __construct( - private readonly int $maxPerRequest = 1000, - private $failureFor = null - ) { + private function provider(): Document + { + return new Document([ + '$id' => 'provider1', + '$sequence' => '1', + 'enabled' => true, + 'type' => MESSAGE_TYPE_EMAIL, + 'provider' => 'mock', + 'name' => 'Mock Email', + 'credentials' => [], + 'options' => [ + 'fromName' => 'Appwrite', + 'fromEmail' => 'noreply@example.com', + ], + ]); } - public function getName(): string + private function message(): Document { - return 'ConcurrencyProbe'; - } - - public function getMaxMessagesPerRequest(): int - { - return $this->maxPerRequest; + return new Document([ + '$id' => 'message1', + '$sequence' => '1', + 'providerType' => MESSAGE_TYPE_EMAIL, + 'topics' => ['topic1'], + 'targets' => [], + 'users' => [], + 'data' => [ + 'subject' => 'Hello', + 'content' => 'World', + 'html' => false, + ], + ]); } /** - * @param EmailMessage $message - * @return array{deliveredTo: int, type: string, results: array>} + * Invoke the worker's private {@see Messaging::retrySend()} for an email batch. + * + * @param array $batch + * @return array{delivered: int, errors: array} */ - protected function process(EmailMessage $message): array + private function retrySend(array $batch, EmailAdapter $adapter, Database $database): array { - $this->sendCalls++; - $this->inFlight++; - $this->peakConcurrency = \max($this->peakConcurrency, $this->inFlight); + $worker = new TestableMessaging(); + $method = new \ReflectionMethod(Messaging::class, 'retrySend'); - // Yield so other coroutines run while this "request" is in flight. - \Swoole\Coroutine::sleep(0.001); + $device = new Local(\sys_get_temp_dir()); + $project = new Document(['$id' => 'project1', '$sequence' => '1']); - $response = new Response($this->getType()); - $delivered = 0; + /** @var array{delivered: int, errors: array} $result */ + $result = $method->invoke( + $worker, + $batch, + $this->message(), + $this->provider(), + MESSAGE_TYPE_EMAIL, + $adapter, + $database, + $device, + $project + ); - foreach ($message->getTo() as $recipient) { - // Email recipients are normalized to ['email' => ..., 'name' => ...] arrays by the message. - $email = $recipient['email']; - $fails = \is_callable($this->failureFor) && ($this->failureFor)($email); + return $result; + } - if ($fails) { - $response->addResult($email, 'Simulated delivery failure'); + public function testIsRetryableErrorMatchesTransientFailures(): void + { + $worker = new TestableMessaging(); + $method = new \ReflectionMethod(Messaging::class, 'isRetryableError'); + + $retryable = [ + 'Throttling', + 'rate exceeded', + 'Rate limit reached', + 'Too Many Requests', + '429 Too Many Requests', + 'quota exceeded', + 'Service Unavailable', + '503 Service Unavailable', + 'Connection timed out', + 'Request timeout', + 'Service temporarily unavailable', + ]; + + foreach ($retryable as $error) { + $this->assertTrue( + $method->invoke($worker, $error), + "Expected '{$error}' to be classified as retryable" + ); + } + + $permanent = [ + 'invalid recipient', + 'Mailbox does not exist', + 'Authentication failed', + 'Expired device token', + ]; + + foreach ($permanent as $error) { + $this->assertFalse( + $method->invoke($worker, $error), + "Expected '{$error}' to be classified as permanent" + ); + } + + $this->assertFalse($method->invoke($worker, null), 'A null error must never be retryable'); + $this->assertFalse($method->invoke($worker, ''), 'An empty error must never be retryable'); + } + + public function testThrottledBatchIsRetriedThenFullyDelivered(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com', 'user4@example.com', 'user5@example.com']; + + // First attempt throttles every recipient; the script then runs out so the second attempt succeeds. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): string => 'Rate limit exceeded, please retry', + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // Retried exactly once, then everyone delivered with no double-counting and no terminal errors. + $this->assertSame(2, $adapter->sendCalls, 'Throttled batch must be retried once'); + $this->assertSame(\count($batch), $result['delivered']); + $this->assertEmpty($result['errors']); + + // Both attempts targeted the full recipient set (none had succeeded on the first attempt). + $this->assertSame($batch, $adapter->recipientsPerCall[0]); + $this->assertSame($batch, $adapter->recipientsPerCall[1]); + } + + public function testRetryTargetsOnlyThrottledRecipients(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com', 'user4@example.com', 'user5@example.com']; + + // user2@ and user4@ throttle on the first attempt; everyone else succeeds. The second attempt has no + // script step, so the two retried recipients then succeed. + $throttled = ['user2@example.com', 'user4@example.com']; + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => \in_array($recipient, $throttled, true) + ? 'Too Many Requests' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + $this->assertSame(2, $adapter->sendCalls, 'A partially throttled batch must be retried once'); + + // The retry must re-send to EXACTLY the throttled recipients — never the ones already delivered. + $this->assertSame($throttled, $adapter->recipientsPerCall[1]); + + // Every recipient is ultimately delivered and counted exactly once. + $this->assertSame(\count($batch), $result['delivered']); + $this->assertEmpty($result['errors']); + } + + public function testNonRetryableFailureIsNotRetried(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com']; + + // user1@ permanently fails with a non-retryable error; the rest succeed. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => $recipient === 'user1@example.com' + ? 'Invalid recipient address' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // A permanent failure is recorded immediately, never retried. + $this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry'); + $this->assertSame(\count($batch) - 1, $result['delivered']); + $this->assertCount(1, $result['errors']); + $this->assertStringContainsString('user1@example.com', $result['errors'][0]); + $this->assertStringContainsString('Invalid recipient address', $result['errors'][0]); + } + + public function testExhaustedRetriesRecordTerminalFailure(): void + { + $batch = ['user1@example.com', 'user2@example.com']; + + // Every attempt throttles, for more steps than MESSAGE_SEND_MAX_RETRIES, so retries are exhausted. + $script = \array_fill( + 0, + MESSAGE_SEND_MAX_RETRIES + 2, + static fn (string $recipient): string => 'Service unavailable, throttled' + ); + $adapter = new ScriptedEmailAdapter($script); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // The batch is attempted exactly MESSAGE_SEND_MAX_RETRIES times, then the still-throttled recipients + // become terminal failures with nothing delivered. + $this->assertSame(MESSAGE_SEND_MAX_RETRIES, $adapter->sendCalls, 'Attempts must be capped at MESSAGE_SEND_MAX_RETRIES'); + $this->assertSame(0, $result['delivered']); + $this->assertCount(\count($batch), $result['errors']); + } + + public function testDeliveryErrorsAreCappedWhileDeliveredCountIsExact(): void + { + // More failing recipients than the retained-error cap, plus a known number of successes, so the + // delivered tally stays exact even though the error list is truncated. + $failingCount = MESSAGE_DELIVERY_ERRORS_LIMIT + 20; + $succeedingCount = 100; + + $batch = []; + for ($i = 1; $i <= $failingCount; $i++) { + $batch[] = "fail{$i}@example.com"; + } + for ($i = 1; $i <= $succeedingCount; $i++) { + $batch[] = "ok{$i}@example.com"; + } + + // Permanent (non-retryable) failure for every "fail*" recipient; everyone else is delivered. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => \str_starts_with($recipient, 'fail') + ? 'Invalid recipient address' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // Errors are capped, but the delivered count reflects every success exactly. + $this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $result['errors'], 'Delivery errors must be capped'); + $this->assertSame($succeedingCount, $result['delivered'], 'Delivered count stays exact despite the error cap'); + $this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry'); + } + + public function testExpiredTokenCleanupFailureDoesNotCorruptAccounting(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com']; + + // The cleanup path always throws, but the worker must isolate it: the send result is what's accounted. + $database = new ExpiredTokenCleanupFailingDatabase(); + + // user1@ comes back with an expired device token (triggering cleanup, which then throws); the other + // two recipients are delivered. The expired-token error is terminal, never retryable. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => $recipient === 'user1@example.com' + ? 'Expired device token' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, $database); + + // The worker reached the cleanup (so the isolation is genuinely exercised) and it threw. + $this->assertGreaterThan(0, $database->targetUpdateAttempts, 'Expired-token cleanup must have been attempted'); + + // The send was attempted exactly once: a thrown cleanup must not look like a transient send failure + // and must not trigger a retry. + $this->assertSame(1, $adapter->sendCalls, 'Cleanup failure must not trigger a send retry'); + + // Accounting reflects the send result, not the cleanup throw: two delivered, one terminal failure. + $this->assertSame(\count($batch) - 1, $result['delivered']); + + // The recorded error is the provider's expired-token failure for the real recipient — proving the + // database exception was swallowed rather than misattributed as the send error. + $this->assertCount(1, $result['errors']); + $this->assertStringContainsString('user1@example.com', $result['errors'][0]); + $this->assertStringContainsString('Expired device token', $result['errors'][0]); + $this->assertStringNotContainsString('Transient database error', $result['errors'][0]); + } + + public function testRecipientsArePaginatedWithoutSubqueryFilter(): void + { + // 4500 subscribers over a 1000-row page is five pages, with a duplicate identifier inside one page to + // exercise per-page deduplication. + $count = 4500; + $data = $this->topicDataset($count); + $database = new RecordingDatabase($data['subscribers'], $data['targets']); + + $identifiers = $this->collectStream($database, topicIds: ['topic1']); + + $topicReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics')); + $subscriberReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'subscribers')); + + // The topic's `targets` attribute (the subQueryTopicTargets filter) is never materialized: topics are + // only read with a $sequence projection, and recipients come from the subscribers collection instead. + $this->assertNotEmpty($topicReads); + foreach ($topicReads as $call) { + $this->assertNotContains('targets', $this->selectFields($call['queries']), 'Topic read must not select the targets subquery attribute'); + } + + // Subscribers are walked in bounded pages via cursorAfter; no query ever requests a runaway page size. + $expectedPages = (int) \ceil($count / MESSAGE_RECIPIENTS_PAGE_SIZE); + $this->assertCount($expectedPages, $subscriberReads); + + foreach ($subscriberReads as $index => $call) { + $this->assertSame(MESSAGE_RECIPIENTS_PAGE_SIZE, $this->limitOf($call['queries'])); + $this->assertSame('100', $this->topicInternalIdOf($call['queries']), 'Subscribers must be filtered by the topic internal id'); + + if ($index === 0) { + $this->assertFalse($this->hasCursor($call['queries']), 'First page must not carry a cursor'); } else { - $delivered++; - $response->addResult($email); + $this->assertTrue($this->hasCursor($call['queries']), 'Subsequent pages must paginate with cursorAfter'); } } - $response->setDeliveredTo($delivered); + // No query in the whole stream issues a runaway (>= 1,000,000) limit. + foreach ($database->findCalls as $call) { + $this->assertLessThan(1_000_000, $this->limitOf($call['queries']), 'No query may request a million-row page'); + } - $this->inFlight--; + // The duplicate identifier collapses: one row per distinct subscriber identifier resolved. + $this->assertCount($count, $identifiers); + for ($i = 1; $i <= $count; $i++) { + $this->assertArrayHasKey("user{$i}@example.com", $identifiers); + } + } - return $response->toArray(); + public function testStreamDeduplicatesIdentifiersWithinPage(): void + { + // Two subscribers in a single page resolve to targets sharing one identifier; the page must collapse + // them to a single recipient entry. + $subscribers = [ + new Document(['$id' => 'sub1', '$sequence' => '1', 'topicInternalId' => '100', 'targetInternalId' => '1', 'providerType' => MESSAGE_TYPE_EMAIL]), + new Document(['$id' => 'sub2', '$sequence' => '2', 'topicInternalId' => '100', 'targetInternalId' => '2', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $targets = [ + '1' => new Document(['$id' => 'target1', '$sequence' => '1', 'providerId' => null, 'identifier' => 'dup@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + '2' => new Document(['$id' => 'target2', '$sequence' => '2', 'providerId' => null, 'identifier' => 'dup@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $database = new RecordingDatabase($subscribers, $targets); + + $pages = $this->collectPages($database, topicIds: ['topic1']); + + $this->assertCount(1, $pages, 'A single subscriber page yields a single grouped result'); + $providerId = $this->provider()->getId(); + $this->assertArrayHasKey($providerId, $pages[0]); + $this->assertSame(['dup@example.com'], \array_keys($pages[0][$providerId]), 'Duplicate identifiers must collapse to one'); + } + + public function testStreamResolvesUsersAndDirectTargets(): void + { + $userTargets = [ + new Document(['$id' => 'utarget1', '$sequence' => '10', 'providerId' => null, 'identifier' => 'byuser@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $directTargets = [ + new Document(['$id' => 'dtarget1', '$sequence' => '20', 'providerId' => null, 'identifier' => 'bytarget@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $database = new RecordingDatabase([], [], $userTargets, $directTargets); + + $identifiers = $this->collectStream($database, userIds: ['user-a'], targetIds: ['target-b']); + + // No topics in play: only the user and direct-target queries run, and both identifiers resolve. + $this->assertSame([], \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics'))); + $this->assertArrayHasKey('byuser@example.com', $identifiers); + $this->assertArrayHasKey('bytarget@example.com', $identifiers); + $this->assertCount(2, $identifiers); + } + + /** + * Build a topic-backed dataset: $count subscribers, each pointing at one email target. A single identifier + * is duplicated so per-page dedup is exercised when the dataset fits in one page. + * + * @return array{subscribers: array, targets: array} + */ + private function topicDataset(int $count): array + { + $subscribers = []; + $targets = []; + + for ($i = 1; $i <= $count; $i++) { + $sequence = (string) $i; + $subscribers[] = new Document([ + '$id' => 'sub' . $i, + '$sequence' => $sequence, + 'topicInternalId' => '100', + 'targetInternalId' => $sequence, + 'providerType' => MESSAGE_TYPE_EMAIL, + ]); + $targets[$sequence] = new Document([ + '$id' => 'target' . $i, + '$sequence' => $sequence, + 'providerId' => null, + 'identifier' => "user{$i}@example.com", + 'providerType' => MESSAGE_TYPE_EMAIL, + ]); + } + + return ['subscribers' => $subscribers, 'targets' => $targets]; + } + + /** + * Drive {@see Messaging::streamRecipients()} and merge every yielded page into a single identifier map. + * + * @param array $topicIds + * @param array $userIds + * @param array $targetIds + * @return array + */ + private function collectStream( + RecordingDatabase $database, + array $topicIds = [], + array $userIds = [], + array $targetIds = [] + ): array { + $identifiers = []; + + foreach ($this->collectPages($database, $topicIds, $userIds, $targetIds) as $page) { + foreach ($page as $perProvider) { + foreach ($perProvider as $identifier => $_) { + $identifiers[$identifier] = null; + } + } + } + + return $identifiers; + } + + /** + * @param array $topicIds + * @param array $userIds + * @param array $targetIds + * @return array>> + */ + private function collectPages( + RecordingDatabase $database, + array $topicIds = [], + array $userIds = [], + array $targetIds = [] + ): array { + $method = new \ReflectionMethod(Messaging::class, 'streamRecipients'); + $worker = new TestableMessaging(); + + /** @var \Generator>> $generator */ + $generator = $method->invoke( + $worker, + $database, + $topicIds, + $userIds, + $targetIds, + MESSAGE_TYPE_EMAIL, + $this->provider() + ); + + return \iterator_to_array($generator, false); + } + + /** + * @param array $queries + * @return array + */ + private function selectFields(array $queries): array + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_SELECT) { + return $query->getValues(); + } + } + + return []; + } + + /** + * @param array $queries + */ + private function limitOf(array $queries): int + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_LIMIT) { + return (int) ($query->getValues()[0] ?? 0); + } + } + + return 0; + } + + /** + * @param array $queries + */ + private function hasCursor(array $queries): bool + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) { + return true; + } + } + + return false; + } + + /** + * @param array $queries + */ + private function topicInternalIdOf(array $queries): ?string + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'topicInternalId') { + return (string) ($query->getValues()[0] ?? null); + } + } + + return null; } } /** - * Worker subclass that injects the instrumented email adapter instead of a network-backed provider adapter. - * Retry backoff is zeroed so the throttle/retry tests stay instant. + * Worker subclass that zeroes the retry backoff so the throttle/retry tests stay instant. The worker already + * skips a non-positive delay, so no coroutine sleep is ever reached. */ class TestableMessaging extends Messaging { - public function __construct(private readonly EmailAdapter $emailAdapter) - { - } - - protected function getEmailAdapter(Document $provider): ?EmailAdapter - { - return $this->emailAdapter; - } - protected function retryDelay(): float { return 0.0; @@ -180,33 +606,15 @@ class ScriptedEmailAdapter extends EmailAdapter } /** - * In-memory {@see Database} double. Serves subscribers/targets through cursor pagination and records every - * query issued so the test can assert the streaming contract (cursorAfter + bounded page size, and that the - * 1M-row subQueryTopicTargets filter is never used). + * Minimal in-memory {@see Database} double for the retry tests, which only build email messages with no + * attachments and never touch the expired-token cleanup path. Every read resolves empty. */ -class RecordingDatabase extends Database +class InMemoryDatabase extends Database { - /** - * @var array}> - */ - public array $findCalls = []; - - public ?Document $updatedMessage = null; - private readonly Authorization $authorization; - /** - * @param array $subscribers Subscriber rows (carry $sequence + targetInternalId + providerType). - * @param array $targets Target rows keyed by $sequence. - * @param array $directTargets Targets addressed directly by id or userId. - * @param Document $provider The single enabled provider. - */ - public function __construct( - private readonly array $subscribers = [], - private readonly array $targets = [], - private readonly array $directTargets = [], - private readonly ?Document $provider = null - ) { + public function __construct() + { $this->authorization = new Authorization(); } @@ -222,10 +630,6 @@ class RecordingDatabase extends Database public function findOne(string $collection, array $queries = []): Document { - if ($collection === 'providers') { - return $this->provider ?? new Document(); - } - return new Document(); } @@ -236,11 +640,83 @@ class RecordingDatabase extends Database public function updateDocument(string $collection, string $id, Document $document): Document { - if ($collection === 'messages') { - $this->updatedMessage = $document; + return $document; + } + + public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array + { + return []; + } +} + +/** + * {@see InMemoryDatabase} variant whose expired-device-token cleanup always fails: `findOne('targets')` + * resolves to a real target so the worker attempts the update, and `updateDocument('targets')` throws a + * transient error. Used to prove that a DB hiccup during best-effort cleanup never leaks into delivery + * accounting or the retry decision. + */ +class ExpiredTokenCleanupFailingDatabase extends InMemoryDatabase +{ + public int $targetUpdateAttempts = 0; + + public function findOne(string $collection, array $queries = []): Document + { + if ($collection === 'targets') { + return new Document(['$id' => 'target-expired', '$sequence' => '1']); } - return $document; + return parent::findOne($collection, $queries); + } + + public function updateDocument(string $collection, string $id, Document $document): Document + { + if ($collection === 'targets') { + $this->targetUpdateAttempts++; + + throw new \RuntimeException('Transient database error during target cleanup'); + } + + return parent::updateDocument($collection, $id, $document); + } +} + +/** + * In-memory {@see Database} double for the streaming tests. Serves subscribers/targets through cursor + * pagination and records every query issued so the test can assert the streaming contract (cursorAfter + + * bounded page size, topic reads never selecting the 1M-row `targets` subquery, no runaway limits). + */ +class RecordingDatabase extends Database +{ + /** + * @var array}> + */ + public array $findCalls = []; + + private readonly Authorization $authorization; + + /** + * @param array $subscribers Subscriber rows (carry $sequence + targetInternalId + providerType). + * @param array $targets Target rows keyed by $sequence (resolved from a subscriber page). + * @param array $userTargets Targets addressed by userId. + * @param array $directTargets Targets addressed directly by id. + */ + public function __construct( + private readonly array $subscribers = [], + private readonly array $targets = [], + private readonly array $userTargets = [], + private readonly array $directTargets = [] + ) { + $this->authorization = new Authorization(); + } + + public function getAuthorization(): Authorization + { + return $this->authorization; + } + + public function skipValidation(callable $callback): mixed + { + return $callback(); } public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array @@ -276,7 +752,7 @@ class RecordingDatabase extends Database } /** - * Either resolve a subscriber page's targets by $sequence, or paginate direct/user targets. + * Either resolve a subscriber page's targets by $sequence, or paginate user/direct targets. * * @param array $queries * @return array @@ -292,6 +768,10 @@ class RecordingDatabase extends Database \array_filter($sequences, fn ($sequence) => isset($this->targets[$sequence])) )); } + + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'userId') { + return $this->paginate($this->userTargets, $queries); + } } return $this->paginate($this->directTargets, $queries); @@ -327,478 +807,3 @@ class RecordingDatabase extends Database return null; } } - -/** - * {@see RecordingDatabase} variant whose expired-device-token cleanup always fails: `findOne('targets')` - * resolves to a real target so the worker attempts the update, and `updateDocument('targets')` throws a - * transient error. Used to prove that a DB hiccup during best-effort cleanup never leaks into delivery - * accounting or the retry decision. - */ -class ExpiredTokenCleanupFailingDatabase extends RecordingDatabase -{ - public int $targetUpdateAttempts = 0; - - public function findOne(string $collection, array $queries = []): Document - { - if ($collection === 'targets') { - return new Document(['$id' => 'target-expired', '$sequence' => '1']); - } - - return parent::findOne($collection, $queries); - } - - public function updateDocument(string $collection, string $id, Document $document): Document - { - if ($collection === 'targets') { - $this->targetUpdateAttempts++; - - throw new \RuntimeException('Transient database error during target cleanup'); - } - - return parent::updateDocument($collection, $id, $document); - } -} - -class MessagingFanoutTest extends TestCase -{ - private const RECIPIENT_COUNT = 4500; - - protected function setUp(): void - { - // These tests drive the worker through real Swoole coroutines (Co\run/batch). Under the native - // extension that scheduler runs in-process and destabilises later tests in the shared unit suite - // (the run aborts mid-suite inside unrelated coroutine-sensitive tests). They run locally against - // the Fiber polyfill in SwooleCoroutinePolyfill.php; the real worker path is exercised on CI by the - // e2e Messaging suite. See PR #12465. - if (\extension_loaded('swoole')) { - $this->markTestSkipped('Fan-out coroutine tests run only without the native Swoole extension; the worker path is covered on CI by the e2e Messaging suite.'); - } - } - - private function provider(): Document - { - return new Document([ - '$id' => 'provider1', - '$sequence' => '1', - 'enabled' => true, - 'type' => MESSAGE_TYPE_EMAIL, - 'provider' => 'mock', - 'name' => 'Mock Email', - 'credentials' => [], - 'options' => [ - 'fromName' => 'Appwrite', - 'fromEmail' => 'noreply@example.com', - ], - ]); - } - - private function message(): Document - { - return new Document([ - '$id' => 'message1', - '$sequence' => '1', - 'providerType' => MESSAGE_TYPE_EMAIL, - 'topics' => ['topic1'], - 'targets' => [], - 'users' => [], - 'status' => MessageStatus::PROCESSING, - 'search' => 'message1', - 'data' => [ - 'subject' => 'Hello', - 'content' => 'World', - 'html' => false, - ], - ]); - } - - /** - * Build a topic-backed dataset: RECIPIENT_COUNT subscribers, each pointing at one email target. - * - * @return array{subscribers: array, targets: array} - */ - private function topicDataset(int $count): array - { - $subscribers = []; - $targets = []; - - for ($i = 1; $i <= $count; $i++) { - $sequence = (string) $i; - $subscribers[] = new Document([ - '$id' => 'sub' . $i, - '$sequence' => $sequence, - 'topicInternalId' => '100', - 'targetInternalId' => $sequence, - 'providerType' => MESSAGE_TYPE_EMAIL, - ]); - $targets[$sequence] = new Document([ - '$id' => 'target' . $i, - '$sequence' => $sequence, - 'providerId' => null, - 'identifier' => "user{$i}@example.com", - 'providerType' => MESSAGE_TYPE_EMAIL, - ]); - } - - return ['subscribers' => $subscribers, 'targets' => $targets]; - } - - private function sendTopic( - RecordingDatabase $database, - EmailAdapter $adapter - ): void { - $worker = new TestableMessaging($adapter); - // No setAccessible(true): reflection invokes private methods without it on PHP 8.1+, and the call is - // deprecated on PHP 8.5 (the CI unit-test runtime), where the suite fails on triggered deprecations. - $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); - - $publisher = $this->createMock(UsagePublisher::class); - $publisher->method('enqueue')->willReturn(true); - - $device = new Local(\sys_get_temp_dir()); - $project = new Document(['$id' => 'project1', '$sequence' => '1']); - - run(function () use ($worker, $method, $database, $device, $project, $publisher) { - $method->invoke($worker, $database, $this->message(), $device, $project, $publisher); - }); - } - - public function testRecipientsArePaginatedWithoutSubqueryFilter(): void - { - $data = $this->topicDataset(self::RECIPIENT_COUNT); - $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); - $adapter = new ConcurrencyProbeAdapter(); - - $this->sendTopic($database, $adapter); - - $topicReads = \array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics'); - $subscriberReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'subscribers')); - - // The topic's `targets` attribute (the subQueryTopicTargets filter) is never materialized: topics are - // only read with a $sequence projection, and recipients come from the subscribers collection instead. - $this->assertNotEmpty($topicReads); - foreach ($topicReads as $call) { - $selected = $this->selectFields($call['queries']); - $this->assertNotContains('targets', $selected, 'Topic read must not select the targets subquery attribute'); - } - - // Subscribers are walked in bounded pages via cursorAfter. 4500 recipients over a 1000 page size is - // five pages; every page after the first carries a cursor and none exceeds the page size. - $expectedPages = (int) \ceil(self::RECIPIENT_COUNT / MESSAGE_RECIPIENTS_PAGE_SIZE); - $this->assertCount($expectedPages, $subscriberReads); - - foreach ($subscriberReads as $index => $call) { - $this->assertSame(MESSAGE_RECIPIENTS_PAGE_SIZE, $this->limitOf($call['queries'])); - - if ($index === 0) { - $this->assertFalse($this->hasCursor($call['queries']), 'First page must not carry a cursor'); - } else { - $this->assertTrue($this->hasCursor($call['queries']), 'Subsequent pages must paginate with cursorAfter'); - } - } - } - - public function testConcurrentSendsNeverExceedConfiguredLimit(): void - { - // One recipient per batch (maxPerRequest = 1) maximises the number of concurrent in-flight sends so - // the Semaphore's bound is actually exercised: 1500 batches contend for MESSAGE_SEND_CONCURRENCY permits. - $count = 1500; - $data = $this->topicDataset($count); - $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); - $adapter = new ConcurrencyProbeAdapter(maxPerRequest: 1); - - $this->sendTopic($database, $adapter); - - $this->assertSame($count, $adapter->sendCalls, 'Every recipient is sent exactly once'); - $this->assertGreaterThan(1, $adapter->peakConcurrency, 'Sends should actually run concurrently'); - $this->assertLessThanOrEqual( - MESSAGE_SEND_CONCURRENCY, - $adapter->peakConcurrency, - 'Concurrent sends must never exceed MESSAGE_SEND_CONCURRENCY' - ); - } - - public function testDeliveryErrorsAreCappedWhileFailedCountIsComplete(): void - { - $count = self::RECIPIENT_COUNT; - $data = $this->topicDataset($count); - // Every recipient fails: 4500 individual failures, but only MESSAGE_DELIVERY_ERRORS_LIMIT are retained. - $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); - $adapter = new ConcurrencyProbeAdapter(failureFor: fn (string $recipient) => true); - - $this->sendTopic($database, $adapter); - - $message = $database->updatedMessage; - $this->assertNotNull($message); - - $errors = $message->getAttribute('deliveryErrors'); - $this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $errors, 'Delivery errors must be capped'); - - // Failed recipients are fully accounted for even though only 100 error strings are kept: every one of - // the 4500 recipients failed, so deliveredTotal is zero and the message is marked FAILED. - $this->assertSame(0, $message->getAttribute('deliveredTotal')); - $this->assertSame(MessageStatus::FAILED, $message->getAttribute('status')); - } - - public function testDeliveredTotalAndStatusReflectPartialSuccess(): void - { - $count = self::RECIPIENT_COUNT; - $data = $this->topicDataset($count); - // Fail exactly the recipients whose local-part index is divisible by 10. - $failing = static function (string $recipient): bool { - \preg_match('/user(\d+)@/', $recipient, $matches); - - return isset($matches[1]) && ((int) $matches[1]) % 10 === 0; - }; - $expectedFailures = \intdiv($count, 10); - - $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); - $adapter = new ConcurrencyProbeAdapter(failureFor: $failing); - - $this->sendTopic($database, $adapter); - - $message = $database->updatedMessage; - $this->assertNotNull($message); - - // deliveredTotal is exact across all pages; failures partition the remainder exactly. - $this->assertSame($count - $expectedFailures, $message->getAttribute('deliveredTotal')); - $this->assertSame(MessageStatus::FAILED, $message->getAttribute('status')); - $this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $message->getAttribute('deliveryErrors')); - } - - public function testFullSuccessMarksMessageSent(): void - { - $data = $this->topicDataset(self::RECIPIENT_COUNT); - $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); - $adapter = new ConcurrencyProbeAdapter(); - - $this->sendTopic($database, $adapter); - - $message = $database->updatedMessage; - $this->assertNotNull($message); - $this->assertSame(self::RECIPIENT_COUNT, $message->getAttribute('deliveredTotal')); - $this->assertSame(MessageStatus::SENT, $message->getAttribute('status')); - $this->assertEmpty($message->getAttribute('deliveryErrors')); - } - - public function testTerminalStatusIsSkipped(): void - { - $data = $this->topicDataset(10); - $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); - $adapter = new ConcurrencyProbeAdapter(); - - $worker = new TestableMessaging($adapter); - // No setAccessible(true): reflection invokes private methods without it on PHP 8.1+, and the call is - // deprecated on PHP 8.5 (the CI unit-test runtime), where the suite fails on triggered deprecations. - $method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage'); - $publisher = $this->createMock(UsagePublisher::class); - $device = new Local(\sys_get_temp_dir()); - $project = new Document(['$id' => 'project1', '$sequence' => '1']); - - $message = $this->message(); - $message->setAttribute('status', MessageStatus::SENT); - - run(function () use ($worker, $method, $database, $device, $project, $publisher, $message) { - $method->invoke($worker, $database, $message, $device, $project, $publisher); - }); - - $this->assertSame(0, $adapter->sendCalls, 'Already-sent messages must not be reprocessed'); - $this->assertNull($database->updatedMessage, 'Terminal messages must not be rewritten'); - } - - public function testThrottledBatchIsRetriedThenFullyDelivered(): void - { - $count = 5; - $data = $this->topicDataset($count); - $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); - - // First attempt throttles every recipient; the script then runs out so the second attempt succeeds. - $adapter = new ScriptedEmailAdapter([ - static fn (string $recipient): string => 'Rate limit exceeded, please retry', - ]); - - $this->sendTopic($database, $adapter); - - $message = $database->updatedMessage; - $this->assertNotNull($message); - - // The batch was retried exactly once and then fully delivered with no double-counting. - $this->assertSame(2, $adapter->sendCalls, 'Throttled batch must be retried once'); - $this->assertSame($count, $message->getAttribute('deliveredTotal')); - $this->assertSame(MessageStatus::SENT, $message->getAttribute('status')); - $this->assertEmpty($message->getAttribute('deliveryErrors')); - - // Both attempts targeted the full recipient set (none had succeeded on the first attempt). - $this->assertCount($count, $adapter->recipientsPerCall[0]); - $this->assertCount($count, $adapter->recipientsPerCall[1]); - } - - public function testRetryTargetsOnlyThrottledRecipients(): void - { - $count = 5; - $data = $this->topicDataset($count); - $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); - - // user2@ and user4@ throttle on the first attempt; everyone else succeeds. The second attempt has no - // script step, so the two retried recipients then succeed. - $throttled = ['user2@example.com', 'user4@example.com']; - $adapter = new ScriptedEmailAdapter([ - static fn (string $recipient): ?string => \in_array($recipient, $throttled, true) - ? 'Too Many Requests' - : null, - ]); - - $this->sendTopic($database, $adapter); - - $message = $database->updatedMessage; - $this->assertNotNull($message); - - $this->assertSame(2, $adapter->sendCalls, 'A partially throttled batch must be retried once'); - - // The retry must re-send to ONLY the throttled recipients — never the ones already delivered. - \sort($adapter->recipientsPerCall[1]); - $this->assertSame($throttled, $adapter->recipientsPerCall[1]); - - // Every recipient is ultimately delivered and counted exactly once. - $this->assertSame($count, $message->getAttribute('deliveredTotal')); - $this->assertSame(MessageStatus::SENT, $message->getAttribute('status')); - $this->assertEmpty($message->getAttribute('deliveryErrors')); - } - - public function testNonRetryableFailureIsNotRetried(): void - { - $count = 3; - $data = $this->topicDataset($count); - $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); - - // user1@ permanently fails with a non-retryable error; the rest succeed. - $adapter = new ScriptedEmailAdapter([ - static fn (string $recipient): ?string => $recipient === 'user1@example.com' - ? 'Invalid recipient address' - : null, - ]); - - $this->sendTopic($database, $adapter); - - $message = $database->updatedMessage; - $this->assertNotNull($message); - - // A permanent failure is recorded immediately, never retried. - $this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry'); - $this->assertSame($count - 1, $message->getAttribute('deliveredTotal')); - $this->assertSame(MessageStatus::FAILED, $message->getAttribute('status')); - $errors = $message->getAttribute('deliveryErrors'); - $this->assertCount(1, $errors); - $this->assertStringContainsString('user1@example.com', $errors[0]); - $this->assertStringContainsString('Invalid recipient address', $errors[0]); - } - - public function testExhaustedRetriesRecordTerminalFailure(): void - { - $count = 2; - $data = $this->topicDataset($count); - $database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); - - // Every attempt throttles, for more steps than MESSAGE_SEND_MAX_RETRIES, so retries are exhausted. - $script = \array_fill( - 0, - MESSAGE_SEND_MAX_RETRIES + 2, - static fn (string $recipient): string => 'Service unavailable, throttled' - ); - $adapter = new ScriptedEmailAdapter($script); - - $this->sendTopic($database, $adapter); - - $message = $database->updatedMessage; - $this->assertNotNull($message); - - // The batch is attempted exactly MESSAGE_SEND_MAX_RETRIES times, then the still-throttled recipients - // become terminal failures with nothing delivered. - $this->assertSame(MESSAGE_SEND_MAX_RETRIES, $adapter->sendCalls, 'Attempts must be capped at MESSAGE_SEND_MAX_RETRIES'); - $this->assertSame(0, $message->getAttribute('deliveredTotal')); - $this->assertSame(MessageStatus::FAILED, $message->getAttribute('status')); - $this->assertCount($count, $message->getAttribute('deliveryErrors')); - } - - public function testExpiredTokenCleanupFailureDoesNotCorruptAccounting(): void - { - $count = 3; - $data = $this->topicDataset($count); - // The cleanup path always throws, but the worker must isolate it: the send result is what's accounted. - $database = new ExpiredTokenCleanupFailingDatabase($data['subscribers'], $data['targets'], [], $this->provider()); - - // user1@ comes back with an expired device token (triggering cleanup, which then throws); the other - // two recipients are delivered. The expired-token error is terminal, never retryable. - $adapter = new ScriptedEmailAdapter([ - static fn (string $recipient): ?string => $recipient === 'user1@example.com' - ? 'Expired device token' - : null, - ]); - - $this->sendTopic($database, $adapter); - - $message = $database->updatedMessage; - $this->assertNotNull($message); - - // The worker reached the cleanup (so the isolation is genuinely exercised) and it threw. - $this->assertGreaterThan(0, $database->targetUpdateAttempts, 'Expired-token cleanup must have been attempted'); - - // The send was attempted exactly once: a thrown cleanup must not look like a transient send failure - // and must not trigger a retry. - $this->assertSame(1, $adapter->sendCalls, 'Cleanup failure must not trigger a send retry'); - - // Accounting reflects the send result, not the cleanup throw: two delivered, one terminal failure. - $this->assertSame($count - 1, $message->getAttribute('deliveredTotal')); - $this->assertSame(MessageStatus::FAILED, $message->getAttribute('status')); - - // The recorded error is the provider's expired-token failure for the real recipient — proving the - // database exception was swallowed rather than misattributed as the send error. - $errors = $message->getAttribute('deliveryErrors'); - $this->assertCount(1, $errors); - $this->assertStringContainsString('user1@example.com', $errors[0]); - $this->assertStringContainsString('Expired device token', $errors[0]); - $this->assertStringNotContainsString('Transient database error', $errors[0]); - } - - /** - * @param array $queries - * @return array - */ - private function selectFields(array $queries): array - { - foreach ($queries as $query) { - if ($query->getMethod() === Query::TYPE_SELECT) { - return $query->getValues(); - } - } - - return []; - } - - /** - * @param array $queries - */ - private function limitOf(array $queries): int - { - foreach ($queries as $query) { - if ($query->getMethod() === Query::TYPE_LIMIT) { - return (int) ($query->getValues()[0] ?? 0); - } - } - - return 0; - } - - /** - * @param array $queries - */ - private function hasCursor(array $queries): bool - { - foreach ($queries as $query) { - if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) { - return true; - } - } - - return false; - } -} diff --git a/tests/unit/Messaging/SwooleCoroutinePolyfill.php b/tests/unit/Messaging/SwooleCoroutinePolyfill.php deleted file mode 100644 index fcf520a911..0000000000 --- a/tests/unit/Messaging/SwooleCoroutinePolyfill.php +++ /dev/null @@ -1,168 +0,0 @@ - - */ - private array $buffer = []; - - public function __construct(private readonly int $capacity = 1) - { - } - - public function push(mixed $data, float $timeout = 0): bool - { - // Enforce capacity so a full channel reports back-pressure exactly like a real Swoole channel. - // A full Semaphore permit channel then fails to acquire, which surfaces as a - // {@see \Utopia\Lock\Exception\Contention} that the batch() scheduler defers to a later tick — - // making the concurrency bound genuinely enforced rather than silently unbounded. - if (\count($this->buffer) >= $this->capacity) { - return false; - } - - $this->buffer[] = $data; - - return true; - } - - public function pop(float $timeout = 0): mixed - { - return \array_shift($this->buffer); - } - - public function isFull(): bool - { - return \count($this->buffer) >= $this->capacity; - } - - public function isEmpty(): bool - { - return \count($this->buffer) === 0; - } - } - } -} - -namespace Swoole\Coroutine { - use Swoole\Coroutine; - use Utopia\Lock\Exception\Contention; - - if (! \function_exists('Swoole\Coroutine\run')) { - function run(callable $callback): void - { - $fiber = new \Fiber($callback); - $previous = Coroutine::$current; - Coroutine::$current = $fiber; - $fiber->start(); - - while (! $fiber->isTerminated()) { - $fiber->resume(); - } - - Coroutine::$current = $previous; - } - } - - if (! \function_exists('Swoole\Coroutine\batch')) { - /** - * @param array $tasks - * @return array - */ - function batch(array $tasks, float $timeout = -1): array - { - $results = []; - $done = []; - $fibers = []; - - foreach ($tasks as $key => $task) { - $fibers[$key] = new \Fiber($task); - } - - $previous = Coroutine::$current; - - while (\count($done) < \count($tasks)) { - $progressed = false; - - foreach ($fibers as $key => $fiber) { - if (isset($done[$key])) { - continue; - } - - Coroutine::$current = $fiber; - - try { - if (! $fiber->isStarted()) { - $fiber->start(); - } elseif (! $fiber->isTerminated()) { - $fiber->resume(); - } - - // Any successful start/resume advances a fiber (toward a yield or termination), which - // counts as progress; fibers advance monotonically so this cannot spin forever. - $progressed = true; - - if ($fiber->isTerminated()) { - $results[$key] = $fiber->getReturn(); - $done[$key] = true; - } - } catch (Contention) { - // All permits held: model a blocking acquire by retrying the task on a later tick. - $fibers[$key] = new \Fiber($tasks[$key]); - $progressed = true; - } - } - - if (! $progressed) { - break; - } - } - - Coroutine::$current = $previous; - \ksort($results); - - return $results; - } - } -} From 2084b21ae3e3b768a285376bbcc7d7c3858dcfa4 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Jun 2026 13:41:19 +1200 Subject: [PATCH 9/9] perf(messaging): raise recipient page size to 5000 to cut DB round-trips Streaming a large-topic send paginated subscribers 1000 at a time, so a 500K-recipient topic issued ~500 subscriber-page queries plus their per-page target resolutions. Raise MESSAGE_RECIPIENTS_PAGE_SIZE to 5000 (APP_LIMIT_COUNT), cutting topic-path round-trips ~5x with no extra memory pressure (bounded concurrency still caps in-flight sends). The streaming test fixture now sizes itself to the page-size constant so it keeps exercising multi-page pagination. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/init/constants.php | 2 +- tests/unit/Messaging/MessagingFanoutTest.php | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/app/init/constants.php b/app/init/constants.php index 2b72f8814f..4fe7c418a9 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -243,7 +243,7 @@ const RULE_STATUS_VERIFIED = 'verified'; const MESSAGE_SEND_TYPE_INTERNAL = 'internal'; const MESSAGE_SEND_TYPE_EXTERNAL = 'external'; // External message fan-out tuning -const MESSAGE_RECIPIENTS_PAGE_SIZE = 1000; // Recipients resolved from the database per page while streaming a send +const MESSAGE_RECIPIENTS_PAGE_SIZE = 5000; // Recipients resolved from the database per page while streaming a send; aligned with APP_LIMIT_COUNT to minimise round-trips on large topics const MESSAGE_SEND_CONCURRENCY = 10; // Maximum adapter send requests running concurrently within a single send job const MESSAGE_DELIVERY_ERRORS_LIMIT = 100; // Maximum number of per-recipient delivery errors retained on a message const MESSAGE_SEND_MAX_RETRIES = 5; // Maximum number of attempts to deliver a batch when the provider throttles or fails transiently diff --git a/tests/unit/Messaging/MessagingFanoutTest.php b/tests/unit/Messaging/MessagingFanoutTest.php index 97e888e228..02b1ee55d4 100644 --- a/tests/unit/Messaging/MessagingFanoutTest.php +++ b/tests/unit/Messaging/MessagingFanoutTest.php @@ -292,9 +292,8 @@ class MessagingFanoutTest extends TestCase public function testRecipientsArePaginatedWithoutSubqueryFilter(): void { - // 4500 subscribers over a 1000-row page is five pages, with a duplicate identifier inside one page to - // exercise per-page deduplication. - $count = 4500; + // Always span several pages regardless of the configured page size, so cursor pagination is exercised. + $count = MESSAGE_RECIPIENTS_PAGE_SIZE * 2 + 500; $data = $this->topicDataset($count); $database = new RecordingDatabase($data['subscribers'], $data['targets']);