fix(messaging): stream large-topic sends and bound send concurrency

Large external sends (e.g. a 500K-recipient topic email) OOM'd the
messaging worker: sendExternalMessage read $topic->getAttribute('targets'),
triggering the subQueryTopicTargets filter that loads up to
APP_LIMIT_SUBSCRIBERS_SUBQUERY (1M) subscribers plus a giant targets IN()
into memory, then fanned every adapter-sized chunk out through a single
unbounded Swoole\Coroutine\batch() — spawning unbounded concurrent sends.

Rework the external send path into a single-job streaming model:

- Recipients are streamed in bounded pages (MESSAGE_RECIPIENTS_PAGE_SIZE)
  via cursor pagination. Topics walk the subscribers collection by
  topicInternalId and resolve each page's targets by $sequence, so the
  topic's targets subquery is never materialized. Users/direct targets
  paginate the targets collection. Peak memory is O(page), not O(topic).
- Send concurrency is bounded with Utopia\Lock\Semaphore
  (MESSAGE_SEND_CONCURRENCY permits): each adapter batch send is wrapped in
  $semaphore->withLock() and awaited per page with Swoole's native batch().
  Excess coroutines block on acquire instead of all running at once.
- Results accumulate across pages: an exact deliveredTotal/failedTotal
  partition (failed = recipients - delivered) and a delivery-errors list
  capped at MESSAGE_DELIVERY_ERRORS_LIMIT, so a 500K send never collects
  one error string per recipient.
- An idempotency guard skips messages already in a terminal status, so
  queue redelivery does not reprocess a completed send.

Only existing message fields (status, deliveredTotal, deliveryErrors) are
written — no schema, migration, model, or queue-payload changes. Internal
SMS and other transactional paths are untouched.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jake BarnbyandClaude Opus 4.8 committed 2026-06-03 02:10:47 +12:00
1 parent 69b49c7228
commit da061ba7d2
4 files changed
+1040 -178

No files matched your search

+4
View File
@@ -242,6 +242,10 @@ const RULE_STATUS_VERIFIED = 'verified';
// Message types
const MESSAGE_SEND_TYPE_INTERNAL = 'internal';
const MESSAGE_SEND_TYPE_EXTERNAL = 'external';
// External message fan-out tuning
const MESSAGE_RECIPIENTS_PAGE_SIZE = 1000; // Recipients resolved from the database per page while streaming a send
const MESSAGE_SEND_CONCURRENCY = 10; // Maximum adapter send requests running concurrently within a single send job
const MESSAGE_DELIVERY_ERRORS_LIMIT = 100; // Maximum number of per-recipient delivery errors retained on a message
// Mail Types
const MAIL_TYPE_VERIFICATION = 'verification';
const MAIL_TYPE_MAGIC_SESSION = 'magicSession';
+348 -178
View File
@@ -16,6 +16,7 @@ use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\DSN\DSN;
use Utopia\Lock\Semaphore;
use Utopia\Logger\Log;
use Utopia\Messaging\Adapter\Email as EmailAdapter;
use Utopia\Messaging\Adapter\Email\Mailgun;
@@ -130,69 +131,19 @@ class Messaging extends Action
Document $project,
UsagePublisher $publisherForUsage
): void {
$status = $message->getAttribute('status');
// Idempotency guard: queue redelivery can hand us a message that already finished.
if (\in_array($status, [MessageStatus::SENT, MessageStatus::FAILED], true)) {
Span::add('message.skipped', 'already_processed');
return;
}
$topicIds = $message->getAttribute('topics', []);
$targetIds = $message->getAttribute('targets', []);
$userIds = $message->getAttribute('users', []);
$providerType = $message->getAttribute('providerType');
/**
* @var array<Document> $allTargets
*/
$allTargets = [];
if (\count($topicIds) > 0) {
$topics = $dbForProject->find('topics', [
Query::equal('$id', $topicIds),
Query::limit(\count($topicIds)),
]);
foreach ($topics as $topic) {
$targets = \array_filter($topic->getAttribute('targets'), function (Document $target) use ($providerType) {
return $target->getAttribute('providerType') === $providerType;
});
\array_push($allTargets, ...$targets);
}
}
if (\count($userIds) > 0) {
$limit = 1000;
$offset = 0;
do {
$targets = $dbForProject->find('targets', [
Query::equal('userId', $userIds),
Query::select(['providerId', 'identifier']),
Query::equal('providerType', [$providerType]),
Query::limit($limit),
Query::offset($offset),
]);
\array_push($allTargets, ...$targets);
$offset += \count($targets);
} while (\count($targets) === $limit);
}
if (\count($targetIds) > 0) {
$targets = $dbForProject->find('targets', [
Query::select(['providerId', 'identifier']),
Query::equal('$id', $targetIds),
Query::equal('providerType', [$providerType]),
Query::limit(\count($targetIds)),
]);
\array_push($allTargets, ...$targets);
}
if (empty($allTargets)) {
$dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([
'status' => MessageStatus::FAILED,
'deliveryErrors' => ['No valid recipients found.']
]));
Span::add('message.skipped', 'no_valid_recipients');
return;
}
$default = $dbForProject->findOne('providers', [
Query::equal('enabled', [true]),
Query::equal('type', [$providerType]),
@@ -209,55 +160,35 @@ class Messaging extends Action
}
/**
* @var array<string, array<string, null>> $identifiers
*/
$identifiers = [];
/**
* @var array<Document> $providers
* Resolved providers cached for the lifetime of this job, keyed by provider id.
* Seeded with the default provider so most sends never touch the providers collection.
*
* @var array<string, Document> $providers
*/
$providers = [
$default->getId() => $default
$default->getId() => $default,
];
foreach ($allTargets as $target) {
$providerId = $target->getAttribute('providerId');
$semaphore = new Semaphore(MESSAGE_SEND_CONCURRENCY);
if (!$providerId) {
$providerId = $default->getId();
}
$deliveredTotal = 0;
$failedTotal = 0;
$deliveryErrors = [];
$hasRecipients = false;
if ($providerId) {
if (!\array_key_exists($providerId, $identifiers)) {
$identifiers[$providerId] = [];
}
// Use null as value to avoid duplicate keys
$identifiers[$providerId][$target->getAttribute('identifier')] = null;
}
}
foreach ($this->streamRecipients($dbForProject, $topicIds, $userIds, $targetIds, $providerType, $default) as $page) {
$hasRecipients = true;
/**
* @var array<array> $results
*/
$results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
if (\array_key_exists($providerId, $providers)) {
$provider = $providers[$providerId];
} else {
$provider = $dbForProject->getDocument('providers', $providerId);
/**
* @var array<callable> $tasks
*/
$tasks = [];
if ($provider->isEmpty() || !$provider->getAttribute('enabled')) {
$provider = $default;
} else {
$providers[$providerId] = $provider;
}
}
foreach ($page as $providerId => $identifiers) {
$provider = $this->resolveProvider($dbForProject, $providerId, $providers, $default);
$resolvedProviderType = $provider->getAttribute('type');
$identifiersForProvider = $identifiers[$providerId];
$providerType = $provider->getAttribute('type');
$adapter = match ($providerType) {
$adapter = match ($resolvedProviderType) {
MESSAGE_TYPE_SMS => $this->getSmsAdapter($provider),
MESSAGE_TYPE_PUSH => $this->getPushAdapter($provider),
MESSAGE_TYPE_EMAIL => $this->getEmailAdapter($provider),
@@ -265,101 +196,66 @@ class Messaging extends Action
};
$batches = \array_chunk(
\array_keys($identifiersForProvider),
\array_keys($identifiers),
$adapter->getMaxMessagesPerRequest()
);
return batch(\array_map(function ($batch) use ($message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
return function () use ($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
$deliveredTotal = 0;
$deliveryErrors = [];
$messageData = clone $message;
$messageData->setAttribute('to', $batch);
foreach ($batches as $batch) {
$tasks[] = fn (): array => $semaphore->withLock(
fn (): array => $this->sendBatch(
$batch,
$message,
$provider,
$resolvedProviderType,
$adapter,
$dbForProject,
$deviceForFiles,
$project,
$publisherForUsage
)
);
}
}
$data = match ($providerType) {
MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider),
MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData),
MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project),
};
/**
* @var array<array{delivered: int, recipients: int, errors: array<string>}> $results
*/
$results = batch($tasks);
try {
$response = $adapter->send($data);
$deliveredTotal += (int) $response['deliveredTo'];
foreach ($response['results'] as $result) {
if ($result['status'] === 'failure') {
$deliveryErrors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}";
}
foreach ($results as $result) {
$deliveredTotal += $result['delivered'];
$failedTotal += $result['recipients'] - $result['delivered'];
// Deleting push targets when token has expired.
if (($result['error'] ?? '') === 'Expired device token') {
$target = $dbForProject->findOne('targets', [
Query::equal('identifier', [$result['recipient']])
]);
foreach ($result['errors'] as $error) {
if (\count($deliveryErrors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) {
break;
}
if (!$target->isEmpty()) {
$dbForProject->updateDocument(
'targets',
$target->getId(),
$target->setAttribute('expired', true)
);
}
}
}
} catch (\Throwable $e) {
$deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage();
} finally {
$errorTotal = \count($deliveryErrors);
$usage = new UsageContext();
$usage
->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal))
->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal)
->addMetric(METRIC_MESSAGES_FAILED, $errorTotal)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), ($deliveredTotal + $errorTotal))
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $deliveredTotal)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $errorTotal)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), ($deliveredTotal + $errorTotal))
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $deliveredTotal)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal);
$deliveryErrors[] = $error;
}
}
}
$publisherForUsage->enqueue(new Usage(
project: $project,
metrics: $usage->getMetrics(),
));
if (!$hasRecipients) {
$dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([
'status' => MessageStatus::FAILED,
'deliveryErrors' => ['No valid recipients found.']
]));
return [
'deliveredTotal' => $deliveredTotal,
'deliveryErrors' => $deliveryErrors,
];
}
};
}, $batches));
};
}, \array_keys($identifiers)));
$results = \array_merge(...$results);
$deliveredTotal = 0;
$deliveryErrors = [];
foreach ($results as $result) {
$deliveredTotal += $result['deliveredTotal'];
$deliveryErrors = \array_merge($deliveryErrors, $result['deliveryErrors']);
Span::add('message.skipped', 'no_valid_recipients');
return;
}
if (empty($deliveryErrors) && $deliveredTotal === 0) {
$deliveryErrors[] = 'Unknown error';
}
$hasFailures = $failedTotal > 0 || \count($deliveryErrors) > 0;
$message->setAttribute('status', $hasFailures ? MessageStatus::FAILED : MessageStatus::SENT);
$message->setAttribute('deliveryErrors', $deliveryErrors);
if (\count($message->getAttribute('deliveryErrors')) > 0) {
$message->setAttribute('status', MessageStatus::FAILED);
} else {
$message->setAttribute('status', MessageStatus::SENT);
}
Span::add('message.delivered_total', $deliveredTotal);
Span::add('message.errors_total', \count($deliveryErrors));
Span::add('message.errors_total', $failedTotal);
$message->removeAttribute('to');
@@ -410,6 +306,280 @@ class Messaging extends Action
}
}
/**
* Stream a message's recipients in bounded pages, grouped by provider and deduplicated by identifier within each page.
*
* Peak memory is O(MESSAGE_RECIPIENTS_PAGE_SIZE), never O(topic size): topics are walked through the
* subscribers collection with cursor pagination rather than reading the topic's `targets` attribute, which
* triggers the subQueryTopicTargets filter and loads up to APP_LIMIT_SUBSCRIBERS_SUBQUERY rows at once.
*
* @param array<string> $topicIds
* @param array<string> $userIds
* @param array<string> $targetIds
* @return \Generator<array<string, array<string, null>>>
* @throws \Exception
*/
private function streamRecipients(
Database $dbForProject,
array $topicIds,
array $userIds,
array $targetIds,
string $providerType,
Document $default
): \Generator {
if (\count($topicIds) > 0) {
$topics = $dbForProject->find('topics', [
Query::select(['$sequence']),
Query::equal('$id', $topicIds),
Query::limit(\count($topicIds)),
]);
foreach ($topics as $topic) {
$cursor = null;
do {
$queries = [
Query::equal('topicInternalId', [$topic->getSequence()]),
Query::equal('providerType', [$providerType]),
Query::select(['$sequence', 'targetInternalId']),
Query::orderAsc('$sequence'),
Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE),
];
if ($cursor !== null) {
$queries[] = Query::cursorAfter($cursor);
}
$subscribers = $dbForProject->getAuthorization()->skip(
fn () => $dbForProject->find('subscribers', $queries)
);
$count = \count($subscribers);
if ($count === 0) {
break;
}
$cursor = $subscribers[$count - 1];
$targetInternalIds = \array_map(
fn (Document $subscriber) => $subscriber->getAttribute('targetInternalId'),
$subscribers
);
$targets = $dbForProject->skipValidation(
fn () => $dbForProject->getAuthorization()->skip(
fn () => $dbForProject->find('targets', [
Query::equal('$sequence', $targetInternalIds),
Query::select(['providerId', 'identifier']),
Query::limit(\count($targetInternalIds)),
])
)
);
yield $this->groupTargetsByProvider($targets, $default);
} while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE);
}
}
if (\count($userIds) > 0) {
$cursor = null;
do {
$queries = [
Query::equal('userId', $userIds),
Query::equal('providerType', [$providerType]),
Query::select(['$sequence', 'providerId', 'identifier']),
Query::orderAsc('$sequence'),
Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE),
];
if ($cursor !== null) {
$queries[] = Query::cursorAfter($cursor);
}
$targets = $dbForProject->find('targets', $queries);
$count = \count($targets);
if ($count === 0) {
break;
}
$cursor = $targets[$count - 1];
yield $this->groupTargetsByProvider($targets, $default);
} while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE);
}
if (\count($targetIds) > 0) {
$cursor = null;
do {
$queries = [
Query::equal('$id', $targetIds),
Query::equal('providerType', [$providerType]),
Query::select(['$sequence', 'providerId', 'identifier']),
Query::orderAsc('$sequence'),
Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE),
];
if ($cursor !== null) {
$queries[] = Query::cursorAfter($cursor);
}
$targets = $dbForProject->find('targets', $queries);
$count = \count($targets);
if ($count === 0) {
break;
}
$cursor = $targets[$count - 1];
yield $this->groupTargetsByProvider($targets, $default);
} while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE);
}
}
/**
* Group a page of target documents by provider id, deduplicating identifiers within the page.
*
* @param array<Document> $targets
* @return array<string, array<string, null>>
*/
private function groupTargetsByProvider(array $targets, Document $default): array
{
/**
* @var array<string, array<string, null>> $identifiers
*/
$identifiers = [];
foreach ($targets as $target) {
$providerId = $target->getAttribute('providerId') ?: $default->getId();
if (!\array_key_exists($providerId, $identifiers)) {
$identifiers[$providerId] = [];
}
// Null values keep identifiers unique without a second lookup structure.
$identifiers[$providerId][$target->getAttribute('identifier')] = null;
}
return $identifiers;
}
/**
* Resolve and cache a provider for the lifetime of a send job, falling back to the default provider.
*
* @param array<string, Document> $providers
*/
private function resolveProvider(
Database $dbForProject,
string $providerId,
array &$providers,
Document $default
): Document {
if (\array_key_exists($providerId, $providers)) {
return $providers[$providerId];
}
$provider = $dbForProject->getDocument('providers', $providerId);
if ($provider->isEmpty() || !$provider->getAttribute('enabled')) {
return $default;
}
$providers[$providerId] = $provider;
return $provider;
}
/**
* Send a single adapter-sized batch and report delivery counts plus a bounded error list.
*
* @param array<string> $batch
* @return array{delivered: int, recipients: int, errors: array<string>}
*/
private function sendBatch(
array $batch,
Document $message,
Document $provider,
string $providerType,
EmailAdapter|SMSAdapter|PushAdapter $adapter,
Database $dbForProject,
Device $deviceForFiles,
Document $project,
UsagePublisher $publisherForUsage
): array {
$recipients = \count($batch);
$delivered = 0;
$errors = [];
$messageData = clone $message;
$messageData->setAttribute('to', $batch);
$data = match ($providerType) {
MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider),
MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData),
MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project),
default => throw new \Exception('Provider with the requested ID is of the incorrect type')
};
try {
$response = $adapter->send($data);
$delivered = (int) $response['deliveredTo'];
foreach ($response['results'] as $result) {
if ($result['status'] === 'failure' && \count($errors) < MESSAGE_DELIVERY_ERRORS_LIMIT) {
$errors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}";
}
// Deleting push targets when token has expired.
if (($result['error'] ?? '') === 'Expired device token') {
$target = $dbForProject->findOne('targets', [
Query::equal('identifier', [$result['recipient']])
]);
if (!$target->isEmpty()) {
$dbForProject->updateDocument(
'targets',
$target->getId(),
$target->setAttribute('expired', true)
);
}
}
}
} catch (\Throwable $e) {
// Whole-batch failure: record one representative error and count every recipient as failed.
$delivered = 0;
$errors = ['Failed sending to targets with error: ' . $e->getMessage()];
} finally {
$failed = $recipients - $delivered;
$usage = new UsageContext();
$usage
->addMetric(METRIC_MESSAGES, $recipients)
->addMetric(METRIC_MESSAGES_SENT, $delivered)
->addMetric(METRIC_MESSAGES_FAILED, $failed)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered)
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered)
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed);
$publisherForUsage->enqueue(new Usage(
project: $project,
metrics: $usage->getMetrics(),
));
}
return [
'delivered' => $delivered,
'recipients' => $recipients,
'errors' => $errors,
];
}
private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void
{
if ($this->adapter === null) {
@@ -451,7 +621,7 @@ class Messaging extends Action
}
private function getSmsAdapter(Document $provider): ?SMSAdapter
protected function getSmsAdapter(Document $provider): ?SMSAdapter
{
$credentials = $provider->getAttribute('credentials');
@@ -494,7 +664,7 @@ class Messaging extends Action
};
}
private function getPushAdapter(Document $provider): ?PushAdapter
protected function getPushAdapter(Document $provider): ?PushAdapter
{
$credentials = $provider->getAttribute('credentials');
$options = $provider->getAttribute('options');
@@ -513,7 +683,7 @@ class Messaging extends Action
};
}
private function getEmailAdapter(Document $provider): ?EmailAdapter
protected function getEmailAdapter(Document $provider): ?EmailAdapter
{
$credentials = $provider->getAttribute('credentials', []);
$options = $provider->getAttribute('options', []);
@@ -0,0 +1,528 @@
<?php
namespace Tests\Unit\Messaging;
use Appwrite\Event\Publisher\Usage as UsagePublisher;
use Appwrite\Messaging\Status as MessageStatus;
use Appwrite\Platform\Workers\Messaging;
use PHPUnit\Framework\TestCase;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Messaging\Adapter\Email as EmailAdapter;
use Utopia\Messaging\Messages\Email as EmailMessage;
use Utopia\Messaging\Response;
use Utopia\Storage\Device\Local;
use function Swoole\Coroutine\run;
require_once __DIR__ . '/SwooleCoroutinePolyfill.php';
/**
* Email adapter double that records peak in-flight {@see process()} calls and applies a configurable
* per-recipient failure predicate. It cooperatively yields inside the send so fibers scheduled by the
* polyfilled (or real) `batch()` interleave, making concurrent sends observable.
*/
class ConcurrencyProbeAdapter extends EmailAdapter
{
public int $peakConcurrency = 0;
public int $sendCalls = 0;
private int $inFlight = 0;
/**
* @param int $maxPerRequest
* @param (callable(string): bool)|null $failureFor Returns true when the given recipient should fail.
*/
public function __construct(
private readonly int $maxPerRequest = 1000,
private $failureFor = null
) {
}
public function getName(): string
{
return 'ConcurrencyProbe';
}
public function getMaxMessagesPerRequest(): int
{
return $this->maxPerRequest;
}
/**
* @param EmailMessage $message
* @return array{deliveredTo: int, type: string, results: array<array<string, mixed>>}
*/
protected function process(EmailMessage $message): array
{
$this->sendCalls++;
$this->inFlight++;
$this->peakConcurrency = \max($this->peakConcurrency, $this->inFlight);
// Yield so other coroutines run while this "request" is in flight.
\Swoole\Coroutine::sleep(0.001);
$response = new Response($this->getType());
$delivered = 0;
foreach ($message->getTo() as $recipient) {
// Email recipients are normalized to ['email' => ..., 'name' => ...] arrays by the message.
$email = $recipient['email'];
$fails = \is_callable($this->failureFor) && ($this->failureFor)($email);
if ($fails) {
$response->addResult($email, 'Simulated delivery failure');
} else {
$delivered++;
$response->addResult($email);
}
}
$response->setDeliveredTo($delivered);
$this->inFlight--;
return $response->toArray();
}
}
/**
* Worker subclass that injects the instrumented email adapter instead of a network-backed provider adapter.
*/
class TestableMessaging extends Messaging
{
public function __construct(private readonly EmailAdapter $emailAdapter)
{
}
protected function getEmailAdapter(Document $provider): ?EmailAdapter
{
return $this->emailAdapter;
}
}
/**
* In-memory {@see Database} double. Serves subscribers/targets through cursor pagination and records every
* query issued so the test can assert the streaming contract (cursorAfter + bounded page size, and that the
* 1M-row subQueryTopicTargets filter is never used).
*/
class RecordingDatabase extends Database
{
/**
* @var array<array{collection: string, queries: array<Query>}>
*/
public array $findCalls = [];
public ?Document $updatedMessage = null;
private readonly Authorization $authorization;
/**
* @param array<Document> $subscribers Subscriber rows (carry $sequence + targetInternalId + providerType).
* @param array<string, Document> $targets Target rows keyed by $sequence.
* @param array<Document> $directTargets Targets addressed directly by id or userId.
* @param Document $provider The single enabled provider.
*/
public function __construct(
private readonly array $subscribers = [],
private readonly array $targets = [],
private readonly array $directTargets = [],
private readonly ?Document $provider = null
) {
$this->authorization = new Authorization();
}
public function getAuthorization(): Authorization
{
return $this->authorization;
}
public function skipValidation(callable $callback): mixed
{
return $callback();
}
public function findOne(string $collection, array $queries = []): Document
{
if ($collection === 'providers') {
return $this->provider ?? new Document();
}
return new Document();
}
public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): Document
{
return new Document();
}
public function updateDocument(string $collection, string $id, Document $document): Document
{
if ($collection === 'messages') {
$this->updatedMessage = $document;
}
return $document;
}
public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array
{
$this->findCalls[] = ['collection' => $collection, 'queries' => $queries];
return match ($collection) {
'topics' => [new Document(['$id' => 'topic1', '$sequence' => '100'])],
'subscribers' => $this->paginate($this->subscribers, $queries),
'targets' => $this->resolveTargets($queries),
default => [],
};
}
/**
* @param array<Document> $rows
* @param array<Query> $queries
* @return array<Document>
*/
private function paginate(array $rows, array $queries): array
{
$limit = $this->limitOf($queries);
$cursor = $this->cursorOf($queries);
if ($cursor !== null) {
$rows = \array_values(\array_filter(
$rows,
fn (Document $row) => $row->getSequence() > $cursor->getSequence()
));
}
return \array_slice($rows, 0, $limit);
}
/**
* Either resolve a subscriber page's targets by $sequence, or paginate direct/user targets.
*
* @param array<Query> $queries
* @return array<Document>
*/
private function resolveTargets(array $queries): array
{
foreach ($queries as $query) {
if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === '$sequence') {
$sequences = $query->getValues();
return \array_values(\array_map(
fn ($sequence) => $this->targets[$sequence],
\array_filter($sequences, fn ($sequence) => isset($this->targets[$sequence]))
));
}
}
return $this->paginate($this->directTargets, $queries);
}
/**
* @param array<Query> $queries
*/
private function limitOf(array $queries): int
{
foreach ($queries as $query) {
if ($query->getMethod() === Query::TYPE_LIMIT) {
return (int) ($query->getValues()[0] ?? 0);
}
}
return 0;
}
/**
* @param array<Query> $queries
*/
private function cursorOf(array $queries): ?Document
{
foreach ($queries as $query) {
if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) {
$value = $query->getValues()[0] ?? null;
return $value instanceof Document ? $value : null;
}
}
return null;
}
}
class MessagingFanoutTest extends TestCase
{
private const RECIPIENT_COUNT = 4500;
private function provider(): Document
{
return new Document([
'$id' => 'provider1',
'$sequence' => '1',
'enabled' => true,
'type' => MESSAGE_TYPE_EMAIL,
'provider' => 'mock',
'name' => 'Mock Email',
'credentials' => [],
'options' => [
'fromName' => 'Appwrite',
'fromEmail' => 'noreply@example.com',
],
]);
}
private function message(): Document
{
return new Document([
'$id' => 'message1',
'$sequence' => '1',
'providerType' => MESSAGE_TYPE_EMAIL,
'topics' => ['topic1'],
'targets' => [],
'users' => [],
'status' => MessageStatus::PROCESSING,
'search' => 'message1',
'data' => [
'subject' => 'Hello',
'content' => 'World',
'html' => false,
],
]);
}
/**
* Build a topic-backed dataset: RECIPIENT_COUNT subscribers, each pointing at one email target.
*
* @return array{subscribers: array<Document>, targets: array<string, Document>}
*/
private function topicDataset(int $count): array
{
$subscribers = [];
$targets = [];
for ($i = 1; $i <= $count; $i++) {
$sequence = (string) $i;
$subscribers[] = new Document([
'$id' => 'sub' . $i,
'$sequence' => $sequence,
'topicInternalId' => '100',
'targetInternalId' => $sequence,
'providerType' => MESSAGE_TYPE_EMAIL,
]);
$targets[$sequence] = new Document([
'$id' => 'target' . $i,
'$sequence' => $sequence,
'providerId' => null,
'identifier' => "user{$i}@example.com",
'providerType' => MESSAGE_TYPE_EMAIL,
]);
}
return ['subscribers' => $subscribers, 'targets' => $targets];
}
private function sendTopic(
RecordingDatabase $database,
ConcurrencyProbeAdapter $adapter
): void {
$worker = new TestableMessaging($adapter);
$method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage');
$publisher = $this->createMock(UsagePublisher::class);
$publisher->method('enqueue')->willReturn(true);
$device = new Local(\sys_get_temp_dir());
$project = new Document(['$id' => 'project1', '$sequence' => '1']);
run(function () use ($worker, $method, $database, $device, $project, $publisher) {
$method->invoke($worker, $database, $this->message(), $device, $project, $publisher);
});
}
public function testRecipientsArePaginatedWithoutSubqueryFilter(): void
{
$data = $this->topicDataset(self::RECIPIENT_COUNT);
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
$adapter = new ConcurrencyProbeAdapter();
$this->sendTopic($database, $adapter);
$topicReads = \array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics');
$subscriberReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'subscribers'));
// The topic's `targets` attribute (the subQueryTopicTargets filter) is never materialized: topics are
// only read with a $sequence projection, and recipients come from the subscribers collection instead.
$this->assertNotEmpty($topicReads);
foreach ($topicReads as $call) {
$selected = $this->selectFields($call['queries']);
$this->assertNotContains('targets', $selected, 'Topic read must not select the targets subquery attribute');
}
// Subscribers are walked in bounded pages via cursorAfter. 4500 recipients over a 1000 page size is
// five pages; every page after the first carries a cursor and none exceeds the page size.
$expectedPages = (int) \ceil(self::RECIPIENT_COUNT / MESSAGE_RECIPIENTS_PAGE_SIZE);
$this->assertCount($expectedPages, $subscriberReads);
foreach ($subscriberReads as $index => $call) {
$this->assertSame(MESSAGE_RECIPIENTS_PAGE_SIZE, $this->limitOf($call['queries']));
if ($index === 0) {
$this->assertFalse($this->hasCursor($call['queries']), 'First page must not carry a cursor');
} else {
$this->assertTrue($this->hasCursor($call['queries']), 'Subsequent pages must paginate with cursorAfter');
}
}
}
public function testConcurrentSendsNeverExceedConfiguredLimit(): void
{
// One recipient per batch (maxPerRequest = 1) maximises the number of concurrent in-flight sends so
// the Semaphore's bound is actually exercised: 1500 batches contend for MESSAGE_SEND_CONCURRENCY permits.
$count = 1500;
$data = $this->topicDataset($count);
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
$adapter = new ConcurrencyProbeAdapter(maxPerRequest: 1);
$this->sendTopic($database, $adapter);
$this->assertSame($count, $adapter->sendCalls, 'Every recipient is sent exactly once');
$this->assertGreaterThan(1, $adapter->peakConcurrency, 'Sends should actually run concurrently');
$this->assertLessThanOrEqual(
MESSAGE_SEND_CONCURRENCY,
$adapter->peakConcurrency,
'Concurrent sends must never exceed MESSAGE_SEND_CONCURRENCY'
);
}
public function testDeliveryErrorsAreCappedWhileFailedCountIsComplete(): void
{
$count = self::RECIPIENT_COUNT;
$data = $this->topicDataset($count);
// Every recipient fails: 4500 individual failures, but only MESSAGE_DELIVERY_ERRORS_LIMIT are retained.
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
$adapter = new ConcurrencyProbeAdapter(failureFor: fn (string $recipient) => true);
$this->sendTopic($database, $adapter);
$message = $database->updatedMessage;
$this->assertNotNull($message);
$errors = $message->getAttribute('deliveryErrors');
$this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $errors, 'Delivery errors must be capped');
// Failed recipients are fully accounted for even though only 100 error strings are kept: every one of
// the 4500 recipients failed, so deliveredTotal is zero and the message is marked FAILED.
$this->assertSame(0, $message->getAttribute('deliveredTotal'));
$this->assertSame(MessageStatus::FAILED, $message->getAttribute('status'));
}
public function testDeliveredTotalAndStatusReflectPartialSuccess(): void
{
$count = self::RECIPIENT_COUNT;
$data = $this->topicDataset($count);
// Fail exactly the recipients whose local-part index is divisible by 10.
$failing = static function (string $recipient): bool {
\preg_match('/user(\d+)@/', $recipient, $matches);
return isset($matches[1]) && ((int) $matches[1]) % 10 === 0;
};
$expectedFailures = \intdiv($count, 10);
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
$adapter = new ConcurrencyProbeAdapter(failureFor: $failing);
$this->sendTopic($database, $adapter);
$message = $database->updatedMessage;
$this->assertNotNull($message);
// deliveredTotal is exact across all pages; failures partition the remainder exactly.
$this->assertSame($count - $expectedFailures, $message->getAttribute('deliveredTotal'));
$this->assertSame(MessageStatus::FAILED, $message->getAttribute('status'));
$this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $message->getAttribute('deliveryErrors'));
}
public function testFullSuccessMarksMessageSent(): void
{
$data = $this->topicDataset(self::RECIPIENT_COUNT);
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
$adapter = new ConcurrencyProbeAdapter();
$this->sendTopic($database, $adapter);
$message = $database->updatedMessage;
$this->assertNotNull($message);
$this->assertSame(self::RECIPIENT_COUNT, $message->getAttribute('deliveredTotal'));
$this->assertSame(MessageStatus::SENT, $message->getAttribute('status'));
$this->assertEmpty($message->getAttribute('deliveryErrors'));
}
public function testTerminalStatusIsSkipped(): void
{
$data = $this->topicDataset(10);
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
$adapter = new ConcurrencyProbeAdapter();
$worker = new TestableMessaging($adapter);
$method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage');
$publisher = $this->createMock(UsagePublisher::class);
$device = new Local(\sys_get_temp_dir());
$project = new Document(['$id' => 'project1', '$sequence' => '1']);
$message = $this->message();
$message->setAttribute('status', MessageStatus::SENT);
run(function () use ($worker, $method, $database, $device, $project, $publisher, $message) {
$method->invoke($worker, $database, $message, $device, $project, $publisher);
});
$this->assertSame(0, $adapter->sendCalls, 'Already-sent messages must not be reprocessed');
$this->assertNull($database->updatedMessage, 'Terminal messages must not be rewritten');
}
/**
* @param array<Query> $queries
* @return array<string>
*/
private function selectFields(array $queries): array
{
foreach ($queries as $query) {
if ($query->getMethod() === Query::TYPE_SELECT) {
return $query->getValues();
}
}
return [];
}
/**
* @param array<Query> $queries
*/
private function limitOf(array $queries): int
{
foreach ($queries as $query) {
if ($query->getMethod() === Query::TYPE_LIMIT) {
return (int) ($query->getValues()[0] ?? 0);
}
}
return 0;
}
/**
* @param array<Query> $queries
*/
private function hasCursor(array $queries): bool
{
foreach ($queries as $query) {
if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) {
return true;
}
}
return false;
}
}
@@ -0,0 +1,160 @@
<?php
/**
* Fiber-backed polyfill for the small slice of the Swoole coroutine API the messaging worker uses
* (`Swoole\Coroutine\batch`, `Swoole\Coroutine\run`, `Swoole\Coroutine\Channel`, `Swoole\Coroutine::sleep`).
*
* Swoole 6.x cannot be compiled against the PHP 8.5 ZTS build used in this environment, so the worker's
* concurrency primitives are exercised here with cooperative fibers instead. Every definition is guarded by
* `extension_loaded('swoole')`, so this file is completely inert when the real extension is present (CI).
*
* The scheduler models Swoole's blocking-acquire semantics faithfully: when a task throws
* {@see \Utopia\Lock\Exception\Contention} (the synchronous Semaphore path's signal that all permits are
* held), the task is suspended and retried on a later tick — exactly as a blocking `acquire` would wait for a
* permit. Because `Semaphore::withLock` acquires before invoking its callback, a retried task never
* double-executes its body, which the test suite asserts.
*/
namespace Swoole {
if (! \extension_loaded('swoole')) {
class Coroutine
{
public static ?\Fiber $current = null;
public static function getCid(): int
{
return self::$current !== null ? 1 : -1;
}
public static function sleep(float $seconds): void
{
if (self::$current !== null) {
\Fiber::suspend();
}
}
}
}
}
namespace Swoole\Coroutine {
if (! \extension_loaded('swoole')) {
/**
* Minimal coroutine channel. Used by {@see \Utopia\Lock\Semaphore} as its permit store.
*/
class Channel
{
/**
* @var array<mixed>
*/
private array $buffer = [];
public function __construct(private readonly int $capacity = 1)
{
}
public function push(mixed $data, float $timeout = 0): bool
{
$this->buffer[] = $data;
return true;
}
public function pop(float $timeout = 0): mixed
{
return \array_shift($this->buffer);
}
public function isFull(): bool
{
return \count($this->buffer) >= $this->capacity;
}
public function isEmpty(): bool
{
return \count($this->buffer) === 0;
}
}
}
}
namespace Swoole\Coroutine {
use Swoole\Coroutine;
use Utopia\Lock\Exception\Contention;
if (! \function_exists('Swoole\Coroutine\run')) {
function run(callable $callback): void
{
$fiber = new \Fiber($callback);
$previous = Coroutine::$current;
Coroutine::$current = $fiber;
$fiber->start();
while (! $fiber->isTerminated()) {
$fiber->resume();
}
Coroutine::$current = $previous;
}
}
if (! \function_exists('Swoole\Coroutine\batch')) {
/**
* @param array<callable> $tasks
* @return array<mixed>
*/
function batch(array $tasks, float $timeout = -1): array
{
$results = [];
$done = [];
$fibers = [];
foreach ($tasks as $key => $task) {
$fibers[$key] = new \Fiber($task);
}
$previous = Coroutine::$current;
while (\count($done) < \count($tasks)) {
$progressed = false;
foreach ($fibers as $key => $fiber) {
if (isset($done[$key])) {
continue;
}
Coroutine::$current = $fiber;
try {
if (! $fiber->isStarted()) {
$fiber->start();
} elseif (! $fiber->isTerminated()) {
$fiber->resume();
}
// Any successful start/resume advances a fiber (toward a yield or termination), which
// counts as progress; fibers advance monotonically so this cannot spin forever.
$progressed = true;
if ($fiber->isTerminated()) {
$results[$key] = $fiber->getReturn();
$done[$key] = true;
}
} catch (Contention) {
// All permits held: model a blocking acquire by retrying the task on a later tick.
$fibers[$key] = new \Fiber($tasks[$key]);
$progressed = true;
}
}
if (! $progressed) {
break;
}
}
Coroutine::$current = $previous;
\ksort($results);
return $results;
}
}
}