mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
fix(messaging): stream large-topic sends and bound send concurrency
Large external sends (e.g. a 500K-recipient topic email) OOM'd the
messaging worker: sendExternalMessage read $topic->getAttribute('targets'),
triggering the subQueryTopicTargets filter that loads up to
APP_LIMIT_SUBSCRIBERS_SUBQUERY (1M) subscribers plus a giant targets IN()
into memory, then fanned every adapter-sized chunk out through a single
unbounded Swoole\Coroutine\batch() — spawning unbounded concurrent sends.
Rework the external send path into a single-job streaming model:
- Recipients are streamed in bounded pages (MESSAGE_RECIPIENTS_PAGE_SIZE)
via cursor pagination. Topics walk the subscribers collection by
topicInternalId and resolve each page's targets by $sequence, so the
topic's targets subquery is never materialized. Users/direct targets
paginate the targets collection. Peak memory is O(page), not O(topic).
- Send concurrency is bounded with Utopia\Lock\Semaphore
(MESSAGE_SEND_CONCURRENCY permits): each adapter batch send is wrapped in
$semaphore->withLock() and awaited per page with Swoole's native batch().
Excess coroutines block on acquire instead of all running at once.
- Results accumulate across pages: an exact deliveredTotal/failedTotal
partition (failed = recipients - delivered) and a delivery-errors list
capped at MESSAGE_DELIVERY_ERRORS_LIMIT, so a 500K send never collects
one error string per recipient.
- An idempotency guard skips messages already in a terminal status, so
queue redelivery does not reprocess a completed send.
Only existing message fields (status, deliveredTotal, deliveryErrors) are
written — no schema, migration, model, or queue-payload changes. Internal
SMS and other transactional paths are untouched.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
69b49c7228
commit
da061ba7d2
4 files changed
+1040
-178
No files matched your search
@@ -242,6 +242,10 @@ const RULE_STATUS_VERIFIED = 'verified';
|
||||
// Message types
|
||||
const MESSAGE_SEND_TYPE_INTERNAL = 'internal';
|
||||
const MESSAGE_SEND_TYPE_EXTERNAL = 'external';
|
||||
// External message fan-out tuning
|
||||
const MESSAGE_RECIPIENTS_PAGE_SIZE = 1000; // Recipients resolved from the database per page while streaming a send
|
||||
const MESSAGE_SEND_CONCURRENCY = 10; // Maximum adapter send requests running concurrently within a single send job
|
||||
const MESSAGE_DELIVERY_ERRORS_LIMIT = 100; // Maximum number of per-recipient delivery errors retained on a message
|
||||
// Mail Types
|
||||
const MAIL_TYPE_VERIFICATION = 'verification';
|
||||
const MAIL_TYPE_MAGIC_SESSION = 'magicSession';
|
||||
|
||||
@@ -16,6 +16,7 @@ use Utopia\Database\Document;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\DSN\DSN;
|
||||
use Utopia\Lock\Semaphore;
|
||||
use Utopia\Logger\Log;
|
||||
use Utopia\Messaging\Adapter\Email as EmailAdapter;
|
||||
use Utopia\Messaging\Adapter\Email\Mailgun;
|
||||
@@ -130,69 +131,19 @@ class Messaging extends Action
|
||||
Document $project,
|
||||
UsagePublisher $publisherForUsage
|
||||
): void {
|
||||
$status = $message->getAttribute('status');
|
||||
|
||||
// Idempotency guard: queue redelivery can hand us a message that already finished.
|
||||
if (\in_array($status, [MessageStatus::SENT, MessageStatus::FAILED], true)) {
|
||||
Span::add('message.skipped', 'already_processed');
|
||||
return;
|
||||
}
|
||||
|
||||
$topicIds = $message->getAttribute('topics', []);
|
||||
$targetIds = $message->getAttribute('targets', []);
|
||||
$userIds = $message->getAttribute('users', []);
|
||||
$providerType = $message->getAttribute('providerType');
|
||||
|
||||
/**
|
||||
* @var array<Document> $allTargets
|
||||
*/
|
||||
$allTargets = [];
|
||||
|
||||
if (\count($topicIds) > 0) {
|
||||
$topics = $dbForProject->find('topics', [
|
||||
Query::equal('$id', $topicIds),
|
||||
Query::limit(\count($topicIds)),
|
||||
]);
|
||||
foreach ($topics as $topic) {
|
||||
$targets = \array_filter($topic->getAttribute('targets'), function (Document $target) use ($providerType) {
|
||||
return $target->getAttribute('providerType') === $providerType;
|
||||
});
|
||||
|
||||
\array_push($allTargets, ...$targets);
|
||||
}
|
||||
}
|
||||
|
||||
if (\count($userIds) > 0) {
|
||||
$limit = 1000;
|
||||
$offset = 0;
|
||||
|
||||
do {
|
||||
$targets = $dbForProject->find('targets', [
|
||||
Query::equal('userId', $userIds),
|
||||
Query::select(['providerId', 'identifier']),
|
||||
Query::equal('providerType', [$providerType]),
|
||||
Query::limit($limit),
|
||||
Query::offset($offset),
|
||||
]);
|
||||
|
||||
\array_push($allTargets, ...$targets);
|
||||
$offset += \count($targets);
|
||||
} while (\count($targets) === $limit);
|
||||
}
|
||||
|
||||
if (\count($targetIds) > 0) {
|
||||
$targets = $dbForProject->find('targets', [
|
||||
Query::select(['providerId', 'identifier']),
|
||||
Query::equal('$id', $targetIds),
|
||||
Query::equal('providerType', [$providerType]),
|
||||
Query::limit(\count($targetIds)),
|
||||
]);
|
||||
|
||||
\array_push($allTargets, ...$targets);
|
||||
}
|
||||
|
||||
if (empty($allTargets)) {
|
||||
$dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([
|
||||
'status' => MessageStatus::FAILED,
|
||||
'deliveryErrors' => ['No valid recipients found.']
|
||||
]));
|
||||
|
||||
Span::add('message.skipped', 'no_valid_recipients');
|
||||
return;
|
||||
}
|
||||
|
||||
$default = $dbForProject->findOne('providers', [
|
||||
Query::equal('enabled', [true]),
|
||||
Query::equal('type', [$providerType]),
|
||||
@@ -209,55 +160,35 @@ class Messaging extends Action
|
||||
}
|
||||
|
||||
/**
|
||||
* @var array<string, array<string, null>> $identifiers
|
||||
*/
|
||||
$identifiers = [];
|
||||
|
||||
/**
|
||||
* @var array<Document> $providers
|
||||
* Resolved providers cached for the lifetime of this job, keyed by provider id.
|
||||
* Seeded with the default provider so most sends never touch the providers collection.
|
||||
*
|
||||
* @var array<string, Document> $providers
|
||||
*/
|
||||
$providers = [
|
||||
$default->getId() => $default
|
||||
$default->getId() => $default,
|
||||
];
|
||||
|
||||
foreach ($allTargets as $target) {
|
||||
$providerId = $target->getAttribute('providerId');
|
||||
$semaphore = new Semaphore(MESSAGE_SEND_CONCURRENCY);
|
||||
|
||||
if (!$providerId) {
|
||||
$providerId = $default->getId();
|
||||
}
|
||||
$deliveredTotal = 0;
|
||||
$failedTotal = 0;
|
||||
$deliveryErrors = [];
|
||||
$hasRecipients = false;
|
||||
|
||||
if ($providerId) {
|
||||
if (!\array_key_exists($providerId, $identifiers)) {
|
||||
$identifiers[$providerId] = [];
|
||||
}
|
||||
// Use null as value to avoid duplicate keys
|
||||
$identifiers[$providerId][$target->getAttribute('identifier')] = null;
|
||||
}
|
||||
}
|
||||
foreach ($this->streamRecipients($dbForProject, $topicIds, $userIds, $targetIds, $providerType, $default) as $page) {
|
||||
$hasRecipients = true;
|
||||
|
||||
/**
|
||||
* @var array<array> $results
|
||||
*/
|
||||
$results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
|
||||
return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
|
||||
if (\array_key_exists($providerId, $providers)) {
|
||||
$provider = $providers[$providerId];
|
||||
} else {
|
||||
$provider = $dbForProject->getDocument('providers', $providerId);
|
||||
/**
|
||||
* @var array<callable> $tasks
|
||||
*/
|
||||
$tasks = [];
|
||||
|
||||
if ($provider->isEmpty() || !$provider->getAttribute('enabled')) {
|
||||
$provider = $default;
|
||||
} else {
|
||||
$providers[$providerId] = $provider;
|
||||
}
|
||||
}
|
||||
foreach ($page as $providerId => $identifiers) {
|
||||
$provider = $this->resolveProvider($dbForProject, $providerId, $providers, $default);
|
||||
$resolvedProviderType = $provider->getAttribute('type');
|
||||
|
||||
$identifiersForProvider = $identifiers[$providerId];
|
||||
|
||||
$providerType = $provider->getAttribute('type');
|
||||
|
||||
$adapter = match ($providerType) {
|
||||
$adapter = match ($resolvedProviderType) {
|
||||
MESSAGE_TYPE_SMS => $this->getSmsAdapter($provider),
|
||||
MESSAGE_TYPE_PUSH => $this->getPushAdapter($provider),
|
||||
MESSAGE_TYPE_EMAIL => $this->getEmailAdapter($provider),
|
||||
@@ -265,101 +196,66 @@ class Messaging extends Action
|
||||
};
|
||||
|
||||
$batches = \array_chunk(
|
||||
\array_keys($identifiersForProvider),
|
||||
\array_keys($identifiers),
|
||||
$adapter->getMaxMessagesPerRequest()
|
||||
);
|
||||
|
||||
return batch(\array_map(function ($batch) use ($message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
|
||||
return function () use ($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) {
|
||||
$deliveredTotal = 0;
|
||||
$deliveryErrors = [];
|
||||
$messageData = clone $message;
|
||||
$messageData->setAttribute('to', $batch);
|
||||
foreach ($batches as $batch) {
|
||||
$tasks[] = fn (): array => $semaphore->withLock(
|
||||
fn (): array => $this->sendBatch(
|
||||
$batch,
|
||||
$message,
|
||||
$provider,
|
||||
$resolvedProviderType,
|
||||
$adapter,
|
||||
$dbForProject,
|
||||
$deviceForFiles,
|
||||
$project,
|
||||
$publisherForUsage
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
$data = match ($providerType) {
|
||||
MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider),
|
||||
MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData),
|
||||
MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project),
|
||||
};
|
||||
/**
|
||||
* @var array<array{delivered: int, recipients: int, errors: array<string>}> $results
|
||||
*/
|
||||
$results = batch($tasks);
|
||||
|
||||
try {
|
||||
$response = $adapter->send($data);
|
||||
$deliveredTotal += (int) $response['deliveredTo'];
|
||||
foreach ($response['results'] as $result) {
|
||||
if ($result['status'] === 'failure') {
|
||||
$deliveryErrors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}";
|
||||
}
|
||||
foreach ($results as $result) {
|
||||
$deliveredTotal += $result['delivered'];
|
||||
$failedTotal += $result['recipients'] - $result['delivered'];
|
||||
|
||||
// Deleting push targets when token has expired.
|
||||
if (($result['error'] ?? '') === 'Expired device token') {
|
||||
$target = $dbForProject->findOne('targets', [
|
||||
Query::equal('identifier', [$result['recipient']])
|
||||
]);
|
||||
foreach ($result['errors'] as $error) {
|
||||
if (\count($deliveryErrors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (!$target->isEmpty()) {
|
||||
$dbForProject->updateDocument(
|
||||
'targets',
|
||||
$target->getId(),
|
||||
$target->setAttribute('expired', true)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
$deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage();
|
||||
} finally {
|
||||
$errorTotal = \count($deliveryErrors);
|
||||
$usage = new UsageContext();
|
||||
$usage
|
||||
->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal))
|
||||
->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal)
|
||||
->addMetric(METRIC_MESSAGES_FAILED, $errorTotal)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), ($deliveredTotal + $errorTotal))
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $deliveredTotal)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $errorTotal)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), ($deliveredTotal + $errorTotal))
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $deliveredTotal)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal);
|
||||
$deliveryErrors[] = $error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$publisherForUsage->enqueue(new Usage(
|
||||
project: $project,
|
||||
metrics: $usage->getMetrics(),
|
||||
));
|
||||
if (!$hasRecipients) {
|
||||
$dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([
|
||||
'status' => MessageStatus::FAILED,
|
||||
'deliveryErrors' => ['No valid recipients found.']
|
||||
]));
|
||||
|
||||
return [
|
||||
'deliveredTotal' => $deliveredTotal,
|
||||
'deliveryErrors' => $deliveryErrors,
|
||||
];
|
||||
}
|
||||
};
|
||||
}, $batches));
|
||||
};
|
||||
}, \array_keys($identifiers)));
|
||||
|
||||
$results = \array_merge(...$results);
|
||||
|
||||
$deliveredTotal = 0;
|
||||
$deliveryErrors = [];
|
||||
|
||||
foreach ($results as $result) {
|
||||
$deliveredTotal += $result['deliveredTotal'];
|
||||
$deliveryErrors = \array_merge($deliveryErrors, $result['deliveryErrors']);
|
||||
Span::add('message.skipped', 'no_valid_recipients');
|
||||
return;
|
||||
}
|
||||
|
||||
if (empty($deliveryErrors) && $deliveredTotal === 0) {
|
||||
$deliveryErrors[] = 'Unknown error';
|
||||
}
|
||||
|
||||
$hasFailures = $failedTotal > 0 || \count($deliveryErrors) > 0;
|
||||
$message->setAttribute('status', $hasFailures ? MessageStatus::FAILED : MessageStatus::SENT);
|
||||
$message->setAttribute('deliveryErrors', $deliveryErrors);
|
||||
|
||||
if (\count($message->getAttribute('deliveryErrors')) > 0) {
|
||||
$message->setAttribute('status', MessageStatus::FAILED);
|
||||
} else {
|
||||
$message->setAttribute('status', MessageStatus::SENT);
|
||||
}
|
||||
|
||||
Span::add('message.delivered_total', $deliveredTotal);
|
||||
Span::add('message.errors_total', \count($deliveryErrors));
|
||||
Span::add('message.errors_total', $failedTotal);
|
||||
|
||||
$message->removeAttribute('to');
|
||||
|
||||
@@ -410,6 +306,280 @@ class Messaging extends Action
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream a message's recipients in bounded pages, grouped by provider and deduplicated by identifier within each page.
|
||||
*
|
||||
* Peak memory is O(MESSAGE_RECIPIENTS_PAGE_SIZE), never O(topic size): topics are walked through the
|
||||
* subscribers collection with cursor pagination rather than reading the topic's `targets` attribute, which
|
||||
* triggers the subQueryTopicTargets filter and loads up to APP_LIMIT_SUBSCRIBERS_SUBQUERY rows at once.
|
||||
*
|
||||
* @param array<string> $topicIds
|
||||
* @param array<string> $userIds
|
||||
* @param array<string> $targetIds
|
||||
* @return \Generator<array<string, array<string, null>>>
|
||||
* @throws \Exception
|
||||
*/
|
||||
private function streamRecipients(
|
||||
Database $dbForProject,
|
||||
array $topicIds,
|
||||
array $userIds,
|
||||
array $targetIds,
|
||||
string $providerType,
|
||||
Document $default
|
||||
): \Generator {
|
||||
if (\count($topicIds) > 0) {
|
||||
$topics = $dbForProject->find('topics', [
|
||||
Query::select(['$sequence']),
|
||||
Query::equal('$id', $topicIds),
|
||||
Query::limit(\count($topicIds)),
|
||||
]);
|
||||
|
||||
foreach ($topics as $topic) {
|
||||
$cursor = null;
|
||||
|
||||
do {
|
||||
$queries = [
|
||||
Query::equal('topicInternalId', [$topic->getSequence()]),
|
||||
Query::equal('providerType', [$providerType]),
|
||||
Query::select(['$sequence', 'targetInternalId']),
|
||||
Query::orderAsc('$sequence'),
|
||||
Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE),
|
||||
];
|
||||
|
||||
if ($cursor !== null) {
|
||||
$queries[] = Query::cursorAfter($cursor);
|
||||
}
|
||||
|
||||
$subscribers = $dbForProject->getAuthorization()->skip(
|
||||
fn () => $dbForProject->find('subscribers', $queries)
|
||||
);
|
||||
|
||||
$count = \count($subscribers);
|
||||
|
||||
if ($count === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
$cursor = $subscribers[$count - 1];
|
||||
|
||||
$targetInternalIds = \array_map(
|
||||
fn (Document $subscriber) => $subscriber->getAttribute('targetInternalId'),
|
||||
$subscribers
|
||||
);
|
||||
|
||||
$targets = $dbForProject->skipValidation(
|
||||
fn () => $dbForProject->getAuthorization()->skip(
|
||||
fn () => $dbForProject->find('targets', [
|
||||
Query::equal('$sequence', $targetInternalIds),
|
||||
Query::select(['providerId', 'identifier']),
|
||||
Query::limit(\count($targetInternalIds)),
|
||||
])
|
||||
)
|
||||
);
|
||||
|
||||
yield $this->groupTargetsByProvider($targets, $default);
|
||||
} while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE);
|
||||
}
|
||||
}
|
||||
|
||||
if (\count($userIds) > 0) {
|
||||
$cursor = null;
|
||||
|
||||
do {
|
||||
$queries = [
|
||||
Query::equal('userId', $userIds),
|
||||
Query::equal('providerType', [$providerType]),
|
||||
Query::select(['$sequence', 'providerId', 'identifier']),
|
||||
Query::orderAsc('$sequence'),
|
||||
Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE),
|
||||
];
|
||||
|
||||
if ($cursor !== null) {
|
||||
$queries[] = Query::cursorAfter($cursor);
|
||||
}
|
||||
|
||||
$targets = $dbForProject->find('targets', $queries);
|
||||
$count = \count($targets);
|
||||
|
||||
if ($count === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
$cursor = $targets[$count - 1];
|
||||
|
||||
yield $this->groupTargetsByProvider($targets, $default);
|
||||
} while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE);
|
||||
}
|
||||
|
||||
if (\count($targetIds) > 0) {
|
||||
$cursor = null;
|
||||
|
||||
do {
|
||||
$queries = [
|
||||
Query::equal('$id', $targetIds),
|
||||
Query::equal('providerType', [$providerType]),
|
||||
Query::select(['$sequence', 'providerId', 'identifier']),
|
||||
Query::orderAsc('$sequence'),
|
||||
Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE),
|
||||
];
|
||||
|
||||
if ($cursor !== null) {
|
||||
$queries[] = Query::cursorAfter($cursor);
|
||||
}
|
||||
|
||||
$targets = $dbForProject->find('targets', $queries);
|
||||
$count = \count($targets);
|
||||
|
||||
if ($count === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
$cursor = $targets[$count - 1];
|
||||
|
||||
yield $this->groupTargetsByProvider($targets, $default);
|
||||
} while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Group a page of target documents by provider id, deduplicating identifiers within the page.
|
||||
*
|
||||
* @param array<Document> $targets
|
||||
* @return array<string, array<string, null>>
|
||||
*/
|
||||
private function groupTargetsByProvider(array $targets, Document $default): array
|
||||
{
|
||||
/**
|
||||
* @var array<string, array<string, null>> $identifiers
|
||||
*/
|
||||
$identifiers = [];
|
||||
|
||||
foreach ($targets as $target) {
|
||||
$providerId = $target->getAttribute('providerId') ?: $default->getId();
|
||||
|
||||
if (!\array_key_exists($providerId, $identifiers)) {
|
||||
$identifiers[$providerId] = [];
|
||||
}
|
||||
|
||||
// Null values keep identifiers unique without a second lookup structure.
|
||||
$identifiers[$providerId][$target->getAttribute('identifier')] = null;
|
||||
}
|
||||
|
||||
return $identifiers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve and cache a provider for the lifetime of a send job, falling back to the default provider.
|
||||
*
|
||||
* @param array<string, Document> $providers
|
||||
*/
|
||||
private function resolveProvider(
|
||||
Database $dbForProject,
|
||||
string $providerId,
|
||||
array &$providers,
|
||||
Document $default
|
||||
): Document {
|
||||
if (\array_key_exists($providerId, $providers)) {
|
||||
return $providers[$providerId];
|
||||
}
|
||||
|
||||
$provider = $dbForProject->getDocument('providers', $providerId);
|
||||
|
||||
if ($provider->isEmpty() || !$provider->getAttribute('enabled')) {
|
||||
return $default;
|
||||
}
|
||||
|
||||
$providers[$providerId] = $provider;
|
||||
|
||||
return $provider;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a single adapter-sized batch and report delivery counts plus a bounded error list.
|
||||
*
|
||||
* @param array<string> $batch
|
||||
* @return array{delivered: int, recipients: int, errors: array<string>}
|
||||
*/
|
||||
private function sendBatch(
|
||||
array $batch,
|
||||
Document $message,
|
||||
Document $provider,
|
||||
string $providerType,
|
||||
EmailAdapter|SMSAdapter|PushAdapter $adapter,
|
||||
Database $dbForProject,
|
||||
Device $deviceForFiles,
|
||||
Document $project,
|
||||
UsagePublisher $publisherForUsage
|
||||
): array {
|
||||
$recipients = \count($batch);
|
||||
$delivered = 0;
|
||||
$errors = [];
|
||||
|
||||
$messageData = clone $message;
|
||||
$messageData->setAttribute('to', $batch);
|
||||
|
||||
$data = match ($providerType) {
|
||||
MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider),
|
||||
MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData),
|
||||
MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project),
|
||||
default => throw new \Exception('Provider with the requested ID is of the incorrect type')
|
||||
};
|
||||
|
||||
try {
|
||||
$response = $adapter->send($data);
|
||||
$delivered = (int) $response['deliveredTo'];
|
||||
|
||||
foreach ($response['results'] as $result) {
|
||||
if ($result['status'] === 'failure' && \count($errors) < MESSAGE_DELIVERY_ERRORS_LIMIT) {
|
||||
$errors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}";
|
||||
}
|
||||
|
||||
// Deleting push targets when token has expired.
|
||||
if (($result['error'] ?? '') === 'Expired device token') {
|
||||
$target = $dbForProject->findOne('targets', [
|
||||
Query::equal('identifier', [$result['recipient']])
|
||||
]);
|
||||
|
||||
if (!$target->isEmpty()) {
|
||||
$dbForProject->updateDocument(
|
||||
'targets',
|
||||
$target->getId(),
|
||||
$target->setAttribute('expired', true)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
// Whole-batch failure: record one representative error and count every recipient as failed.
|
||||
$delivered = 0;
|
||||
$errors = ['Failed sending to targets with error: ' . $e->getMessage()];
|
||||
} finally {
|
||||
$failed = $recipients - $delivered;
|
||||
$usage = new UsageContext();
|
||||
$usage
|
||||
->addMetric(METRIC_MESSAGES, $recipients)
|
||||
->addMetric(METRIC_MESSAGES_SENT, $delivered)
|
||||
->addMetric(METRIC_MESSAGES_FAILED, $failed)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered)
|
||||
->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered)
|
||||
->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed);
|
||||
|
||||
$publisherForUsage->enqueue(new Usage(
|
||||
project: $project,
|
||||
metrics: $usage->getMetrics(),
|
||||
));
|
||||
}
|
||||
|
||||
return [
|
||||
'delivered' => $delivered,
|
||||
'recipients' => $recipients,
|
||||
'errors' => $errors,
|
||||
];
|
||||
}
|
||||
|
||||
private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void
|
||||
{
|
||||
if ($this->adapter === null) {
|
||||
@@ -451,7 +621,7 @@ class Messaging extends Action
|
||||
}
|
||||
|
||||
|
||||
private function getSmsAdapter(Document $provider): ?SMSAdapter
|
||||
protected function getSmsAdapter(Document $provider): ?SMSAdapter
|
||||
{
|
||||
$credentials = $provider->getAttribute('credentials');
|
||||
|
||||
@@ -494,7 +664,7 @@ class Messaging extends Action
|
||||
};
|
||||
}
|
||||
|
||||
private function getPushAdapter(Document $provider): ?PushAdapter
|
||||
protected function getPushAdapter(Document $provider): ?PushAdapter
|
||||
{
|
||||
$credentials = $provider->getAttribute('credentials');
|
||||
$options = $provider->getAttribute('options');
|
||||
@@ -513,7 +683,7 @@ class Messaging extends Action
|
||||
};
|
||||
}
|
||||
|
||||
private function getEmailAdapter(Document $provider): ?EmailAdapter
|
||||
protected function getEmailAdapter(Document $provider): ?EmailAdapter
|
||||
{
|
||||
$credentials = $provider->getAttribute('credentials', []);
|
||||
$options = $provider->getAttribute('options', []);
|
||||
|
||||
@@ -0,0 +1,528 @@
|
||||
<?php
|
||||
|
||||
namespace Tests\Unit\Messaging;
|
||||
|
||||
use Appwrite\Event\Publisher\Usage as UsagePublisher;
|
||||
use Appwrite\Messaging\Status as MessageStatus;
|
||||
use Appwrite\Platform\Workers\Messaging;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Messaging\Adapter\Email as EmailAdapter;
|
||||
use Utopia\Messaging\Messages\Email as EmailMessage;
|
||||
use Utopia\Messaging\Response;
|
||||
use Utopia\Storage\Device\Local;
|
||||
|
||||
use function Swoole\Coroutine\run;
|
||||
|
||||
require_once __DIR__ . '/SwooleCoroutinePolyfill.php';
|
||||
|
||||
/**
|
||||
* Email adapter double that records peak in-flight {@see process()} calls and applies a configurable
|
||||
* per-recipient failure predicate. It cooperatively yields inside the send so fibers scheduled by the
|
||||
* polyfilled (or real) `batch()` interleave, making concurrent sends observable.
|
||||
*/
|
||||
class ConcurrencyProbeAdapter extends EmailAdapter
|
||||
{
|
||||
public int $peakConcurrency = 0;
|
||||
|
||||
public int $sendCalls = 0;
|
||||
|
||||
private int $inFlight = 0;
|
||||
|
||||
/**
|
||||
* @param int $maxPerRequest
|
||||
* @param (callable(string): bool)|null $failureFor Returns true when the given recipient should fail.
|
||||
*/
|
||||
public function __construct(
|
||||
private readonly int $maxPerRequest = 1000,
|
||||
private $failureFor = null
|
||||
) {
|
||||
}
|
||||
|
||||
public function getName(): string
|
||||
{
|
||||
return 'ConcurrencyProbe';
|
||||
}
|
||||
|
||||
public function getMaxMessagesPerRequest(): int
|
||||
{
|
||||
return $this->maxPerRequest;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param EmailMessage $message
|
||||
* @return array{deliveredTo: int, type: string, results: array<array<string, mixed>>}
|
||||
*/
|
||||
protected function process(EmailMessage $message): array
|
||||
{
|
||||
$this->sendCalls++;
|
||||
$this->inFlight++;
|
||||
$this->peakConcurrency = \max($this->peakConcurrency, $this->inFlight);
|
||||
|
||||
// Yield so other coroutines run while this "request" is in flight.
|
||||
\Swoole\Coroutine::sleep(0.001);
|
||||
|
||||
$response = new Response($this->getType());
|
||||
$delivered = 0;
|
||||
|
||||
foreach ($message->getTo() as $recipient) {
|
||||
// Email recipients are normalized to ['email' => ..., 'name' => ...] arrays by the message.
|
||||
$email = $recipient['email'];
|
||||
$fails = \is_callable($this->failureFor) && ($this->failureFor)($email);
|
||||
|
||||
if ($fails) {
|
||||
$response->addResult($email, 'Simulated delivery failure');
|
||||
} else {
|
||||
$delivered++;
|
||||
$response->addResult($email);
|
||||
}
|
||||
}
|
||||
|
||||
$response->setDeliveredTo($delivered);
|
||||
|
||||
$this->inFlight--;
|
||||
|
||||
return $response->toArray();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Worker subclass that injects the instrumented email adapter instead of a network-backed provider adapter.
|
||||
*/
|
||||
class TestableMessaging extends Messaging
|
||||
{
|
||||
public function __construct(private readonly EmailAdapter $emailAdapter)
|
||||
{
|
||||
}
|
||||
|
||||
protected function getEmailAdapter(Document $provider): ?EmailAdapter
|
||||
{
|
||||
return $this->emailAdapter;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* In-memory {@see Database} double. Serves subscribers/targets through cursor pagination and records every
|
||||
* query issued so the test can assert the streaming contract (cursorAfter + bounded page size, and that the
|
||||
* 1M-row subQueryTopicTargets filter is never used).
|
||||
*/
|
||||
class RecordingDatabase extends Database
|
||||
{
|
||||
/**
|
||||
* @var array<array{collection: string, queries: array<Query>}>
|
||||
*/
|
||||
public array $findCalls = [];
|
||||
|
||||
public ?Document $updatedMessage = null;
|
||||
|
||||
private readonly Authorization $authorization;
|
||||
|
||||
/**
|
||||
* @param array<Document> $subscribers Subscriber rows (carry $sequence + targetInternalId + providerType).
|
||||
* @param array<string, Document> $targets Target rows keyed by $sequence.
|
||||
* @param array<Document> $directTargets Targets addressed directly by id or userId.
|
||||
* @param Document $provider The single enabled provider.
|
||||
*/
|
||||
public function __construct(
|
||||
private readonly array $subscribers = [],
|
||||
private readonly array $targets = [],
|
||||
private readonly array $directTargets = [],
|
||||
private readonly ?Document $provider = null
|
||||
) {
|
||||
$this->authorization = new Authorization();
|
||||
}
|
||||
|
||||
public function getAuthorization(): Authorization
|
||||
{
|
||||
return $this->authorization;
|
||||
}
|
||||
|
||||
public function skipValidation(callable $callback): mixed
|
||||
{
|
||||
return $callback();
|
||||
}
|
||||
|
||||
public function findOne(string $collection, array $queries = []): Document
|
||||
{
|
||||
if ($collection === 'providers') {
|
||||
return $this->provider ?? new Document();
|
||||
}
|
||||
|
||||
return new Document();
|
||||
}
|
||||
|
||||
public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): Document
|
||||
{
|
||||
return new Document();
|
||||
}
|
||||
|
||||
public function updateDocument(string $collection, string $id, Document $document): Document
|
||||
{
|
||||
if ($collection === 'messages') {
|
||||
$this->updatedMessage = $document;
|
||||
}
|
||||
|
||||
return $document;
|
||||
}
|
||||
|
||||
public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array
|
||||
{
|
||||
$this->findCalls[] = ['collection' => $collection, 'queries' => $queries];
|
||||
|
||||
return match ($collection) {
|
||||
'topics' => [new Document(['$id' => 'topic1', '$sequence' => '100'])],
|
||||
'subscribers' => $this->paginate($this->subscribers, $queries),
|
||||
'targets' => $this->resolveTargets($queries),
|
||||
default => [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Document> $rows
|
||||
* @param array<Query> $queries
|
||||
* @return array<Document>
|
||||
*/
|
||||
private function paginate(array $rows, array $queries): array
|
||||
{
|
||||
$limit = $this->limitOf($queries);
|
||||
$cursor = $this->cursorOf($queries);
|
||||
|
||||
if ($cursor !== null) {
|
||||
$rows = \array_values(\array_filter(
|
||||
$rows,
|
||||
fn (Document $row) => $row->getSequence() > $cursor->getSequence()
|
||||
));
|
||||
}
|
||||
|
||||
return \array_slice($rows, 0, $limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Either resolve a subscriber page's targets by $sequence, or paginate direct/user targets.
|
||||
*
|
||||
* @param array<Query> $queries
|
||||
* @return array<Document>
|
||||
*/
|
||||
private function resolveTargets(array $queries): array
|
||||
{
|
||||
foreach ($queries as $query) {
|
||||
if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === '$sequence') {
|
||||
$sequences = $query->getValues();
|
||||
|
||||
return \array_values(\array_map(
|
||||
fn ($sequence) => $this->targets[$sequence],
|
||||
\array_filter($sequences, fn ($sequence) => isset($this->targets[$sequence]))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
return $this->paginate($this->directTargets, $queries);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Query> $queries
|
||||
*/
|
||||
private function limitOf(array $queries): int
|
||||
{
|
||||
foreach ($queries as $query) {
|
||||
if ($query->getMethod() === Query::TYPE_LIMIT) {
|
||||
return (int) ($query->getValues()[0] ?? 0);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Query> $queries
|
||||
*/
|
||||
private function cursorOf(array $queries): ?Document
|
||||
{
|
||||
foreach ($queries as $query) {
|
||||
if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) {
|
||||
$value = $query->getValues()[0] ?? null;
|
||||
|
||||
return $value instanceof Document ? $value : null;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
class MessagingFanoutTest extends TestCase
|
||||
{
|
||||
private const RECIPIENT_COUNT = 4500;
|
||||
|
||||
private function provider(): Document
|
||||
{
|
||||
return new Document([
|
||||
'$id' => 'provider1',
|
||||
'$sequence' => '1',
|
||||
'enabled' => true,
|
||||
'type' => MESSAGE_TYPE_EMAIL,
|
||||
'provider' => 'mock',
|
||||
'name' => 'Mock Email',
|
||||
'credentials' => [],
|
||||
'options' => [
|
||||
'fromName' => 'Appwrite',
|
||||
'fromEmail' => 'noreply@example.com',
|
||||
],
|
||||
]);
|
||||
}
|
||||
|
||||
private function message(): Document
|
||||
{
|
||||
return new Document([
|
||||
'$id' => 'message1',
|
||||
'$sequence' => '1',
|
||||
'providerType' => MESSAGE_TYPE_EMAIL,
|
||||
'topics' => ['topic1'],
|
||||
'targets' => [],
|
||||
'users' => [],
|
||||
'status' => MessageStatus::PROCESSING,
|
||||
'search' => 'message1',
|
||||
'data' => [
|
||||
'subject' => 'Hello',
|
||||
'content' => 'World',
|
||||
'html' => false,
|
||||
],
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a topic-backed dataset: RECIPIENT_COUNT subscribers, each pointing at one email target.
|
||||
*
|
||||
* @return array{subscribers: array<Document>, targets: array<string, Document>}
|
||||
*/
|
||||
private function topicDataset(int $count): array
|
||||
{
|
||||
$subscribers = [];
|
||||
$targets = [];
|
||||
|
||||
for ($i = 1; $i <= $count; $i++) {
|
||||
$sequence = (string) $i;
|
||||
$subscribers[] = new Document([
|
||||
'$id' => 'sub' . $i,
|
||||
'$sequence' => $sequence,
|
||||
'topicInternalId' => '100',
|
||||
'targetInternalId' => $sequence,
|
||||
'providerType' => MESSAGE_TYPE_EMAIL,
|
||||
]);
|
||||
$targets[$sequence] = new Document([
|
||||
'$id' => 'target' . $i,
|
||||
'$sequence' => $sequence,
|
||||
'providerId' => null,
|
||||
'identifier' => "user{$i}@example.com",
|
||||
'providerType' => MESSAGE_TYPE_EMAIL,
|
||||
]);
|
||||
}
|
||||
|
||||
return ['subscribers' => $subscribers, 'targets' => $targets];
|
||||
}
|
||||
|
||||
private function sendTopic(
|
||||
RecordingDatabase $database,
|
||||
ConcurrencyProbeAdapter $adapter
|
||||
): void {
|
||||
$worker = new TestableMessaging($adapter);
|
||||
$method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage');
|
||||
|
||||
$publisher = $this->createMock(UsagePublisher::class);
|
||||
$publisher->method('enqueue')->willReturn(true);
|
||||
|
||||
$device = new Local(\sys_get_temp_dir());
|
||||
$project = new Document(['$id' => 'project1', '$sequence' => '1']);
|
||||
|
||||
run(function () use ($worker, $method, $database, $device, $project, $publisher) {
|
||||
$method->invoke($worker, $database, $this->message(), $device, $project, $publisher);
|
||||
});
|
||||
}
|
||||
|
||||
public function testRecipientsArePaginatedWithoutSubqueryFilter(): void
|
||||
{
|
||||
$data = $this->topicDataset(self::RECIPIENT_COUNT);
|
||||
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
|
||||
$adapter = new ConcurrencyProbeAdapter();
|
||||
|
||||
$this->sendTopic($database, $adapter);
|
||||
|
||||
$topicReads = \array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics');
|
||||
$subscriberReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'subscribers'));
|
||||
|
||||
// The topic's `targets` attribute (the subQueryTopicTargets filter) is never materialized: topics are
|
||||
// only read with a $sequence projection, and recipients come from the subscribers collection instead.
|
||||
$this->assertNotEmpty($topicReads);
|
||||
foreach ($topicReads as $call) {
|
||||
$selected = $this->selectFields($call['queries']);
|
||||
$this->assertNotContains('targets', $selected, 'Topic read must not select the targets subquery attribute');
|
||||
}
|
||||
|
||||
// Subscribers are walked in bounded pages via cursorAfter. 4500 recipients over a 1000 page size is
|
||||
// five pages; every page after the first carries a cursor and none exceeds the page size.
|
||||
$expectedPages = (int) \ceil(self::RECIPIENT_COUNT / MESSAGE_RECIPIENTS_PAGE_SIZE);
|
||||
$this->assertCount($expectedPages, $subscriberReads);
|
||||
|
||||
foreach ($subscriberReads as $index => $call) {
|
||||
$this->assertSame(MESSAGE_RECIPIENTS_PAGE_SIZE, $this->limitOf($call['queries']));
|
||||
|
||||
if ($index === 0) {
|
||||
$this->assertFalse($this->hasCursor($call['queries']), 'First page must not carry a cursor');
|
||||
} else {
|
||||
$this->assertTrue($this->hasCursor($call['queries']), 'Subsequent pages must paginate with cursorAfter');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function testConcurrentSendsNeverExceedConfiguredLimit(): void
|
||||
{
|
||||
// One recipient per batch (maxPerRequest = 1) maximises the number of concurrent in-flight sends so
|
||||
// the Semaphore's bound is actually exercised: 1500 batches contend for MESSAGE_SEND_CONCURRENCY permits.
|
||||
$count = 1500;
|
||||
$data = $this->topicDataset($count);
|
||||
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
|
||||
$adapter = new ConcurrencyProbeAdapter(maxPerRequest: 1);
|
||||
|
||||
$this->sendTopic($database, $adapter);
|
||||
|
||||
$this->assertSame($count, $adapter->sendCalls, 'Every recipient is sent exactly once');
|
||||
$this->assertGreaterThan(1, $adapter->peakConcurrency, 'Sends should actually run concurrently');
|
||||
$this->assertLessThanOrEqual(
|
||||
MESSAGE_SEND_CONCURRENCY,
|
||||
$adapter->peakConcurrency,
|
||||
'Concurrent sends must never exceed MESSAGE_SEND_CONCURRENCY'
|
||||
);
|
||||
}
|
||||
|
||||
public function testDeliveryErrorsAreCappedWhileFailedCountIsComplete(): void
|
||||
{
|
||||
$count = self::RECIPIENT_COUNT;
|
||||
$data = $this->topicDataset($count);
|
||||
// Every recipient fails: 4500 individual failures, but only MESSAGE_DELIVERY_ERRORS_LIMIT are retained.
|
||||
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
|
||||
$adapter = new ConcurrencyProbeAdapter(failureFor: fn (string $recipient) => true);
|
||||
|
||||
$this->sendTopic($database, $adapter);
|
||||
|
||||
$message = $database->updatedMessage;
|
||||
$this->assertNotNull($message);
|
||||
|
||||
$errors = $message->getAttribute('deliveryErrors');
|
||||
$this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $errors, 'Delivery errors must be capped');
|
||||
|
||||
// Failed recipients are fully accounted for even though only 100 error strings are kept: every one of
|
||||
// the 4500 recipients failed, so deliveredTotal is zero and the message is marked FAILED.
|
||||
$this->assertSame(0, $message->getAttribute('deliveredTotal'));
|
||||
$this->assertSame(MessageStatus::FAILED, $message->getAttribute('status'));
|
||||
}
|
||||
|
||||
public function testDeliveredTotalAndStatusReflectPartialSuccess(): void
|
||||
{
|
||||
$count = self::RECIPIENT_COUNT;
|
||||
$data = $this->topicDataset($count);
|
||||
// Fail exactly the recipients whose local-part index is divisible by 10.
|
||||
$failing = static function (string $recipient): bool {
|
||||
\preg_match('/user(\d+)@/', $recipient, $matches);
|
||||
|
||||
return isset($matches[1]) && ((int) $matches[1]) % 10 === 0;
|
||||
};
|
||||
$expectedFailures = \intdiv($count, 10);
|
||||
|
||||
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
|
||||
$adapter = new ConcurrencyProbeAdapter(failureFor: $failing);
|
||||
|
||||
$this->sendTopic($database, $adapter);
|
||||
|
||||
$message = $database->updatedMessage;
|
||||
$this->assertNotNull($message);
|
||||
|
||||
// deliveredTotal is exact across all pages; failures partition the remainder exactly.
|
||||
$this->assertSame($count - $expectedFailures, $message->getAttribute('deliveredTotal'));
|
||||
$this->assertSame(MessageStatus::FAILED, $message->getAttribute('status'));
|
||||
$this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $message->getAttribute('deliveryErrors'));
|
||||
}
|
||||
|
||||
public function testFullSuccessMarksMessageSent(): void
|
||||
{
|
||||
$data = $this->topicDataset(self::RECIPIENT_COUNT);
|
||||
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
|
||||
$adapter = new ConcurrencyProbeAdapter();
|
||||
|
||||
$this->sendTopic($database, $adapter);
|
||||
|
||||
$message = $database->updatedMessage;
|
||||
$this->assertNotNull($message);
|
||||
$this->assertSame(self::RECIPIENT_COUNT, $message->getAttribute('deliveredTotal'));
|
||||
$this->assertSame(MessageStatus::SENT, $message->getAttribute('status'));
|
||||
$this->assertEmpty($message->getAttribute('deliveryErrors'));
|
||||
}
|
||||
|
||||
public function testTerminalStatusIsSkipped(): void
|
||||
{
|
||||
$data = $this->topicDataset(10);
|
||||
$database = new RecordingDatabase($data['subscribers'], $data['targets'], [], $this->provider());
|
||||
$adapter = new ConcurrencyProbeAdapter();
|
||||
|
||||
$worker = new TestableMessaging($adapter);
|
||||
$method = new \ReflectionMethod(Messaging::class, 'sendExternalMessage');
|
||||
$publisher = $this->createMock(UsagePublisher::class);
|
||||
$device = new Local(\sys_get_temp_dir());
|
||||
$project = new Document(['$id' => 'project1', '$sequence' => '1']);
|
||||
|
||||
$message = $this->message();
|
||||
$message->setAttribute('status', MessageStatus::SENT);
|
||||
|
||||
run(function () use ($worker, $method, $database, $device, $project, $publisher, $message) {
|
||||
$method->invoke($worker, $database, $message, $device, $project, $publisher);
|
||||
});
|
||||
|
||||
$this->assertSame(0, $adapter->sendCalls, 'Already-sent messages must not be reprocessed');
|
||||
$this->assertNull($database->updatedMessage, 'Terminal messages must not be rewritten');
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Query> $queries
|
||||
* @return array<string>
|
||||
*/
|
||||
private function selectFields(array $queries): array
|
||||
{
|
||||
foreach ($queries as $query) {
|
||||
if ($query->getMethod() === Query::TYPE_SELECT) {
|
||||
return $query->getValues();
|
||||
}
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Query> $queries
|
||||
*/
|
||||
private function limitOf(array $queries): int
|
||||
{
|
||||
foreach ($queries as $query) {
|
||||
if ($query->getMethod() === Query::TYPE_LIMIT) {
|
||||
return (int) ($query->getValues()[0] ?? 0);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Query> $queries
|
||||
*/
|
||||
private function hasCursor(array $queries): bool
|
||||
{
|
||||
foreach ($queries as $query) {
|
||||
if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
<?php
|
||||
|
||||
/**
|
||||
* Fiber-backed polyfill for the small slice of the Swoole coroutine API the messaging worker uses
|
||||
* (`Swoole\Coroutine\batch`, `Swoole\Coroutine\run`, `Swoole\Coroutine\Channel`, `Swoole\Coroutine::sleep`).
|
||||
*
|
||||
* Swoole 6.x cannot be compiled against the PHP 8.5 ZTS build used in this environment, so the worker's
|
||||
* concurrency primitives are exercised here with cooperative fibers instead. Every definition is guarded by
|
||||
* `extension_loaded('swoole')`, so this file is completely inert when the real extension is present (CI).
|
||||
*
|
||||
* The scheduler models Swoole's blocking-acquire semantics faithfully: when a task throws
|
||||
* {@see \Utopia\Lock\Exception\Contention} (the synchronous Semaphore path's signal that all permits are
|
||||
* held), the task is suspended and retried on a later tick — exactly as a blocking `acquire` would wait for a
|
||||
* permit. Because `Semaphore::withLock` acquires before invoking its callback, a retried task never
|
||||
* double-executes its body, which the test suite asserts.
|
||||
*/
|
||||
|
||||
namespace Swoole {
|
||||
if (! \extension_loaded('swoole')) {
|
||||
class Coroutine
|
||||
{
|
||||
public static ?\Fiber $current = null;
|
||||
|
||||
public static function getCid(): int
|
||||
{
|
||||
return self::$current !== null ? 1 : -1;
|
||||
}
|
||||
|
||||
public static function sleep(float $seconds): void
|
||||
{
|
||||
if (self::$current !== null) {
|
||||
\Fiber::suspend();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace Swoole\Coroutine {
|
||||
if (! \extension_loaded('swoole')) {
|
||||
/**
|
||||
* Minimal coroutine channel. Used by {@see \Utopia\Lock\Semaphore} as its permit store.
|
||||
*/
|
||||
class Channel
|
||||
{
|
||||
/**
|
||||
* @var array<mixed>
|
||||
*/
|
||||
private array $buffer = [];
|
||||
|
||||
public function __construct(private readonly int $capacity = 1)
|
||||
{
|
||||
}
|
||||
|
||||
public function push(mixed $data, float $timeout = 0): bool
|
||||
{
|
||||
$this->buffer[] = $data;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public function pop(float $timeout = 0): mixed
|
||||
{
|
||||
return \array_shift($this->buffer);
|
||||
}
|
||||
|
||||
public function isFull(): bool
|
||||
{
|
||||
return \count($this->buffer) >= $this->capacity;
|
||||
}
|
||||
|
||||
public function isEmpty(): bool
|
||||
{
|
||||
return \count($this->buffer) === 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace Swoole\Coroutine {
|
||||
use Swoole\Coroutine;
|
||||
use Utopia\Lock\Exception\Contention;
|
||||
|
||||
if (! \function_exists('Swoole\Coroutine\run')) {
|
||||
function run(callable $callback): void
|
||||
{
|
||||
$fiber = new \Fiber($callback);
|
||||
$previous = Coroutine::$current;
|
||||
Coroutine::$current = $fiber;
|
||||
$fiber->start();
|
||||
|
||||
while (! $fiber->isTerminated()) {
|
||||
$fiber->resume();
|
||||
}
|
||||
|
||||
Coroutine::$current = $previous;
|
||||
}
|
||||
}
|
||||
|
||||
if (! \function_exists('Swoole\Coroutine\batch')) {
|
||||
/**
|
||||
* @param array<callable> $tasks
|
||||
* @return array<mixed>
|
||||
*/
|
||||
function batch(array $tasks, float $timeout = -1): array
|
||||
{
|
||||
$results = [];
|
||||
$done = [];
|
||||
$fibers = [];
|
||||
|
||||
foreach ($tasks as $key => $task) {
|
||||
$fibers[$key] = new \Fiber($task);
|
||||
}
|
||||
|
||||
$previous = Coroutine::$current;
|
||||
|
||||
while (\count($done) < \count($tasks)) {
|
||||
$progressed = false;
|
||||
|
||||
foreach ($fibers as $key => $fiber) {
|
||||
if (isset($done[$key])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Coroutine::$current = $fiber;
|
||||
|
||||
try {
|
||||
if (! $fiber->isStarted()) {
|
||||
$fiber->start();
|
||||
} elseif (! $fiber->isTerminated()) {
|
||||
$fiber->resume();
|
||||
}
|
||||
|
||||
// Any successful start/resume advances a fiber (toward a yield or termination), which
|
||||
// counts as progress; fibers advance monotonically so this cannot spin forever.
|
||||
$progressed = true;
|
||||
|
||||
if ($fiber->isTerminated()) {
|
||||
$results[$key] = $fiber->getReturn();
|
||||
$done[$key] = true;
|
||||
}
|
||||
} catch (Contention) {
|
||||
// All permits held: model a blocking acquire by retrying the task on a later tick.
|
||||
$fibers[$key] = new \Fiber($tasks[$key]);
|
||||
$progressed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (! $progressed) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Coroutine::$current = $previous;
|
||||
\ksort($results);
|
||||
|
||||
return $results;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user