diff --git a/.env b/.env index aa7438f6bb..57deb8cc0a 100644 --- a/.env +++ b/.env @@ -150,3 +150,6 @@ _APP_POOL_ADAPTER=stack _APP_WORKER_SCREENSHOTS_ROUTER=http://appwrite _TESTS_OAUTH2_GITHUB_CLIENT_ID= _TESTS_OAUTH2_GITHUB_CLIENT_SECRET= + +_APP_WORKERS_NUM=1 +_APP_WORKER_MAX_COROUTINES=1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 343d22fdf2..c3278d76af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -406,7 +406,7 @@ jobs: e2e_service: name: Tests / E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service }} - runs-on: ${{ matrix.runner || format('runs-on={0}/runner=4cpu-linux-x64/volume=120g/spot=false', github.run_id) }} + runs-on: ${{ matrix.runner || format('runs-on={0}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel', github.run_id) }} needs: [build, matrix] permissions: contents: read @@ -447,11 +447,11 @@ jobs: ] include: - service: Databases - runner: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/volume=120g/spot=false + runner: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/volume=120g/spot=false/extras=otel paratest_processes: 3 timeout_minutes: 30 - service: TablesDB - runner: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/volume=120g/spot=false + runner: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/volume=120g/spot=false/extras=otel paratest_processes: 3 timeout_minutes: 30 - service: Migrations diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 68ab657213..a74afaf9f0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -33,7 +33,7 @@ jobs: id: meta uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: - images: appwrite/cloud + images: appwrite/ce tags: | type=ref,event=tag diff --git a/app/assets/dbip/dbip-country-lite-2025-12.mmdb b/app/assets/dbip/dbip-country-lite-2025-12.mmdb deleted file mode 100644 index 4ecabbf735..0000000000 Binary files a/app/assets/dbip/dbip-country-lite-2025-12.mmdb and /dev/null differ diff --git a/app/assets/dbip/dbip-country-lite-2026-06.mmdb b/app/assets/dbip/dbip-country-lite-2026-06.mmdb new file mode 100644 index 0000000000..df09a565b3 Binary files /dev/null and b/app/assets/dbip/dbip-country-lite-2026-06.mmdb differ diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 26f6bf21c4..8665fbf24c 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -236,6 +236,100 @@ Http::post('/v1/messaging/providers/sendgrid') ->dynamic($provider, Response::MODEL_PROVIDER); }); +Http::post('/v1/messaging/providers/ses') + ->desc('Create Amazon SES provider') + ->groups(['api', 'messaging']) + ->label('audits.event', 'provider.create') + ->label('audits.resource', 'provider/{response.$id}') + ->label('event', 'providers.[providerId].create') + ->label('scope', 'providers.write') + ->label('resourceType', RESOURCE_TYPE_PROVIDERS) + ->label('sdk', new Method( + namespace: 'messaging', + group: 'providers', + name: 'createSesProvider', + description: '/docs/references/messaging/create-ses-provider.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_CREATED, + model: Response::MODEL_PROVIDER, + ) + ] + )) + ->param('providerId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Provider ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', false, ['dbForProject']) + ->param('name', '', new Text(128), 'Provider name.') + ->param('accessKey', '', new Text(0), 'AWS access key ID.', true) + ->param('secretKey', '', new Text(0), 'AWS secret access key.', true) + ->param('region', '', new Text(128), 'AWS region, for example us-east-1.', true) + ->param('fromName', '', new Text(128, 0), 'Sender Name.', true) + ->param('fromEmail', '', new Email(), 'Sender email address.', true) + ->param('replyToName', '', new Text(128, 0), 'Name set in the reply to field for the mail. Default value is sender name.', true) + ->param('replyToEmail', '', new Email(), 'Email set in the reply to field for the mail. Default value is sender email.', true) + ->param('enabled', null, new Nullable(new Boolean()), 'Set as enabled.', true) + ->inject('queueForEvents') + ->inject('dbForProject') + ->inject('response') + ->action(function (string $providerId, string $name, string $accessKey, string $secretKey, string $region, string $fromName, string $fromEmail, string $replyToName, string $replyToEmail, ?bool $enabled, Event $queueForEvents, Database $dbForProject, Response $response) { + $providerId = $providerId == 'unique()' ? ID::unique() : $providerId; + + $credentials = []; + + if (!empty($accessKey)) { + $credentials['accessKey'] = $accessKey; + } + + if (!empty($secretKey)) { + $credentials['secretKey'] = $secretKey; + } + + if (!empty($region)) { + $credentials['region'] = $region; + } + + $options = [ + 'fromName' => $fromName, + 'fromEmail' => $fromEmail, + 'replyToName' => $replyToName, + 'replyToEmail' => $replyToEmail, + ]; + + if ( + $enabled === true + && !empty($fromEmail) + && !empty($credentials['accessKey']) + && !empty($credentials['secretKey']) + && !empty($credentials['region']) + ) { + $enabled = true; + } else { + $enabled = false; + } + + $provider = new Document([ + '$id' => $providerId, + 'name' => $name, + 'provider' => 'ses', + 'type' => MESSAGE_TYPE_EMAIL, + 'enabled' => $enabled, + 'credentials' => $credentials, + 'options' => $options, + ]); + + try { + $provider = $dbForProject->createDocument('providers', $provider); + } catch (DuplicateException) { + throw new Exception(Exception::PROVIDER_ALREADY_EXISTS); + } + + $queueForEvents + ->setParam('providerId', $provider->getId()); + + $response + ->setStatusCode(Response::STATUS_CODE_CREATED) + ->dynamic($provider, Response::MODEL_PROVIDER); + }); + Http::post('/v1/messaging/providers/resend') ->desc('Create Resend provider') ->groups(['api', 'messaging']) @@ -1458,6 +1552,119 @@ Http::patch('/v1/messaging/providers/sendgrid/:providerId') ->dynamic($provider, Response::MODEL_PROVIDER); }); +Http::patch('/v1/messaging/providers/ses/:providerId') + ->desc('Update Amazon SES provider') + ->groups(['api', 'messaging']) + ->label('audits.event', 'provider.update') + ->label('audits.resource', 'provider/{response.$id}') + ->label('event', 'providers.[providerId].update') + ->label('scope', 'providers.write') + ->label('resourceType', RESOURCE_TYPE_PROVIDERS) + ->label('sdk', new Method( + namespace: 'messaging', + group: 'providers', + name: 'updateSesProvider', + description: '/docs/references/messaging/update-ses-provider.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_PROVIDER, + ) + ] + )) + ->param('providerId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Provider ID.', false, ['dbForProject']) + ->param('name', '', new Text(128), 'Provider name.', true) + ->param('enabled', null, new Nullable(new Boolean()), 'Set as enabled.', true) + ->param('accessKey', '', new Text(0), 'AWS access key ID.', true) + ->param('secretKey', '', new Text(0), 'AWS secret access key.', true) + ->param('region', '', new Text(128), 'AWS region, for example us-east-1.', true) + ->param('fromName', '', new Text(128), 'Sender Name.', true) + ->param('fromEmail', '', new Email(), 'Sender email address.', true) + ->param('replyToName', '', new Text(128), 'Name set in the Reply To field for the mail. Default value is Sender Name.', true) + ->param('replyToEmail', '', new Text(128), 'Email set in the Reply To field for the mail. Default value is Sender Email.', true) + ->inject('queueForEvents') + ->inject('dbForProject') + ->inject('response') + ->action(function (string $providerId, string $name, ?bool $enabled, string $accessKey, string $secretKey, string $region, string $fromName, string $fromEmail, string $replyToName, string $replyToEmail, Event $queueForEvents, Database $dbForProject, Response $response) { + $provider = $dbForProject->getDocument('providers', $providerId); + + if ($provider->isEmpty()) { + throw new Exception(Exception::PROVIDER_NOT_FOUND); + } + + $providerAttr = $provider->getAttribute('provider'); + + if ($providerAttr !== 'ses') { + throw new Exception(Exception::PROVIDER_INCORRECT_TYPE); + } + + if (!empty($name)) { + $provider->setAttribute('name', $name); + } + + $options = $provider->getAttribute('options'); + + if (!empty($fromName)) { + $options['fromName'] = $fromName; + } + + if (!empty($fromEmail)) { + $options['fromEmail'] = $fromEmail; + } + + if (!empty($replyToName)) { + $options['replyToName'] = $replyToName; + } + + if (!empty($replyToEmail)) { + $options['replyToEmail'] = $replyToEmail; + } + + $provider->setAttribute('options', $options); + + $credentials = $provider->getAttribute('credentials'); + + if (!empty($accessKey)) { + $credentials['accessKey'] = $accessKey; + } + + if (!empty($secretKey)) { + $credentials['secretKey'] = $secretKey; + } + + if (!empty($region)) { + $credentials['region'] = $region; + } + + $provider->setAttribute('credentials', $credentials); + + if (!\is_null($enabled)) { + if ($enabled) { + if ( + !empty($credentials['accessKey']) && + !empty($credentials['secretKey']) && + !empty($credentials['region']) && + !empty($options['fromEmail']) + ) { + $provider->setAttribute('enabled', true); + } else { + throw new Exception(Exception::PROVIDER_MISSING_CREDENTIALS); + } + } else { + $provider->setAttribute('enabled', false); + } + } + + $provider = $dbForProject->updateDocument('providers', $provider->getId(), $provider); + + $queueForEvents + ->setParam('providerId', $provider->getId()); + + $response + ->dynamic($provider, Response::MODEL_PROVIDER); + }); + Http::patch('/v1/messaging/providers/resend/:providerId') ->desc('Update Resend provider') ->groups(['api', 'messaging']) diff --git a/app/init/constants.php b/app/init/constants.php index d426e51fb6..4fe7c418a9 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -242,6 +242,12 @@ const RULE_STATUS_VERIFIED = 'verified'; // Message types const MESSAGE_SEND_TYPE_INTERNAL = 'internal'; const MESSAGE_SEND_TYPE_EXTERNAL = 'external'; +// External message fan-out tuning +const MESSAGE_RECIPIENTS_PAGE_SIZE = 5000; // Recipients resolved from the database per page while streaming a send; aligned with APP_LIMIT_COUNT to minimise round-trips on large topics +const MESSAGE_SEND_CONCURRENCY = 10; // Maximum adapter send requests running concurrently within a single send job +const MESSAGE_DELIVERY_ERRORS_LIMIT = 100; // Maximum number of per-recipient delivery errors retained on a message +const MESSAGE_SEND_MAX_RETRIES = 5; // Maximum number of attempts to deliver a batch when the provider throttles or fails transiently +const MESSAGE_SEND_RETRY_DELAY = 1.0; // Base seconds for exponential backoff between batch send retries; overridable in tests for an instant suite // Mail Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; diff --git a/app/init/registers.php b/app/init/registers.php index 9ea19eee24..f47cc43e60 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -254,6 +254,10 @@ $register->set('pools', function () { $workerCount = intval(System::getEnv('_APP_CPU_NUM', swoole_cpu_num())) * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); $poolSize = max(1, (int)($instanceConnections / $workerCount)); + // Queue workers consume jobs concurrently with coroutines; each in-flight + // job may hold a connection, so pools must cover the coroutine count. + $poolSize = max($poolSize, (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1)); + foreach ($connections as $key => $connection) { $type = $connection['type']; $multiple = $connection['multiple']; @@ -340,9 +344,13 @@ $register->set('pools', function () { // PubSub workers hold one long-lived subscribed connection and also need // spare capacity for publishes from the same process. - $connectionPoolSize = $type === 'pubsub' - ? max(2, $poolSize) - : $poolSize; + // Consumer pools lease one connection per consume coroutine for the + // lifetime of the worker, plus headroom for queue-size reads. + $connectionPoolSize = match ($type) { + 'pubsub' => max(2, $poolSize), + 'consumer' => $poolSize + 2, + default => $poolSize, + }; $pool = new Pool($poolAdapter, $name, $connectionPoolSize, function () use ($type, $resource, $dsn) { // Get Adapter @@ -451,7 +459,7 @@ $register->set('smtp', function () { ); }); $register->set('geodb', function () { - return new Reader(__DIR__ . '/../assets/dbip/dbip-country-lite-2025-12.mmdb'); + return new Reader(__DIR__ . '/../assets/dbip/dbip-country-lite-2026-06.mmdb'); }); $register->set('passwordsDictionary', function () { $content = \file_get_contents(__DIR__ . '/../assets/security/10k-common-passwords'); diff --git a/app/worker.php b/app/worker.php index 169b8e9770..063d4746ac 100644 --- a/app/worker.php +++ b/app/worker.php @@ -83,17 +83,25 @@ if (\str_starts_with($workerName, 'databases')) { /** @var \Utopia\Pools\Group $pools */ $pools = $container->get('pools'); +// Each consume coroutine leases its own broker connection from the pool for +// the lifetime of its consume loop, so concurrent coroutines never share a +// connection. The publisher side backs the queue-depth telemetry gauge. $adapter = new Swoole( - $pools->get('consumer')->pop()->getResource(), + new BrokerPool( + publisher: $pools->get('consumer'), + consumer: $pools->get('consumer'), + ), System::getEnv('_APP_WORKERS_NUM', 1), - $queueName + $queueName, + maxCoroutines: (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1), + resources: $container, ); -$worker = new Server($adapter, $container); +$worker = new Server($adapter); try { $worker->init()->action(function () use ($worker, $registerWorkerMessageResources, $queueName) { - $registerWorkerMessageResources($worker->getContainer()); + $registerWorkerMessageResources($worker->context()); Span::init("worker.{$queueName}"); }); @@ -103,7 +111,7 @@ try { $container->set('bus', function ($register) use ($worker) { return $register->get('bus')->setResolver( - fn (string $name) => $worker->getContainer()->get($name) + fn (string $name) => $worker->context()->get($name) ); }, ['register']); diff --git a/composer.json b/composer.json index ad18771699..b78637ccd9 100644 --- a/composer.json +++ b/composer.json @@ -74,13 +74,13 @@ "utopia-php/locale": "0.8.*", "utopia-php/lock": "0.2.*", "utopia-php/logger": "0.8.*", - "utopia-php/messaging": "0.22.*", + "utopia-php/messaging": "1.0.*", "utopia-php/migration": "1.*", - "utopia-php/platform": "1.0.0-rc3", + "utopia-php/platform": "1.0.0-rc4", "utopia-php/pools": "1.*", "utopia-php/span": "1.1.*", "utopia-php/preloader": "0.2.*", - "utopia-php/queue": "0.18.*", + "utopia-php/queue": "0.19.*", "utopia-php/servers": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/storage": "2.*", diff --git a/composer.lock b/composer.lock index 70c843efe4..0a15ec10e2 100644 --- a/composer.lock +++ b/composer.lock @@ -4,20 +4,20 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ec80b1465e57bc223903fa102249f216", + "content-hash": "0c23c30bba86ad84377d4d1abce0992a", "packages": [ { "name": "adhocore/jwt", - "version": "1.1.3", + "version": "v1.1.4", "source": { "type": "git", "url": "https://github.com/adhocore/php-jwt.git", - "reference": "ad417603d9d45578b6af2089ad5b78f101c82367" + "reference": "b7055ae9024a6627f19f9873b5db19b0f1dcc4e7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/adhocore/php-jwt/zipball/ad417603d9d45578b6af2089ad5b78f101c82367", - "reference": "ad417603d9d45578b6af2089ad5b78f101c82367", + "url": "https://api.github.com/repos/adhocore/php-jwt/zipball/b7055ae9024a6627f19f9873b5db19b0f1dcc4e7", + "reference": "b7055ae9024a6627f19f9873b5db19b0f1dcc4e7", "shasum": "" }, "require": { @@ -53,7 +53,7 @@ ], "support": { "issues": "https://github.com/adhocore/php-jwt/issues", - "source": "https://github.com/adhocore/php-jwt/tree/1.1.3" + "source": "https://github.com/adhocore/php-jwt/tree/v1.1.4" }, "funding": [ { @@ -65,7 +65,7 @@ "type": "github" } ], - "time": "2025-02-18T01:00:50+00:00" + "time": "2026-05-30T07:58:15+00:00" }, { "name": "appwrite/appwrite", @@ -2571,16 +2571,16 @@ }, { "name": "spomky-labs/otphp", - "version": "11.4.2", + "version": "11.4.3", "source": { "type": "git", "url": "https://github.com/Spomky-Labs/otphp.git", - "reference": "2a1b503fd1c1a5c751ab3c5cd37f2d2d26ab74ad" + "reference": "46663316285bf1f001a31584213b327494b00ae7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Spomky-Labs/otphp/zipball/2a1b503fd1c1a5c751ab3c5cd37f2d2d26ab74ad", - "reference": "2a1b503fd1c1a5c751ab3c5cd37f2d2d26ab74ad", + "url": "https://api.github.com/repos/Spomky-Labs/otphp/zipball/46663316285bf1f001a31584213b327494b00ae7", + "reference": "46663316285bf1f001a31584213b327494b00ae7", "shasum": "" }, "require": { @@ -2625,7 +2625,7 @@ ], "support": { "issues": "https://github.com/Spomky-Labs/otphp/issues", - "source": "https://github.com/Spomky-Labs/otphp/tree/11.4.2" + "source": "https://github.com/Spomky-Labs/otphp/tree/11.4.3" }, "funding": [ { @@ -2637,7 +2637,7 @@ "type": "patreon" } ], - "time": "2026-01-23T10:53:01+00:00" + "time": "2026-05-31T12:42:43+00:00" }, { "name": "symfony/deprecation-contracts", @@ -3140,16 +3140,16 @@ }, { "name": "symfony/polyfill-php85", - "version": "v1.37.0", + "version": "v1.38.1", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php85.git", - "reference": "fcfa4973a9917cef23f2e38774da74a2b7d115ee" + "reference": "ba2ba04f3352cfa2dcbbcb90aee13ed967f505b1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/fcfa4973a9917cef23f2e38774da74a2b7d115ee", - "reference": "fcfa4973a9917cef23f2e38774da74a2b7d115ee", + "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/ba2ba04f3352cfa2dcbbcb90aee13ed967f505b1", + "reference": "ba2ba04f3352cfa2dcbbcb90aee13ed967f505b1", "shasum": "" }, "require": { @@ -3196,7 +3196,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php85/tree/v1.37.0" + "source": "https://github.com/symfony/polyfill-php85/tree/v1.38.1" }, "funding": [ { @@ -3216,7 +3216,7 @@ "type": "tidelift" } ], - "time": "2026-04-26T13:10:57+00:00" + "time": "2026-05-26T02:25:22+00:00" }, { "name": "symfony/service-contracts", @@ -3510,16 +3510,16 @@ }, { "name": "utopia-php/audit", - "version": "2.4.1", + "version": "2.4.2", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "eddd79d93f23ed2851c0df2b1e2e2dfb25ba06c6" + "reference": "6f15caaac5455c98cec06e02fe8da3af9477ff38" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/eddd79d93f23ed2851c0df2b1e2e2dfb25ba06c6", - "reference": "eddd79d93f23ed2851c0df2b1e2e2dfb25ba06c6", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/6f15caaac5455c98cec06e02fe8da3af9477ff38", + "reference": "6f15caaac5455c98cec06e02fe8da3af9477ff38", "shasum": "" }, "require": { @@ -3554,26 +3554,27 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/2.4.1" + "source": "https://github.com/utopia-php/audit/tree/2.4.2" }, - "time": "2026-05-20T06:25:45+00:00" + "time": "2026-06-03T03:42:22+00:00" }, { "name": "utopia-php/auth", - "version": "0.5.0", + "version": "0.5.1", "source": { "type": "git", "url": "https://github.com/utopia-php/auth.git", - "reference": "5ad0ded3a79f153ee904b97b49f8dfe4669e4fd0" + "reference": "9245a6342ed94f00425aef6963a32b8af9b0dc9d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/auth/zipball/5ad0ded3a79f153ee904b97b49f8dfe4669e4fd0", - "reference": "5ad0ded3a79f153ee904b97b49f8dfe4669e4fd0", + "url": "https://api.github.com/repos/utopia-php/auth/zipball/9245a6342ed94f00425aef6963a32b8af9b0dc9d", + "reference": "9245a6342ed94f00425aef6963a32b8af9b0dc9d", "shasum": "" }, "require": { "ext-hash": "*", + "ext-openssl": "*", "ext-scrypt": "*", "ext-sodium": "*", "php": ">=8.0" @@ -3609,9 +3610,9 @@ ], "support": { "issues": "https://github.com/utopia-php/auth/issues", - "source": "https://github.com/utopia-php/auth/tree/0.5.0" + "source": "https://github.com/utopia-php/auth/tree/0.5.1" }, - "time": "2025-10-29T07:11:43+00:00" + "time": "2026-05-30T12:21:17+00:00" }, { "name": "utopia-php/cache", @@ -4606,16 +4607,16 @@ }, { "name": "utopia-php/messaging", - "version": "0.22.3", + "version": "1.0.0", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "67366d5f45cc92efe7adb6aab5d6dcd2342f2f9e" + "reference": "1aca8968c3bf3c78c428a399dfe97cbe9b366ec4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/67366d5f45cc92efe7adb6aab5d6dcd2342f2f9e", - "reference": "67366d5f45cc92efe7adb6aab5d6dcd2342f2f9e", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/1aca8968c3bf3c78c428a399dfe97cbe9b366ec4", + "reference": "1aca8968c3bf3c78c428a399dfe97cbe9b366ec4", "shasum": "" }, "require": { @@ -4651,9 +4652,9 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.22.3" + "source": "https://github.com/utopia-php/messaging/tree/1.0.0" }, - "time": "2026-05-19T05:31:20+00:00" + "time": "2026-06-02T14:54:22+00:00" }, { "name": "utopia-php/migration", @@ -4773,16 +4774,16 @@ }, { "name": "utopia-php/platform", - "version": "1.0.0-rc3", + "version": "1.0.0-rc4", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "674fd3cea6d1f8db660b01c5be89e8333dfb6de9" + "reference": "455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/platform/zipball/674fd3cea6d1f8db660b01c5be89e8333dfb6de9", - "reference": "674fd3cea6d1f8db660b01c5be89e8333dfb6de9", + "url": "https://api.github.com/repos/utopia-php/platform/zipball/455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0", + "reference": "455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0", "shasum": "" }, "require": { @@ -4791,7 +4792,7 @@ "php": ">=8.3", "utopia-php/cli": "0.23.*", "utopia-php/http": "^2.0@RC", - "utopia-php/queue": "0.18.*", + "utopia-php/queue": "^0.19", "utopia-php/servers": "0.4.*" }, "require-dev": { @@ -4818,27 +4819,27 @@ ], "support": { "issues": "https://github.com/utopia-php/platform/issues", - "source": "https://github.com/utopia-php/platform/tree/1.0.0-rc3" + "source": "https://github.com/utopia-php/platform/tree/1.0.0-rc4" }, - "time": "2026-05-25T11:48:18+00:00" + "time": "2026-06-02T20:19:26+00:00" }, { "name": "utopia-php/pools", - "version": "1.0.3", + "version": "1.0.4", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "74de7c5457a2c447f27e7ec4d72e8412a7d68c10" + "reference": "19d8b82f6772a901eb1e16aa7c0e7dfba0ba711b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/74de7c5457a2c447f27e7ec4d72e8412a7d68c10", - "reference": "74de7c5457a2c447f27e7ec4d72e8412a7d68c10", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/19d8b82f6772a901eb1e16aa7c0e7dfba0ba711b", + "reference": "19d8b82f6772a901eb1e16aa7c0e7dfba0ba711b", "shasum": "" }, "require": { "php": ">=8.4", - "utopia-php/telemetry": "*" + "utopia-php/telemetry": "^0.4" }, "require-dev": { "laravel/pint": "1.*", @@ -4871,9 +4872,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/1.0.3" + "source": "https://github.com/utopia-php/pools/tree/1.0.4" }, - "time": "2026-02-26T08:42:40+00:00" + "time": "2026-06-03T05:20:57+00:00" }, { "name": "utopia-php/preloader", @@ -4976,16 +4977,16 @@ }, { "name": "utopia-php/queue", - "version": "0.18.6", + "version": "0.19.0", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "954cd3788b37697b83fd2757adcd93422f21a846" + "reference": "cd4248af69006b2568c3486a675f27cb21191859" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/954cd3788b37697b83fd2757adcd93422f21a846", - "reference": "954cd3788b37697b83fd2757adcd93422f21a846", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/cd4248af69006b2568c3486a675f27cb21191859", + "reference": "cd4248af69006b2568c3486a675f27cb21191859", "shasum": "" }, "require": { @@ -5037,9 +5038,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.18.6" + "source": "https://github.com/utopia-php/queue/tree/0.19.0" }, - "time": "2026-06-01T19:47:57+00:00" + "time": "2026-06-02T20:12:18+00:00" }, { "name": "utopia-php/registry", @@ -5193,16 +5194,16 @@ }, { "name": "utopia-php/storage", - "version": "2.0.4", + "version": "2.0.5", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "a604b0ee011c4f366d53167caf2cf6b0f1d62d7e" + "reference": "b8af52e8ae1f9e670b32255a8936bbb46634b711" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/a604b0ee011c4f366d53167caf2cf6b0f1d62d7e", - "reference": "a604b0ee011c4f366d53167caf2cf6b0f1d62d7e", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/b8af52e8ae1f9e670b32255a8936bbb46634b711", + "reference": "b8af52e8ae1f9e670b32255a8936bbb46634b711", "shasum": "" }, "require": { @@ -5239,9 +5240,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/2.0.4" + "source": "https://github.com/utopia-php/storage/tree/2.0.5" }, - "time": "2026-06-02T04:17:22+00:00" + "time": "2026-06-03T05:21:46+00:00" }, { "name": "utopia-php/system", @@ -5401,16 +5402,16 @@ }, { "name": "utopia-php/vcs", - "version": "4.2.0", + "version": "4.2.1", "source": { "type": "git", "url": "https://github.com/utopia-php/vcs.git", - "reference": "49d7751f0ae94634b00057177d9823928f6777c6" + "reference": "59b64833fafb6c6e88c1eff8545e4e6adb8f5993" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/vcs/zipball/49d7751f0ae94634b00057177d9823928f6777c6", - "reference": "49d7751f0ae94634b00057177d9823928f6777c6", + "url": "https://api.github.com/repos/utopia-php/vcs/zipball/59b64833fafb6c6e88c1eff8545e4e6adb8f5993", + "reference": "59b64833fafb6c6e88c1eff8545e4e6adb8f5993", "shasum": "" }, "require": { @@ -5444,9 +5445,9 @@ ], "support": { "issues": "https://github.com/utopia-php/vcs/issues", - "source": "https://github.com/utopia-php/vcs/tree/4.2.0" + "source": "https://github.com/utopia-php/vcs/tree/4.2.1" }, - "time": "2026-05-17T15:58:27+00:00" + "time": "2026-05-29T10:47:49+00:00" }, { "name": "utopia-php/websocket", @@ -5639,16 +5640,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "1.31.8", + "version": "1.31.9", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "3bc6c697e60807dd036b5c02c66d03183afad79f" + "reference": "e7fa2eb215091df41a4535b306f60dad92652426" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/3bc6c697e60807dd036b5c02c66d03183afad79f", - "reference": "3bc6c697e60807dd036b5c02c66d03183afad79f", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/e7fa2eb215091df41a4535b306f60dad92652426", + "reference": "e7fa2eb215091df41a4535b306f60dad92652426", "shasum": "" }, "require": { @@ -5684,9 +5685,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/1.31.8" + "source": "https://github.com/appwrite/sdk-generator/tree/1.31.9" }, - "time": "2026-05-28T05:01:17+00:00" + "time": "2026-05-31T06:40:47+00:00" }, { "name": "brianium/paratest", @@ -6395,11 +6396,11 @@ }, { "name": "phpstan/phpstan", - "version": "2.1.55", + "version": "2.2.1", "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/9eaac3826ed5e9b8427350a43cac825eeca3f566", - "reference": "9eaac3826ed5e9b8427350a43cac825eeca3f566", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/dea9c8f2d25cc849391042b71e429c1a4bf82660", + "reference": "dea9c8f2d25cc849391042b71e429c1a4bf82660", "shasum": "" }, "require": { @@ -6422,6 +6423,17 @@ "license": [ "MIT" ], + "authors": [ + { + "name": "Ondřej Mirtes" + }, + { + "name": "Markus Staab" + }, + { + "name": "Vincent Langlet" + } + ], "description": "PHPStan - PHP Static Analysis Tool", "keywords": [ "dev", @@ -6444,20 +6456,20 @@ "type": "github" } ], - "time": "2026-05-18T11:57:34+00:00" + "time": "2026-05-28T14:44:12+00:00" }, { "name": "phpunit/php-code-coverage", - "version": "12.5.6", + "version": "12.5.7", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "876099a072646c7745f673d7aeab5382c4439691" + "reference": "186dab580576598076de6818596d12b61801880e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/876099a072646c7745f673d7aeab5382c4439691", - "reference": "876099a072646c7745f673d7aeab5382c4439691", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/186dab580576598076de6818596d12b61801880e", + "reference": "186dab580576598076de6818596d12b61801880e", "shasum": "" }, "require": { @@ -6468,13 +6480,13 @@ "php": ">=8.3", "phpunit/php-text-template": "^5.0", "sebastian/complexity": "^5.0", - "sebastian/environment": "^8.0.3", - "sebastian/lines-of-code": "^4.0", + "sebastian/environment": "^8.1.2", + "sebastian/lines-of-code": "^4.0.1", "sebastian/version": "^6.0", "theseer/tokenizer": "^2.0.1" }, "require-dev": { - "phpunit/phpunit": "^12.5.1" + "phpunit/phpunit": "^12.5.28" }, "suggest": { "ext-pcov": "PHP extension that provides line coverage", @@ -6512,7 +6524,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/12.5.6" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/12.5.7" }, "funding": [ { @@ -6532,7 +6544,7 @@ "type": "tidelift" } ], - "time": "2026-04-15T08:23:17+00:00" + "time": "2026-06-01T13:24:19+00:00" }, { "name": "phpunit/php-file-iterator", @@ -6793,16 +6805,16 @@ }, { "name": "phpunit/phpunit", - "version": "12.5.25", + "version": "12.5.28", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "792c2980442dfce319226b88fa845b8b6de3b333" + "reference": "5895d05f5bf421ed230fbd76e1277e4b8955def4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/792c2980442dfce319226b88fa845b8b6de3b333", - "reference": "792c2980442dfce319226b88fa845b8b6de3b333", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/5895d05f5bf421ed230fbd76e1277e4b8955def4", + "reference": "5895d05f5bf421ed230fbd76e1277e4b8955def4", "shasum": "" }, "require": { @@ -6821,15 +6833,15 @@ "phpunit/php-invoker": "^6.0.0", "phpunit/php-text-template": "^5.0.0", "phpunit/php-timer": "^8.0.0", - "sebastian/cli-parser": "^4.2.0", - "sebastian/comparator": "^7.1.6", + "sebastian/cli-parser": "^4.2.1", + "sebastian/comparator": "^7.1.8", "sebastian/diff": "^7.0.0", - "sebastian/environment": "^8.1.0", - "sebastian/exporter": "^7.0.2", + "sebastian/environment": "^8.1.2", + "sebastian/exporter": "^7.0.3", "sebastian/global-state": "^8.0.2", "sebastian/object-enumerator": "^7.0.0", "sebastian/recursion-context": "^7.0.1", - "sebastian/type": "^6.0.3", + "sebastian/type": "^6.0.4", "sebastian/version": "^6.0.0", "staabm/side-effects-detector": "^1.0.5" }, @@ -6871,7 +6883,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/12.5.25" + "source": "https://github.com/sebastianbergmann/phpunit/tree/12.5.28" }, "funding": [ { @@ -6879,7 +6891,7 @@ "type": "other" } ], - "time": "2026-05-13T03:56:57+00:00" + "time": "2026-05-27T14:01:10+00:00" }, { "name": "sebastian/cli-parser", @@ -6952,16 +6964,16 @@ }, { "name": "sebastian/comparator", - "version": "7.1.6", + "version": "7.1.8", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/comparator.git", - "reference": "c769009dee98f494e0edc3fd4f4087501688f11e" + "reference": "7c65c1e79836812819705b473a90c12399542485" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/c769009dee98f494e0edc3fd4f4087501688f11e", - "reference": "c769009dee98f494e0edc3fd4f4087501688f11e", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/7c65c1e79836812819705b473a90c12399542485", + "reference": "7c65c1e79836812819705b473a90c12399542485", "shasum": "" }, "require": { @@ -6969,10 +6981,10 @@ "ext-mbstring": "*", "php": ">=8.3", "sebastian/diff": "^7.0", - "sebastian/exporter": "^7.0" + "sebastian/exporter": "^7.0.3" }, "require-dev": { - "phpunit/phpunit": "^12.2" + "phpunit/phpunit": "^12.5.25" }, "suggest": { "ext-bcmath": "For comparing BcMath\\Number objects" @@ -7020,7 +7032,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/comparator/issues", "security": "https://github.com/sebastianbergmann/comparator/security/policy", - "source": "https://github.com/sebastianbergmann/comparator/tree/7.1.6" + "source": "https://github.com/sebastianbergmann/comparator/tree/7.1.8" }, "funding": [ { @@ -7040,7 +7052,7 @@ "type": "tidelift" } ], - "time": "2026-04-14T08:23:15+00:00" + "time": "2026-05-21T04:45:25+00:00" }, { "name": "sebastian/complexity", @@ -7169,23 +7181,23 @@ }, { "name": "sebastian/environment", - "version": "8.1.0", + "version": "8.1.2", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/environment.git", - "reference": "b121608b28a13f721e76ffbbd386d08eff58f3f6" + "reference": "9d32c685773823b1983e256ae4ecd48a10d6e439" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/b121608b28a13f721e76ffbbd386d08eff58f3f6", - "reference": "b121608b28a13f721e76ffbbd386d08eff58f3f6", + "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/9d32c685773823b1983e256ae4ecd48a10d6e439", + "reference": "9d32c685773823b1983e256ae4ecd48a10d6e439", "shasum": "" }, "require": { "php": ">=8.3" }, "require-dev": { - "phpunit/phpunit": "^12.0" + "phpunit/phpunit": "^12.5.26" }, "suggest": { "ext-posix": "*" @@ -7221,7 +7233,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/environment/issues", "security": "https://github.com/sebastianbergmann/environment/security/policy", - "source": "https://github.com/sebastianbergmann/environment/tree/8.1.0" + "source": "https://github.com/sebastianbergmann/environment/tree/8.1.2" }, "funding": [ { @@ -7241,7 +7253,7 @@ "type": "tidelift" } ], - "time": "2026-04-15T12:13:01+00:00" + "time": "2026-05-25T13:40:20+00:00" }, { "name": "sebastian/exporter", @@ -7335,26 +7347,26 @@ }, { "name": "sebastian/global-state", - "version": "8.0.2", + "version": "8.0.3", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/global-state.git", - "reference": "ef1377171613d09edd25b7816f05be8313f9115d" + "reference": "b164d3274d6537ab462591c5755f76a8f5b1aae9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/ef1377171613d09edd25b7816f05be8313f9115d", - "reference": "ef1377171613d09edd25b7816f05be8313f9115d", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/b164d3274d6537ab462591c5755f76a8f5b1aae9", + "reference": "b164d3274d6537ab462591c5755f76a8f5b1aae9", "shasum": "" }, "require": { "php": ">=8.3", "sebastian/object-reflector": "^5.0", - "sebastian/recursion-context": "^7.0" + "sebastian/recursion-context": "^7.0.1" }, "require-dev": { "ext-dom": "*", - "phpunit/phpunit": "^12.0" + "phpunit/phpunit": "^12.5.28" }, "type": "library", "extra": { @@ -7385,7 +7397,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/global-state/issues", "security": "https://github.com/sebastianbergmann/global-state/security/policy", - "source": "https://github.com/sebastianbergmann/global-state/tree/8.0.2" + "source": "https://github.com/sebastianbergmann/global-state/tree/8.0.3" }, "funding": [ { @@ -7405,7 +7417,7 @@ "type": "tidelift" } ], - "time": "2025-08-29T11:29:25+00:00" + "time": "2026-06-01T15:10:33+00:00" }, { "name": "sebastian/lines-of-code", @@ -7876,23 +7888,29 @@ }, { "name": "symfony/console", - "version": "v8.0.11", + "version": "v8.1.0", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "3156577f46a38aa1b9323aad223de7a9cd426782" + "reference": "f5a856c6ecb56b3c21ed94a5b7bf940d857d110a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/3156577f46a38aa1b9323aad223de7a9cd426782", - "reference": "3156577f46a38aa1b9323aad223de7a9cd426782", + "url": "https://api.github.com/repos/symfony/console/zipball/f5a856c6ecb56b3c21ed94a5b7bf940d857d110a", + "reference": "f5a856c6ecb56b3c21ed94a5b7bf940d857d110a", "shasum": "" }, "require": { - "php": ">=8.4", + "php": ">=8.4.1", + "symfony/deprecation-contracts": "^2.5|^3", "symfony/polyfill-mbstring": "^1.0", + "symfony/polyfill-php85": "^1.32", "symfony/service-contracts": "^2.5|^3", - "symfony/string": "^7.4|^8.0" + "symfony/string": "^7.4.6|^8.0.6" + }, + "conflict": { + "symfony/dependency-injection": "<8.1", + "symfony/event-dispatcher": "<8.1" }, "provide": { "psr/log-implementation": "1.0|2.0|3.0" @@ -7900,14 +7918,18 @@ "require-dev": { "psr/log": "^1|^2|^3", "symfony/config": "^7.4|^8.0", - "symfony/dependency-injection": "^7.4|^8.0", - "symfony/event-dispatcher": "^7.4|^8.0", + "symfony/dependency-injection": "^8.1", + "symfony/event-dispatcher": "^8.1", + "symfony/filesystem": "^7.4|^8.0", "symfony/http-foundation": "^7.4|^8.0", "symfony/http-kernel": "^7.4|^8.0", "symfony/lock": "^7.4|^8.0", "symfony/messenger": "^7.4|^8.0", + "symfony/mime": "^7.4|^8.0", "symfony/process": "^7.4|^8.0", "symfony/stopwatch": "^7.4|^8.0", + "symfony/uid": "^7.4|^8.0", + "symfony/validator": "^7.4|^8.0", "symfony/var-dumper": "^7.4|^8.0" }, "type": "library", @@ -7942,7 +7964,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v8.0.11" + "source": "https://github.com/symfony/console/tree/v8.1.0" }, "funding": [ { @@ -7962,7 +7984,7 @@ "type": "tidelift" } ], - "time": "2026-05-13T12:07:53+00:00" + "time": "2026-05-29T05:06:50+00:00" }, { "name": "symfony/polyfill-ctype", @@ -8049,16 +8071,16 @@ }, { "name": "symfony/polyfill-intl-grapheme", - "version": "v1.37.0", + "version": "v1.38.1", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-grapheme.git", - "reference": "4864388bfbd3001ce88e234fab652acd91fdc57e" + "reference": "e9247d281d694a5120554d9afaf54e070e88a603" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/4864388bfbd3001ce88e234fab652acd91fdc57e", - "reference": "4864388bfbd3001ce88e234fab652acd91fdc57e", + "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/e9247d281d694a5120554d9afaf54e070e88a603", + "reference": "e9247d281d694a5120554d9afaf54e070e88a603", "shasum": "" }, "require": { @@ -8107,7 +8129,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.37.0" + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.38.1" }, "funding": [ { @@ -8127,20 +8149,20 @@ "type": "tidelift" } ], - "time": "2026-04-26T13:13:48+00:00" + "time": "2026-05-26T05:58:03+00:00" }, { "name": "symfony/polyfill-intl-normalizer", - "version": "v1.37.0", + "version": "v1.38.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-normalizer.git", - "reference": "3833d7255cc303546435cb650316bff708a1c75c" + "reference": "2d446c214bdbe5b71bde5011b060a05fece3ae6b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/3833d7255cc303546435cb650316bff708a1c75c", - "reference": "3833d7255cc303546435cb650316bff708a1c75c", + "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/2d446c214bdbe5b71bde5011b060a05fece3ae6b", + "reference": "2d446c214bdbe5b71bde5011b060a05fece3ae6b", "shasum": "" }, "require": { @@ -8192,7 +8214,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.37.0" + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.38.0" }, "funding": [ { @@ -8212,24 +8234,24 @@ "type": "tidelift" } ], - "time": "2024-09-09T11:45:10+00:00" + "time": "2026-05-25T13:48:31+00:00" }, { "name": "symfony/process", - "version": "v8.0.11", + "version": "v8.1.0", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "26d89e459f037d2873300605d0a07e7a8ef84db0" + "reference": "c4a9e58f235a6bf7f97ffbfedae2687353ac79e5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/26d89e459f037d2873300605d0a07e7a8ef84db0", - "reference": "26d89e459f037d2873300605d0a07e7a8ef84db0", + "url": "https://api.github.com/repos/symfony/process/zipball/c4a9e58f235a6bf7f97ffbfedae2687353ac79e5", + "reference": "c4a9e58f235a6bf7f97ffbfedae2687353ac79e5", "shasum": "" }, "require": { - "php": ">=8.4" + "php": ">=8.4.1" }, "type": "library", "autoload": { @@ -8257,7 +8279,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v8.0.11" + "source": "https://github.com/symfony/process/tree/v8.1.0" }, "funding": [ { @@ -8277,24 +8299,24 @@ "type": "tidelift" } ], - "time": "2026-05-11T16:56:32+00:00" + "time": "2026-05-29T05:06:50+00:00" }, { "name": "symfony/string", - "version": "v8.0.11", + "version": "v8.1.0", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "39be2ad058a3c0bd558edca23e65f009865d75ff" + "reference": "afd5944f4005862d961efb85c8bbd5c523c4e3c9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/39be2ad058a3c0bd558edca23e65f009865d75ff", - "reference": "39be2ad058a3c0bd558edca23e65f009865d75ff", + "url": "https://api.github.com/repos/symfony/string/zipball/afd5944f4005862d961efb85c8bbd5c523c4e3c9", + "reference": "afd5944f4005862d961efb85c8bbd5c523c4e3c9", "shasum": "" }, "require": { - "php": ">=8.4", + "php": ">=8.4.1", "symfony/polyfill-ctype": "^1.8", "symfony/polyfill-intl-grapheme": "^1.33", "symfony/polyfill-intl-normalizer": "^1.0", @@ -8347,7 +8369,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v8.0.11" + "source": "https://github.com/symfony/string/tree/v8.1.0" }, "funding": [ { @@ -8367,7 +8389,7 @@ "type": "tidelift" } ], - "time": "2026-05-13T12:07:53+00:00" + "time": "2026-05-29T05:06:50+00:00" }, { "name": "textalk/websocket", @@ -8470,16 +8492,16 @@ }, { "name": "twig/twig", - "version": "v3.27.0", + "version": "v3.27.1", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "04ae1bfe9463c816cf72ca0abe7eae2c77a9a9ed" + "reference": "ae2071bffb38f04847fc0864d730c94b9cb8ab74" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/04ae1bfe9463c816cf72ca0abe7eae2c77a9a9ed", - "reference": "04ae1bfe9463c816cf72ca0abe7eae2c77a9a9ed", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/ae2071bffb38f04847fc0864d730c94b9cb8ab74", + "reference": "ae2071bffb38f04847fc0864d730c94b9cb8ab74", "shasum": "" }, "require": { @@ -8534,7 +8556,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v3.27.0" + "source": "https://github.com/twigphp/Twig/tree/v3.27.1" }, "funding": [ { @@ -8546,7 +8568,7 @@ "type": "tidelift" } ], - "time": "2026-05-27T13:05:51+00:00" + "time": "2026-05-30T17:09:26+00:00" } ], "aliases": [], diff --git a/docker-compose.yml b/docker-compose.yml index ab51ad32c0..90bea1dc5d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -349,6 +349,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -383,6 +385,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -424,6 +428,8 @@ services: environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -484,6 +490,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -504,7 +512,6 @@ services: - _APP_DB_USER_VECTORSDB - _APP_DB_PASS_VECTORSDB - _APP_LOGGING_CONFIG - - _APP_WORKERS_NUM - _APP_QUEUE_NAME - _APP_DATABASE_SHARED_TABLES @@ -527,6 +534,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -605,6 +614,8 @@ services: - _APP_OPTIONS_FORCE_HTTPS # Basic - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_LOGGING_CONFIG @@ -665,6 +676,8 @@ services: - ./src:/usr/src/code/src environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -705,6 +718,8 @@ services: - ${_APP_DB_HOST:-mongodb} environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -736,6 +751,8 @@ services: - redis environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -782,6 +799,8 @@ services: # - smtp environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -823,6 +842,8 @@ services: - redis environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -883,6 +904,8 @@ services: - ${_APP_DB_HOST:-mongodb} environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1044,6 +1067,8 @@ services: - ${_APP_DB_HOST:-mongodb} environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1078,6 +1103,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 diff --git a/docs/references/messaging/create-ses-provider.md b/docs/references/messaging/create-ses-provider.md new file mode 100644 index 0000000000..c612414f99 --- /dev/null +++ b/docs/references/messaging/create-ses-provider.md @@ -0,0 +1 @@ +Create a new Amazon SES provider. \ No newline at end of file diff --git a/docs/references/messaging/update-ses-provider.md b/docs/references/messaging/update-ses-provider.md new file mode 100644 index 0000000000..79947186e8 --- /dev/null +++ b/docs/references/messaging/update-ses-provider.md @@ -0,0 +1 @@ +Update an Amazon SES provider by its unique ID. \ No newline at end of file diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index 69b27af25c..11f388d3c0 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -260,8 +260,8 @@ class Create extends Action }; try { - $locks($lockKey, 600, function () use ($bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $response, &$completed): void { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + $locks($lockKey, 600, function () use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $response, &$completed): void { + $file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); if (!$file->isEmpty()) { $chunks = $file->getAttribute('chunksTotal', 1); $uploaded = $file->getAttribute('chunksUploaded', 0); @@ -325,7 +325,7 @@ class Create extends Action } $finalizeUpload = function (int $chunksUploaded) use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $mergeUploadMetadata, $path, $permissions, $queueForEvents, $response): void { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + $file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); $uploaded = 0; if (!$file->isEmpty()) { diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 7af079e914..ddd7edf641 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -16,11 +16,13 @@ use Utopia\Database\Document; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\DSN\DSN; +use Utopia\Lock\Semaphore; use Utopia\Logger\Log; use Utopia\Messaging\Adapter\Email as EmailAdapter; use Utopia\Messaging\Adapter\Email\Mailgun; use Utopia\Messaging\Adapter\Email\Resend; use Utopia\Messaging\Adapter\Email\Sendgrid; +use Utopia\Messaging\Adapter\Email\SES; use Utopia\Messaging\Adapter\Email\SMTP; use Utopia\Messaging\Adapter\Push\APNS; use Utopia\Messaging\Adapter\Push as PushAdapter; @@ -52,8 +54,6 @@ use function Swoole\Coroutine\batch; class Messaging extends Action { - private ?Local $localDevice = null; - private ?SMSAdapter $adapter = null; public static function getName(): string @@ -130,69 +130,19 @@ class Messaging extends Action Document $project, UsagePublisher $publisherForUsage ): void { + $status = $message->getAttribute('status'); + + // Idempotency guard: queue redelivery can hand us a message that already finished. + if (\in_array($status, [MessageStatus::SENT, MessageStatus::FAILED], true)) { + Span::add('message.skipped', 'already_processed'); + return; + } + $topicIds = $message->getAttribute('topics', []); $targetIds = $message->getAttribute('targets', []); $userIds = $message->getAttribute('users', []); $providerType = $message->getAttribute('providerType'); - /** - * @var array $allTargets - */ - $allTargets = []; - - if (\count($topicIds) > 0) { - $topics = $dbForProject->find('topics', [ - Query::equal('$id', $topicIds), - Query::limit(\count($topicIds)), - ]); - foreach ($topics as $topic) { - $targets = \array_filter($topic->getAttribute('targets'), function (Document $target) use ($providerType) { - return $target->getAttribute('providerType') === $providerType; - }); - - \array_push($allTargets, ...$targets); - } - } - - if (\count($userIds) > 0) { - $limit = 1000; - $offset = 0; - - do { - $targets = $dbForProject->find('targets', [ - Query::equal('userId', $userIds), - Query::select(['providerId', 'identifier']), - Query::equal('providerType', [$providerType]), - Query::limit($limit), - Query::offset($offset), - ]); - - \array_push($allTargets, ...$targets); - $offset += \count($targets); - } while (\count($targets) === $limit); - } - - if (\count($targetIds) > 0) { - $targets = $dbForProject->find('targets', [ - Query::select(['providerId', 'identifier']), - Query::equal('$id', $targetIds), - Query::equal('providerType', [$providerType]), - Query::limit(\count($targetIds)), - ]); - - \array_push($allTargets, ...$targets); - } - - if (empty($allTargets)) { - $dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([ - 'status' => MessageStatus::FAILED, - 'deliveryErrors' => ['No valid recipients found.'] - ])); - - Span::add('message.skipped', 'no_valid_recipients'); - return; - } - $default = $dbForProject->findOne('providers', [ Query::equal('enabled', [true]), Query::equal('type', [$providerType]), @@ -209,55 +159,33 @@ class Messaging extends Action } /** - * @var array> $identifiers - */ - $identifiers = []; - - /** - * @var array $providers + * Resolved providers cached for the lifetime of this job, keyed by provider id. + * Seeded with the default provider so most sends never touch the providers collection. + * + * @var array $providers */ $providers = [ - $default->getId() => $default + $default->getId() => $default, ]; - foreach ($allTargets as $target) { - $providerId = $target->getAttribute('providerId'); + $semaphore = new Semaphore(MESSAGE_SEND_CONCURRENCY); - if (!$providerId) { - $providerId = $default->getId(); - } + $deliveredTotal = 0; + $failedTotal = 0; + $deliveryErrors = []; + $hasRecipients = false; - if ($providerId) { - if (!\array_key_exists($providerId, $identifiers)) { - $identifiers[$providerId] = []; - } - // Use null as value to avoid duplicate keys - $identifiers[$providerId][$target->getAttribute('identifier')] = null; - } - } + foreach ($this->streamRecipients($dbForProject, $topicIds, $userIds, $targetIds, $providerType, $default) as $page) { + /** + * @var array $tasks + */ + $tasks = []; - /** - * @var array $results - */ - $results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - if (\array_key_exists($providerId, $providers)) { - $provider = $providers[$providerId]; - } else { - $provider = $dbForProject->getDocument('providers', $providerId); + foreach ($page as $providerId => $identifiers) { + $provider = $this->resolveProvider($dbForProject, $providerId, $providers, $default); + $resolvedProviderType = $provider->getAttribute('type'); - if ($provider->isEmpty() || !$provider->getAttribute('enabled')) { - $provider = $default; - } else { - $providers[$providerId] = $provider; - } - } - - $identifiersForProvider = $identifiers[$providerId]; - - $providerType = $provider->getAttribute('type'); - - $adapter = match ($providerType) { + $adapter = match ($resolvedProviderType) { MESSAGE_TYPE_SMS => $this->getSmsAdapter($provider), MESSAGE_TYPE_PUSH => $this->getPushAdapter($provider), MESSAGE_TYPE_EMAIL => $this->getEmailAdapter($provider), @@ -265,101 +193,74 @@ class Messaging extends Action }; $batches = \array_chunk( - \array_keys($identifiersForProvider), + \array_keys($identifiers), $adapter->getMaxMessagesPerRequest() ); - return batch(\array_map(function ($batch) use ($message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - return function () use ($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { - $deliveredTotal = 0; - $deliveryErrors = []; - $messageData = clone $message; - $messageData->setAttribute('to', $batch); + foreach ($batches as $batch) { + $tasks[] = fn (): array => $semaphore->withLock( + fn (): array => $this->sendBatch( + $batch, + $message, + $provider, + $resolvedProviderType, + $adapter, + $dbForProject, + $deviceForFiles, + $project, + $publisherForUsage + ) + ); + } + } - $data = match ($providerType) { - MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider), - MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData), - MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project), - }; + // A page can resolve to zero identifiers (e.g. subscribers whose targetInternalId matches no targets + // row), so an empty task list must not count as recipients or run batch() for nothing. + if (empty($tasks)) { + continue; + } - try { - $response = $adapter->send($data); - $deliveredTotal += (int) $response['deliveredTo']; - foreach ($response['results'] as $result) { - if ($result['status'] === 'failure') { - $deliveryErrors[] = "Failed sending to target {$result['recipient']} with error: {$result['error']}"; - } + $hasRecipients = true; - // Deleting push targets when token has expired. - if (($result['error'] ?? '') === 'Expired device token') { - $target = $dbForProject->findOne('targets', [ - Query::equal('identifier', [$result['recipient']]) - ]); + /** + * @var array}> $results + */ + $results = batch($tasks); - if (!$target->isEmpty()) { - $dbForProject->updateDocument( - 'targets', - $target->getId(), - $target->setAttribute('expired', true) - ); - } - } - } - } catch (\Throwable $e) { - $deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage(); - } finally { - $errorTotal = \count($deliveryErrors); - $usage = new UsageContext(); - $usage - ->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal)) - ->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal) - ->addMetric(METRIC_MESSAGES_FAILED, $errorTotal) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), ($deliveredTotal + $errorTotal)) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $deliveredTotal) - ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $errorTotal) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), ($deliveredTotal + $errorTotal)) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $deliveredTotal) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal); + foreach ($results as $result) { + $deliveredTotal += $result['delivered']; + $failedTotal += $result['recipients'] - $result['delivered']; - $publisherForUsage->enqueue(new Usage( - project: $project, - metrics: $usage->getMetrics(), - )); + foreach ($result['errors'] as $error) { + if (\count($deliveryErrors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) { + break; + } - return [ - 'deliveredTotal' => $deliveredTotal, - 'deliveryErrors' => $deliveryErrors, - ]; - } - }; - }, $batches)); - }; - }, \array_keys($identifiers))); + $deliveryErrors[] = $error; + } + } + } - $results = \array_merge(...$results); + if (!$hasRecipients) { + $dbForProject->updateDocument('messages', $message->getId(), $message->setAttributes([ + 'status' => MessageStatus::FAILED, + 'deliveryErrors' => ['No valid recipients found.'] + ])); - $deliveredTotal = 0; - $deliveryErrors = []; - - foreach ($results as $result) { - $deliveredTotal += $result['deliveredTotal']; - $deliveryErrors = \array_merge($deliveryErrors, $result['deliveryErrors']); + Span::add('message.skipped', 'no_valid_recipients'); + return; } if (empty($deliveryErrors) && $deliveredTotal === 0) { $deliveryErrors[] = 'Unknown error'; } + $hasFailures = $failedTotal > 0 || \count($deliveryErrors) > 0; + $message->setAttribute('status', $hasFailures ? MessageStatus::FAILED : MessageStatus::SENT); $message->setAttribute('deliveryErrors', $deliveryErrors); - if (\count($message->getAttribute('deliveryErrors')) > 0) { - $message->setAttribute('status', MessageStatus::FAILED); - } else { - $message->setAttribute('status', MessageStatus::SENT); - } - Span::add('message.delivered_total', $deliveredTotal); - Span::add('message.errors_total', \count($deliveryErrors)); + Span::add('message.errors_total', $failedTotal); $message->removeAttribute('to'); @@ -410,6 +311,461 @@ class Messaging extends Action } } + /** + * Stream a message's recipients in bounded pages, grouped by provider and deduplicated by identifier within each page. + * + * Peak memory is O(MESSAGE_RECIPIENTS_PAGE_SIZE), never O(topic size): topics are walked through the + * subscribers collection with cursor pagination rather than reading the topic's `targets` attribute, which + * triggers the subQueryTopicTargets filter and loads up to APP_LIMIT_SUBSCRIBERS_SUBQUERY rows at once. + * + * @param array $topicIds + * @param array $userIds + * @param array $targetIds + * @return \Generator>> + * @throws \Exception + */ + private function streamRecipients( + Database $dbForProject, + array $topicIds, + array $userIds, + array $targetIds, + string $providerType, + Document $default + ): \Generator { + if (\count($topicIds) > 0) { + $topics = $dbForProject->find('topics', [ + Query::select(['$sequence']), + Query::equal('$id', $topicIds), + Query::limit(\count($topicIds)), + ]); + + foreach ($topics as $topic) { + $cursor = null; + + do { + $queries = [ + Query::equal('topicInternalId', [$topic->getSequence()]), + Query::equal('providerType', [$providerType]), + Query::select(['$sequence', 'targetInternalId']), + Query::orderAsc('$sequence'), + Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE), + ]; + + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + + $subscribers = $dbForProject->getAuthorization()->skip( + fn () => $dbForProject->find('subscribers', $queries) + ); + + $count = \count($subscribers); + + if ($count === 0) { + break; + } + + $cursor = $subscribers[$count - 1]; + + $targetInternalIds = \array_map( + fn (Document $subscriber) => $subscriber->getAttribute('targetInternalId'), + $subscribers + ); + + $targets = $dbForProject->skipValidation( + fn () => $dbForProject->getAuthorization()->skip( + fn () => $dbForProject->find('targets', [ + Query::equal('$sequence', $targetInternalIds), + Query::select(['providerId', 'identifier']), + Query::limit(\count($targetInternalIds)), + ]) + ) + ); + + yield $this->groupTargetsByProvider($targets, $default); + } while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE); + } + } + + if (\count($userIds) > 0) { + $cursor = null; + + do { + $queries = [ + Query::equal('userId', $userIds), + Query::equal('providerType', [$providerType]), + Query::select(['$sequence', 'providerId', 'identifier']), + Query::orderAsc('$sequence'), + Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE), + ]; + + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + + $targets = $dbForProject->find('targets', $queries); + $count = \count($targets); + + if ($count === 0) { + break; + } + + $cursor = $targets[$count - 1]; + + yield $this->groupTargetsByProvider($targets, $default); + } while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE); + } + + if (\count($targetIds) > 0) { + $cursor = null; + + do { + $queries = [ + Query::equal('$id', $targetIds), + Query::equal('providerType', [$providerType]), + Query::select(['$sequence', 'providerId', 'identifier']), + Query::orderAsc('$sequence'), + Query::limit(MESSAGE_RECIPIENTS_PAGE_SIZE), + ]; + + if ($cursor !== null) { + $queries[] = Query::cursorAfter($cursor); + } + + $targets = $dbForProject->find('targets', $queries); + $count = \count($targets); + + if ($count === 0) { + break; + } + + $cursor = $targets[$count - 1]; + + yield $this->groupTargetsByProvider($targets, $default); + } while ($count === MESSAGE_RECIPIENTS_PAGE_SIZE); + } + } + + /** + * Group a page of target documents by provider id, deduplicating identifiers within the page. + * + * @param array $targets + * @return array> + */ + private function groupTargetsByProvider(array $targets, Document $default): array + { + /** + * @var array> $identifiers + */ + $identifiers = []; + + foreach ($targets as $target) { + $providerId = $target->getAttribute('providerId') ?: $default->getId(); + + if (!\array_key_exists($providerId, $identifiers)) { + $identifiers[$providerId] = []; + } + + // Null values keep identifiers unique without a second lookup structure. + $identifiers[$providerId][$target->getAttribute('identifier')] = null; + } + + return $identifiers; + } + + /** + * Resolve and cache a provider for the lifetime of a send job, falling back to the default provider. + * + * @param array $providers + */ + private function resolveProvider( + Database $dbForProject, + string $providerId, + array &$providers, + Document $default + ): Document { + if (\array_key_exists($providerId, $providers)) { + return $providers[$providerId]; + } + + $provider = $dbForProject->getDocument('providers', $providerId); + + if ($provider->isEmpty() || !$provider->getAttribute('enabled')) { + // Cache the fallback under this id too, so a topic full of targets pointing at the same + // disabled/missing provider does not re-query the providers collection once per page. + $providers[$providerId] = $default; + + return $default; + } + + $providers[$providerId] = $provider; + + return $provider; + } + + /** + * Send a single adapter-sized batch and report delivery counts plus a bounded error list. + * + * Wraps the provider call in a backoff/retry loop that reacts to provider rate limiting and transient + * failures (see {@see retrySend()}). Accounting stays exact: only still-failing recipients are ever + * retried, so `delivered` is summed across attempts without double-counting, and `recipients` always + * reports the original batch size so the caller's `failed = recipients - delivered` holds. + * + * @param array $batch + * @return array{delivered: int, recipients: int, errors: array} + */ + private function sendBatch( + array $batch, + Document $message, + Document $provider, + string $providerType, + EmailAdapter|SMSAdapter|PushAdapter $adapter, + Database $dbForProject, + Device $deviceForFiles, + Document $project, + UsagePublisher $publisherForUsage + ): array { + $recipients = \count($batch); + + [ + 'delivered' => $delivered, + 'errors' => $errors, + ] = $this->retrySend($batch, $message, $provider, $providerType, $adapter, $dbForProject, $deviceForFiles, $project); + + $failed = $recipients - $delivered; + $usage = new UsageContext(); + $usage + ->addMetric(METRIC_MESSAGES, $recipients) + ->addMetric(METRIC_MESSAGES_SENT, $delivered) + ->addMetric(METRIC_MESSAGES_FAILED, $failed) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE), $recipients) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_SENT), $delivered) + ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $failed) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), $recipients) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $delivered) + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $failed); + + $publisherForUsage->enqueue(new Usage( + project: $project, + metrics: $usage->getMetrics(), + )); + + return [ + 'delivered' => $delivered, + 'recipients' => $recipients, + 'errors' => $errors, + ]; + } + + /** + * Drive a batch through provider sends with exponential backoff, reacting to throttling and transient + * errors by retrying only the recipients that are still failing. + * + * Partitioning per attempt: the provider's per-recipient results are split into delivered (summed into + * the running total, never retried) and failures. A failure whose error matches {@see isRetryableError()} + * stays pending for the next attempt; any other failure becomes a terminal error immediately. When + * `send()` throws, the throw is retryable only if its message matches; otherwise every still-pending + * recipient is recorded as terminal. The next attempt rebuilds the provider message with only the pending + * recipients in `to`, so an already-delivered recipient is never re-sent. After {@see MESSAGE_SEND_MAX_RETRIES} + * attempts, any recipients still pending are flushed to terminal errors. + * + * Note: provider `Retry-After` hints are not honored — the agnostic adapter {@see \Utopia\Messaging\Response} + * exposes only generic per-recipient strings, never a structured retry delay, so we keep the library + * adapter-agnostic and rely on exponential backoff alone. + * + * @param array $batch + * @return array{delivered: int, errors: array} + */ + private function retrySend( + array $batch, + Document $message, + Document $provider, + string $providerType, + EmailAdapter|SMSAdapter|PushAdapter $adapter, + Database $dbForProject, + Device $deviceForFiles, + Document $project + ): array { + $delivered = 0; + $errors = []; + + // Recipients still awaiting a successful (or terminal) outcome; shrinks to only the failing ones each attempt. + $pending = $batch; + + for ($attempt = 1; $attempt <= MESSAGE_SEND_MAX_RETRIES; $attempt++) { + $hasRetriesLeft = $attempt < MESSAGE_SEND_MAX_RETRIES; + + // Rebuild the provider message scoped to only the still-pending recipients so a partially-delivered + // batch never re-sends to recipients that already succeeded on an earlier attempt. + $data = $this->buildMessage($pending, $message, $provider, $providerType, $dbForProject, $deviceForFiles, $project); + + $retry = []; + + // The try/catch wraps ONLY the provider send. A whole-batch throw is retryable when transient, + // otherwise it records one representative terminal error. The previous behaviour of resetting + // $delivered to 0 on a throw is gone — the retry refactor sums delivered across attempts, and the + // expired-device-token cleanup below is isolated in its own try so a DB hiccup there can never be + // misattributed as a send failure. + try { + $response = $adapter->send($data); + } catch (\Throwable $e) { + if ($hasRetriesLeft && $this->isRetryableError($e->getMessage())) { + $retry = $pending; + } else { + $this->recordError($errors, 'Failed sending to targets with error: ' . $e->getMessage()); + } + + $response = null; + } + + if ($response !== null) { + $delivered += (int) $response['deliveredTo']; + + foreach ($response['results'] as $result) { + if ($result['status'] !== 'failure') { + continue; + } + + $error = $result['error'] ?? null; + $recipient = $result['recipient']; + + // Best-effort: deleting push targets when the token has expired. Isolated so a transient DB + // error here never affects delivery accounting or the retry decision below. + if ($error === 'Expired device token') { + try { + $target = $dbForProject->findOne('targets', [ + Query::equal('identifier', [$recipient]) + ]); + + if (!$target->isEmpty()) { + $dbForProject->updateDocument( + 'targets', + $target->getId(), + $target->setAttribute('expired', true) + ); + } + } catch (\Throwable) { + // Best-effort; must not affect accounting or retries. + } + } + + if ($hasRetriesLeft && $this->isRetryableError($error)) { + $retry[] = $recipient; + continue; + } + + $this->recordError($errors, "Failed sending to target {$recipient} with error: {$error}"); + } + } + + $pending = $retry; + + if (empty($pending)) { + break; + } + + // Exponential backoff with jitter; non-blocking under Swoole so sibling sends keep progressing. + // Skip a non-positive delay: Swoole\Coroutine::sleep() rejects 0/negative values (which tests + // produce by overriding the base delay to 0 for speed), and it would otherwise emit a warning. + $delay = $this->retryDelay() * (2 ** ($attempt - 1)); + $delay += $delay * (\random_int(0, 100) / 1000); + if ($delay > 0) { + \Swoole\Coroutine::sleep($delay); + } + } + + return [ + 'delivered' => $delivered, + 'errors' => $errors, + ]; + } + + /** + * Append a delivery error while keeping the retained list bounded by {@see MESSAGE_DELIVERY_ERRORS_LIMIT}. + * + * @param array $errors + */ + private function recordError(array &$errors, string $error): void + { + if (\count($errors) >= MESSAGE_DELIVERY_ERRORS_LIMIT) { + return; + } + + $errors[] = $error; + } + + /** + * Conservatively classify a provider error string as retryable. The agnostic adapter + * {@see \Utopia\Messaging\Response} returns only free-form error strings, so string matching is the only + * provider-agnostic signal available. The pattern list is intentionally narrow — throttling, rate limits, + * quota, service-unavailable and timeout phrasing — so permanent failures (e.g. invalid recipients) are + * never retried. + */ + private function isRetryableError(?string $error): bool + { + if ($error === null || $error === '') { + return false; + } + + $patterns = [ + 'throttl', + 'rate exceeded', + 'rate limit', + 'too many requests', + '429', + 'quota', + 'service unavailable', + '503', + 'timed out', + 'timeout', + 'temporarily', + ]; + + $needle = \strtolower($error); + + foreach ($patterns as $pattern) { + if (\str_contains($needle, $pattern)) { + return true; + } + } + + return false; + } + + /** + * Base seconds for exponential backoff between send retries. Isolated so tests can override it to keep the + * suite instant without touching the production constant. + */ + protected function retryDelay(): float + { + return MESSAGE_SEND_RETRY_DELAY; + } + + /** + * Build the provider-specific message for a set of recipients. + * + * @param array $to + */ + private function buildMessage( + array $to, + Document $message, + Document $provider, + string $providerType, + Database $dbForProject, + Device $deviceForFiles, + Document $project + ): Email|SMS|Push { + $messageData = clone $message; + $messageData->setAttribute('to', $to); + + return match ($providerType) { + MESSAGE_TYPE_SMS => $this->buildSmsMessage($messageData, $provider), + MESSAGE_TYPE_PUSH => $this->buildPushMessage($messageData), + MESSAGE_TYPE_EMAIL => $this->buildEmailMessage($dbForProject, $messageData, $provider, $deviceForFiles, $project), + default => throw new \Exception('Provider with the requested ID is of the incorrect type') + }; + } + private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void { if ($this->adapter === null) { @@ -451,7 +807,7 @@ class Messaging extends Action } - private function getSmsAdapter(Document $provider): ?SMSAdapter + protected function getSmsAdapter(Document $provider): ?SMSAdapter { $credentials = $provider->getAttribute('credentials'); @@ -494,7 +850,7 @@ class Messaging extends Action }; } - private function getPushAdapter(Document $provider): ?PushAdapter + protected function getPushAdapter(Document $provider): ?PushAdapter { $credentials = $provider->getAttribute('credentials'); $options = $provider->getAttribute('options'); @@ -513,7 +869,7 @@ class Messaging extends Action }; } - private function getEmailAdapter(Document $provider): ?EmailAdapter + protected function getEmailAdapter(Document $provider): ?EmailAdapter { $credentials = $provider->getAttribute('credentials', []); $options = $provider->getAttribute('options', []); @@ -537,6 +893,12 @@ class Messaging extends Action ), 'sendgrid' => new Sendgrid($apiKey), 'resend' => new Resend($apiKey), + 'ses' => new SES( + $credentials['accessKey'] ?? '', + $credentials['secretKey'] ?? '', + $credentials['region'] ?? '', + $credentials['sessionToken'] ?? null, + ), default => null }; } @@ -709,11 +1071,9 @@ class Messaging extends Action private function getLocalDevice($project): Local { - if ($this->localDevice === null) { - $this->localDevice = new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); - } - - return $this->localDevice; + // Not cached: the path is project-scoped and the worker handles + // messages from many projects (and coroutines run them concurrently). + return new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); } private function createInternalSMSAdapter(): ?SMSAdapter diff --git a/tests/e2e/Services/Messaging/MessagingBase.php b/tests/e2e/Services/Messaging/MessagingBase.php index 43dc58d44c..c291dd94c9 100644 --- a/tests/e2e/Services/Messaging/MessagingBase.php +++ b/tests/e2e/Services/Messaging/MessagingBase.php @@ -1047,6 +1047,95 @@ trait MessagingBase } } + public function testSesProvider(): void + { + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]; + + // Create with full credentials but no fromEmail, so the provider stays disabled. + $response = $this->client->call(Client::METHOD_POST, '/messaging/providers/ses', $headers, [ + 'providerId' => ID::unique(), + 'name' => 'SES1', + 'accessKey' => 'my-access-key', + 'secretKey' => 'my-secret-key', + 'region' => 'us-east-1', + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals('SES1', $response['body']['name']); + $this->assertEquals('ses', $response['body']['provider']); + $this->assertEquals('email', $response['body']['type']); + $this->assertEquals(false, $response['body']['enabled']); + $this->assertEquals('my-access-key', $response['body']['credentials']['accessKey']); + $this->assertEquals('my-secret-key', $response['body']['credentials']['secretKey']); + $this->assertEquals('us-east-1', $response['body']['credentials']['region']); + $this->assertArrayHasKey('fromEmail', $response['body']['options']); + $this->assertArrayNotHasKey('accessKey', $response['body']['options']); + + $providerId = $response['body']['$id']; + + // Create enabled: all credentials plus fromEmail present. + $enabledResponse = $this->client->call(Client::METHOD_POST, '/messaging/providers/ses', $headers, [ + 'providerId' => ID::unique(), + 'name' => 'SES-enabled', + 'accessKey' => 'my-access-key', + 'secretKey' => 'my-secret-key', + 'region' => 'us-east-1', + 'fromName' => 'Sender Name', + 'fromEmail' => 'sender-email@my-domain.com', + 'enabled' => true, + ]); + + $this->assertEquals(201, $enabledResponse['headers']['status-code']); + $this->assertEquals(true, $enabledResponse['body']['enabled']); + $this->assertEquals('sender-email@my-domain.com', $enabledResponse['body']['options']['fromEmail']); + + // Sparse update: change only the region, credentials must be preserved. + $updateResponse = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/ses/' . $providerId, $headers, [ + 'name' => 'SES2', + 'region' => 'eu-west-1', + ]); + + $this->assertEquals(200, $updateResponse['headers']['status-code']); + $this->assertEquals('SES2', $updateResponse['body']['name']); + $this->assertEquals('eu-west-1', $updateResponse['body']['credentials']['region']); + $this->assertEquals('my-access-key', $updateResponse['body']['credentials']['accessKey']); + $this->assertEquals('my-secret-key', $updateResponse['body']['credentials']['secretKey']); + + // Regression: enabling while fromEmail is still empty must be rejected, not silently enabled. + $enableWithoutFromEmail = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/ses/' . $providerId, $headers, [ + 'enabled' => true, + ]); + + $this->assertEquals(400, $enableWithoutFromEmail['headers']['status-code']); + $this->assertEquals('provider_missing_credentials', $enableWithoutFromEmail['body']['type']); + + // Enable the first provider once fromEmail is supplied. + $enableResponse = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/ses/' . $providerId, $headers, [ + 'fromEmail' => 'sender-email@my-domain.com', + 'enabled' => true, + ]); + + $this->assertEquals(200, $enableResponse['headers']['status-code']); + $this->assertEquals(true, $enableResponse['body']['enabled']); + + // Updating an SES provider through a different provider route is rejected. + $wrongTypeResponse = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/sendgrid/' . $providerId, $headers, [ + 'name' => 'Wrong', + ]); + + $this->assertEquals(400, $wrongTypeResponse['headers']['status-code']); + + $deleteResponse = $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, $headers); + $this->assertEquals(204, $deleteResponse['headers']['status-code']); + + $deleteEnabledResponse = $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $enabledResponse['body']['$id'], $headers); + $this->assertEquals(204, $deleteEnabledResponse['headers']['status-code']); + } + public function testCreateTopic(): void { $response1 = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 375e526fcf..f654083d80 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -1289,6 +1289,100 @@ trait StorageBase $this->assertEquals(204, $deleteBucketResponse['headers']['status-code']); } + public function testCreateBucketFileChunkedUploadWithoutReadPermission(): void + { + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket Chunked Upload No Read Permission', + 'fileSecurity' => true, + 'permissions' => [ + Permission::create(Role::any()), + ], + ]); + + $this->assertEquals(201, $bucket['headers']['status-code']); + + $bucketId = $bucket['body']['$id']; + $source = __DIR__ . "/../../../resources/disk-a/large-file.mp4"; + $totalSize = \filesize($source); + $chunkSize = 5 * 1024 * 1024; + $mimeType = mime_content_type($source); + $fileId = ID::unique(); + $uploadedFileId = null; + $chunksTotal = (int) ceil($totalSize / $chunkSize); + + $this->assertGreaterThan($chunkSize, $totalSize, 'Test file must span at least 2 chunks'); + + try { + $handle = fopen($source, 'rb'); + $this->assertNotFalse($handle, "Could not open test resource: $source"); + + $permissions = [ + Permission::delete(Role::user($this->getUser()['$id'])), + ]; + $upload = null; + + for ($chunk = 0; $chunk < $chunksTotal; $chunk++) { + $start = $chunk * $chunkSize; + $chunkData = fread($handle, min($chunkSize, $totalSize - $start)); + $end = $start + strlen($chunkData) - 1; + + $headers = [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'content-range' => 'bytes ' . $start . '-' . $end . '/' . $totalSize, + ]; + + if (!empty($uploadedFileId)) { + $headers['x-appwrite-id'] = $uploadedFileId; + } + + $upload = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge($headers, $this->getHeaders()), [ + 'fileId' => $fileId, + 'file' => new CURLFile('data://' . $mimeType . ';base64,' . base64_encode($chunkData), $mimeType, 'large-file.mp4'), + 'permissions' => $permissions, + ]); + + $this->assertEquals(201, $upload['headers']['status-code'], $upload['body']['message'] ?? ''); + $this->assertEquals($chunk + 1, $upload['body']['chunksUploaded']); + + $uploadedFileId ??= $upload['body']['$id']; + } + + fclose($handle); + + if ($upload === null) { + $this->fail('Expected at least one chunk upload response.'); + } + + $this->assertNotContains(Permission::read(Role::user($this->getUser()['$id'])), $upload['body']['$permissions']); + $this->assertEquals($chunksTotal, $upload['body']['chunksTotal']); + $this->assertEquals($chunksTotal, $upload['body']['chunksUploaded']); + } finally { + if (isset($handle) && \is_resource($handle)) { + fclose($handle); + } + + if (!empty($uploadedFileId)) { + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $uploadedFileId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + } + public function testCreateBucketFileOutOfOrder(): void { // Create a bucket for this test diff --git a/tests/unit/Messaging/MessagingFanoutTest.php b/tests/unit/Messaging/MessagingFanoutTest.php new file mode 100644 index 0000000000..02b1ee55d4 --- /dev/null +++ b/tests/unit/Messaging/MessagingFanoutTest.php @@ -0,0 +1,808 @@ + 'provider1', + '$sequence' => '1', + 'enabled' => true, + 'type' => MESSAGE_TYPE_EMAIL, + 'provider' => 'mock', + 'name' => 'Mock Email', + 'credentials' => [], + 'options' => [ + 'fromName' => 'Appwrite', + 'fromEmail' => 'noreply@example.com', + ], + ]); + } + + private function message(): Document + { + return new Document([ + '$id' => 'message1', + '$sequence' => '1', + 'providerType' => MESSAGE_TYPE_EMAIL, + 'topics' => ['topic1'], + 'targets' => [], + 'users' => [], + 'data' => [ + 'subject' => 'Hello', + 'content' => 'World', + 'html' => false, + ], + ]); + } + + /** + * Invoke the worker's private {@see Messaging::retrySend()} for an email batch. + * + * @param array $batch + * @return array{delivered: int, errors: array} + */ + private function retrySend(array $batch, EmailAdapter $adapter, Database $database): array + { + $worker = new TestableMessaging(); + $method = new \ReflectionMethod(Messaging::class, 'retrySend'); + + $device = new Local(\sys_get_temp_dir()); + $project = new Document(['$id' => 'project1', '$sequence' => '1']); + + /** @var array{delivered: int, errors: array} $result */ + $result = $method->invoke( + $worker, + $batch, + $this->message(), + $this->provider(), + MESSAGE_TYPE_EMAIL, + $adapter, + $database, + $device, + $project + ); + + return $result; + } + + public function testIsRetryableErrorMatchesTransientFailures(): void + { + $worker = new TestableMessaging(); + $method = new \ReflectionMethod(Messaging::class, 'isRetryableError'); + + $retryable = [ + 'Throttling', + 'rate exceeded', + 'Rate limit reached', + 'Too Many Requests', + '429 Too Many Requests', + 'quota exceeded', + 'Service Unavailable', + '503 Service Unavailable', + 'Connection timed out', + 'Request timeout', + 'Service temporarily unavailable', + ]; + + foreach ($retryable as $error) { + $this->assertTrue( + $method->invoke($worker, $error), + "Expected '{$error}' to be classified as retryable" + ); + } + + $permanent = [ + 'invalid recipient', + 'Mailbox does not exist', + 'Authentication failed', + 'Expired device token', + ]; + + foreach ($permanent as $error) { + $this->assertFalse( + $method->invoke($worker, $error), + "Expected '{$error}' to be classified as permanent" + ); + } + + $this->assertFalse($method->invoke($worker, null), 'A null error must never be retryable'); + $this->assertFalse($method->invoke($worker, ''), 'An empty error must never be retryable'); + } + + public function testThrottledBatchIsRetriedThenFullyDelivered(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com', 'user4@example.com', 'user5@example.com']; + + // First attempt throttles every recipient; the script then runs out so the second attempt succeeds. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): string => 'Rate limit exceeded, please retry', + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // Retried exactly once, then everyone delivered with no double-counting and no terminal errors. + $this->assertSame(2, $adapter->sendCalls, 'Throttled batch must be retried once'); + $this->assertSame(\count($batch), $result['delivered']); + $this->assertEmpty($result['errors']); + + // Both attempts targeted the full recipient set (none had succeeded on the first attempt). + $this->assertSame($batch, $adapter->recipientsPerCall[0]); + $this->assertSame($batch, $adapter->recipientsPerCall[1]); + } + + public function testRetryTargetsOnlyThrottledRecipients(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com', 'user4@example.com', 'user5@example.com']; + + // user2@ and user4@ throttle on the first attempt; everyone else succeeds. The second attempt has no + // script step, so the two retried recipients then succeed. + $throttled = ['user2@example.com', 'user4@example.com']; + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => \in_array($recipient, $throttled, true) + ? 'Too Many Requests' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + $this->assertSame(2, $adapter->sendCalls, 'A partially throttled batch must be retried once'); + + // The retry must re-send to EXACTLY the throttled recipients — never the ones already delivered. + $this->assertSame($throttled, $adapter->recipientsPerCall[1]); + + // Every recipient is ultimately delivered and counted exactly once. + $this->assertSame(\count($batch), $result['delivered']); + $this->assertEmpty($result['errors']); + } + + public function testNonRetryableFailureIsNotRetried(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com']; + + // user1@ permanently fails with a non-retryable error; the rest succeed. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => $recipient === 'user1@example.com' + ? 'Invalid recipient address' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // A permanent failure is recorded immediately, never retried. + $this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry'); + $this->assertSame(\count($batch) - 1, $result['delivered']); + $this->assertCount(1, $result['errors']); + $this->assertStringContainsString('user1@example.com', $result['errors'][0]); + $this->assertStringContainsString('Invalid recipient address', $result['errors'][0]); + } + + public function testExhaustedRetriesRecordTerminalFailure(): void + { + $batch = ['user1@example.com', 'user2@example.com']; + + // Every attempt throttles, for more steps than MESSAGE_SEND_MAX_RETRIES, so retries are exhausted. + $script = \array_fill( + 0, + MESSAGE_SEND_MAX_RETRIES + 2, + static fn (string $recipient): string => 'Service unavailable, throttled' + ); + $adapter = new ScriptedEmailAdapter($script); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // The batch is attempted exactly MESSAGE_SEND_MAX_RETRIES times, then the still-throttled recipients + // become terminal failures with nothing delivered. + $this->assertSame(MESSAGE_SEND_MAX_RETRIES, $adapter->sendCalls, 'Attempts must be capped at MESSAGE_SEND_MAX_RETRIES'); + $this->assertSame(0, $result['delivered']); + $this->assertCount(\count($batch), $result['errors']); + } + + public function testDeliveryErrorsAreCappedWhileDeliveredCountIsExact(): void + { + // More failing recipients than the retained-error cap, plus a known number of successes, so the + // delivered tally stays exact even though the error list is truncated. + $failingCount = MESSAGE_DELIVERY_ERRORS_LIMIT + 20; + $succeedingCount = 100; + + $batch = []; + for ($i = 1; $i <= $failingCount; $i++) { + $batch[] = "fail{$i}@example.com"; + } + for ($i = 1; $i <= $succeedingCount; $i++) { + $batch[] = "ok{$i}@example.com"; + } + + // Permanent (non-retryable) failure for every "fail*" recipient; everyone else is delivered. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => \str_starts_with($recipient, 'fail') + ? 'Invalid recipient address' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, new InMemoryDatabase()); + + // Errors are capped, but the delivered count reflects every success exactly. + $this->assertCount(MESSAGE_DELIVERY_ERRORS_LIMIT, $result['errors'], 'Delivery errors must be capped'); + $this->assertSame($succeedingCount, $result['delivered'], 'Delivered count stays exact despite the error cap'); + $this->assertSame(1, $adapter->sendCalls, 'Non-retryable failures must not trigger a retry'); + } + + public function testExpiredTokenCleanupFailureDoesNotCorruptAccounting(): void + { + $batch = ['user1@example.com', 'user2@example.com', 'user3@example.com']; + + // The cleanup path always throws, but the worker must isolate it: the send result is what's accounted. + $database = new ExpiredTokenCleanupFailingDatabase(); + + // user1@ comes back with an expired device token (triggering cleanup, which then throws); the other + // two recipients are delivered. The expired-token error is terminal, never retryable. + $adapter = new ScriptedEmailAdapter([ + static fn (string $recipient): ?string => $recipient === 'user1@example.com' + ? 'Expired device token' + : null, + ]); + + $result = $this->retrySend($batch, $adapter, $database); + + // The worker reached the cleanup (so the isolation is genuinely exercised) and it threw. + $this->assertGreaterThan(0, $database->targetUpdateAttempts, 'Expired-token cleanup must have been attempted'); + + // The send was attempted exactly once: a thrown cleanup must not look like a transient send failure + // and must not trigger a retry. + $this->assertSame(1, $adapter->sendCalls, 'Cleanup failure must not trigger a send retry'); + + // Accounting reflects the send result, not the cleanup throw: two delivered, one terminal failure. + $this->assertSame(\count($batch) - 1, $result['delivered']); + + // The recorded error is the provider's expired-token failure for the real recipient — proving the + // database exception was swallowed rather than misattributed as the send error. + $this->assertCount(1, $result['errors']); + $this->assertStringContainsString('user1@example.com', $result['errors'][0]); + $this->assertStringContainsString('Expired device token', $result['errors'][0]); + $this->assertStringNotContainsString('Transient database error', $result['errors'][0]); + } + + public function testRecipientsArePaginatedWithoutSubqueryFilter(): void + { + // Always span several pages regardless of the configured page size, so cursor pagination is exercised. + $count = MESSAGE_RECIPIENTS_PAGE_SIZE * 2 + 500; + $data = $this->topicDataset($count); + $database = new RecordingDatabase($data['subscribers'], $data['targets']); + + $identifiers = $this->collectStream($database, topicIds: ['topic1']); + + $topicReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics')); + $subscriberReads = \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'subscribers')); + + // The topic's `targets` attribute (the subQueryTopicTargets filter) is never materialized: topics are + // only read with a $sequence projection, and recipients come from the subscribers collection instead. + $this->assertNotEmpty($topicReads); + foreach ($topicReads as $call) { + $this->assertNotContains('targets', $this->selectFields($call['queries']), 'Topic read must not select the targets subquery attribute'); + } + + // Subscribers are walked in bounded pages via cursorAfter; no query ever requests a runaway page size. + $expectedPages = (int) \ceil($count / MESSAGE_RECIPIENTS_PAGE_SIZE); + $this->assertCount($expectedPages, $subscriberReads); + + foreach ($subscriberReads as $index => $call) { + $this->assertSame(MESSAGE_RECIPIENTS_PAGE_SIZE, $this->limitOf($call['queries'])); + $this->assertSame('100', $this->topicInternalIdOf($call['queries']), 'Subscribers must be filtered by the topic internal id'); + + if ($index === 0) { + $this->assertFalse($this->hasCursor($call['queries']), 'First page must not carry a cursor'); + } else { + $this->assertTrue($this->hasCursor($call['queries']), 'Subsequent pages must paginate with cursorAfter'); + } + } + + // No query in the whole stream issues a runaway (>= 1,000,000) limit. + foreach ($database->findCalls as $call) { + $this->assertLessThan(1_000_000, $this->limitOf($call['queries']), 'No query may request a million-row page'); + } + + // The duplicate identifier collapses: one row per distinct subscriber identifier resolved. + $this->assertCount($count, $identifiers); + for ($i = 1; $i <= $count; $i++) { + $this->assertArrayHasKey("user{$i}@example.com", $identifiers); + } + } + + public function testStreamDeduplicatesIdentifiersWithinPage(): void + { + // Two subscribers in a single page resolve to targets sharing one identifier; the page must collapse + // them to a single recipient entry. + $subscribers = [ + new Document(['$id' => 'sub1', '$sequence' => '1', 'topicInternalId' => '100', 'targetInternalId' => '1', 'providerType' => MESSAGE_TYPE_EMAIL]), + new Document(['$id' => 'sub2', '$sequence' => '2', 'topicInternalId' => '100', 'targetInternalId' => '2', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $targets = [ + '1' => new Document(['$id' => 'target1', '$sequence' => '1', 'providerId' => null, 'identifier' => 'dup@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + '2' => new Document(['$id' => 'target2', '$sequence' => '2', 'providerId' => null, 'identifier' => 'dup@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $database = new RecordingDatabase($subscribers, $targets); + + $pages = $this->collectPages($database, topicIds: ['topic1']); + + $this->assertCount(1, $pages, 'A single subscriber page yields a single grouped result'); + $providerId = $this->provider()->getId(); + $this->assertArrayHasKey($providerId, $pages[0]); + $this->assertSame(['dup@example.com'], \array_keys($pages[0][$providerId]), 'Duplicate identifiers must collapse to one'); + } + + public function testStreamResolvesUsersAndDirectTargets(): void + { + $userTargets = [ + new Document(['$id' => 'utarget1', '$sequence' => '10', 'providerId' => null, 'identifier' => 'byuser@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $directTargets = [ + new Document(['$id' => 'dtarget1', '$sequence' => '20', 'providerId' => null, 'identifier' => 'bytarget@example.com', 'providerType' => MESSAGE_TYPE_EMAIL]), + ]; + $database = new RecordingDatabase([], [], $userTargets, $directTargets); + + $identifiers = $this->collectStream($database, userIds: ['user-a'], targetIds: ['target-b']); + + // No topics in play: only the user and direct-target queries run, and both identifiers resolve. + $this->assertSame([], \array_values(\array_filter($database->findCalls, fn ($call) => $call['collection'] === 'topics'))); + $this->assertArrayHasKey('byuser@example.com', $identifiers); + $this->assertArrayHasKey('bytarget@example.com', $identifiers); + $this->assertCount(2, $identifiers); + } + + /** + * Build a topic-backed dataset: $count subscribers, each pointing at one email target. A single identifier + * is duplicated so per-page dedup is exercised when the dataset fits in one page. + * + * @return array{subscribers: array, targets: array} + */ + private function topicDataset(int $count): array + { + $subscribers = []; + $targets = []; + + for ($i = 1; $i <= $count; $i++) { + $sequence = (string) $i; + $subscribers[] = new Document([ + '$id' => 'sub' . $i, + '$sequence' => $sequence, + 'topicInternalId' => '100', + 'targetInternalId' => $sequence, + 'providerType' => MESSAGE_TYPE_EMAIL, + ]); + $targets[$sequence] = new Document([ + '$id' => 'target' . $i, + '$sequence' => $sequence, + 'providerId' => null, + 'identifier' => "user{$i}@example.com", + 'providerType' => MESSAGE_TYPE_EMAIL, + ]); + } + + return ['subscribers' => $subscribers, 'targets' => $targets]; + } + + /** + * Drive {@see Messaging::streamRecipients()} and merge every yielded page into a single identifier map. + * + * @param array $topicIds + * @param array $userIds + * @param array $targetIds + * @return array + */ + private function collectStream( + RecordingDatabase $database, + array $topicIds = [], + array $userIds = [], + array $targetIds = [] + ): array { + $identifiers = []; + + foreach ($this->collectPages($database, $topicIds, $userIds, $targetIds) as $page) { + foreach ($page as $perProvider) { + foreach ($perProvider as $identifier => $_) { + $identifiers[$identifier] = null; + } + } + } + + return $identifiers; + } + + /** + * @param array $topicIds + * @param array $userIds + * @param array $targetIds + * @return array>> + */ + private function collectPages( + RecordingDatabase $database, + array $topicIds = [], + array $userIds = [], + array $targetIds = [] + ): array { + $method = new \ReflectionMethod(Messaging::class, 'streamRecipients'); + $worker = new TestableMessaging(); + + /** @var \Generator>> $generator */ + $generator = $method->invoke( + $worker, + $database, + $topicIds, + $userIds, + $targetIds, + MESSAGE_TYPE_EMAIL, + $this->provider() + ); + + return \iterator_to_array($generator, false); + } + + /** + * @param array $queries + * @return array + */ + private function selectFields(array $queries): array + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_SELECT) { + return $query->getValues(); + } + } + + return []; + } + + /** + * @param array $queries + */ + private function limitOf(array $queries): int + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_LIMIT) { + return (int) ($query->getValues()[0] ?? 0); + } + } + + return 0; + } + + /** + * @param array $queries + */ + private function hasCursor(array $queries): bool + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) { + return true; + } + } + + return false; + } + + /** + * @param array $queries + */ + private function topicInternalIdOf(array $queries): ?string + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'topicInternalId') { + return (string) ($query->getValues()[0] ?? null); + } + } + + return null; + } +} + +/** + * Worker subclass that zeroes the retry backoff so the throttle/retry tests stay instant. The worker already + * skips a non-positive delay, so no coroutine sleep is ever reached. + */ +class TestableMessaging extends Messaging +{ + protected function retryDelay(): float + { + return 0.0; + } +} + +/** + * Email adapter double whose per-attempt outcome is scripted. Each {@see process()} call records the exact + * `to` recipients it received and consults the next scripted step, which either throws (simulating a + * connection-level provider error) or returns a per-recipient success/failure verdict. When the script runs + * out, every recipient succeeds — modelling a provider that has stopped throttling. + */ +class ScriptedEmailAdapter extends EmailAdapter +{ + public int $sendCalls = 0; + + /** + * @var array> Recipient email lists captured per {@see process()} call, in call order. + */ + public array $recipientsPerCall = []; + + /** + * @param array<\Throwable|(callable(string): ?string)> $script One step per attempt. A Throwable is thrown + * for that attempt; a callable returns an error string for a failing recipient or null for success. + */ + public function __construct(private readonly array $script = []) + { + } + + public function getName(): string + { + return 'ScriptedEmail'; + } + + public function getMaxMessagesPerRequest(): int + { + return 1000; + } + + /** + * @param EmailMessage $message + * @return array{deliveredTo: int, type: string, results: array>} + */ + protected function process(EmailMessage $message): array + { + $recipients = \array_map(static fn (array $entry): string => $entry['email'], $message->getTo()); + $this->recipientsPerCall[] = $recipients; + + $step = $this->script[$this->sendCalls] ?? null; + $this->sendCalls++; + + if ($step instanceof \Throwable) { + throw $step; + } + + $response = new Response($this->getType()); + $delivered = 0; + + foreach ($recipients as $recipient) { + $error = \is_callable($step) ? $step($recipient) : null; + + if ($error === null) { + $delivered++; + $response->addResult($recipient); + } else { + $response->addResult($recipient, $error); + } + } + + $response->setDeliveredTo($delivered); + + return $response->toArray(); + } +} + +/** + * Minimal in-memory {@see Database} double for the retry tests, which only build email messages with no + * attachments and never touch the expired-token cleanup path. Every read resolves empty. + */ +class InMemoryDatabase extends Database +{ + private readonly Authorization $authorization; + + public function __construct() + { + $this->authorization = new Authorization(); + } + + public function getAuthorization(): Authorization + { + return $this->authorization; + } + + public function skipValidation(callable $callback): mixed + { + return $callback(); + } + + public function findOne(string $collection, array $queries = []): Document + { + return new Document(); + } + + public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): Document + { + return new Document(); + } + + public function updateDocument(string $collection, string $id, Document $document): Document + { + return $document; + } + + public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array + { + return []; + } +} + +/** + * {@see InMemoryDatabase} variant whose expired-device-token cleanup always fails: `findOne('targets')` + * resolves to a real target so the worker attempts the update, and `updateDocument('targets')` throws a + * transient error. Used to prove that a DB hiccup during best-effort cleanup never leaks into delivery + * accounting or the retry decision. + */ +class ExpiredTokenCleanupFailingDatabase extends InMemoryDatabase +{ + public int $targetUpdateAttempts = 0; + + public function findOne(string $collection, array $queries = []): Document + { + if ($collection === 'targets') { + return new Document(['$id' => 'target-expired', '$sequence' => '1']); + } + + return parent::findOne($collection, $queries); + } + + public function updateDocument(string $collection, string $id, Document $document): Document + { + if ($collection === 'targets') { + $this->targetUpdateAttempts++; + + throw new \RuntimeException('Transient database error during target cleanup'); + } + + return parent::updateDocument($collection, $id, $document); + } +} + +/** + * In-memory {@see Database} double for the streaming tests. Serves subscribers/targets through cursor + * pagination and records every query issued so the test can assert the streaming contract (cursorAfter + + * bounded page size, topic reads never selecting the 1M-row `targets` subquery, no runaway limits). + */ +class RecordingDatabase extends Database +{ + /** + * @var array}> + */ + public array $findCalls = []; + + private readonly Authorization $authorization; + + /** + * @param array $subscribers Subscriber rows (carry $sequence + targetInternalId + providerType). + * @param array $targets Target rows keyed by $sequence (resolved from a subscriber page). + * @param array $userTargets Targets addressed by userId. + * @param array $directTargets Targets addressed directly by id. + */ + public function __construct( + private readonly array $subscribers = [], + private readonly array $targets = [], + private readonly array $userTargets = [], + private readonly array $directTargets = [] + ) { + $this->authorization = new Authorization(); + } + + public function getAuthorization(): Authorization + { + return $this->authorization; + } + + public function skipValidation(callable $callback): mixed + { + return $callback(); + } + + public function find(string $collection, array $queries = [], string $forPermission = Database::PERMISSION_READ): array + { + $this->findCalls[] = ['collection' => $collection, 'queries' => $queries]; + + return match ($collection) { + 'topics' => [new Document(['$id' => 'topic1', '$sequence' => '100'])], + 'subscribers' => $this->paginate($this->subscribers, $queries), + 'targets' => $this->resolveTargets($queries), + default => [], + }; + } + + /** + * @param array $rows + * @param array $queries + * @return array + */ + private function paginate(array $rows, array $queries): array + { + $limit = $this->limitOf($queries); + $cursor = $this->cursorOf($queries); + + if ($cursor !== null) { + $rows = \array_values(\array_filter( + $rows, + fn (Document $row) => $row->getSequence() > $cursor->getSequence() + )); + } + + return \array_slice($rows, 0, $limit); + } + + /** + * Either resolve a subscriber page's targets by $sequence, or paginate user/direct targets. + * + * @param array $queries + * @return array + */ + private function resolveTargets(array $queries): array + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === '$sequence') { + $sequences = $query->getValues(); + + return \array_values(\array_map( + fn ($sequence) => $this->targets[$sequence], + \array_filter($sequences, fn ($sequence) => isset($this->targets[$sequence])) + )); + } + + if ($query->getMethod() === Query::TYPE_EQUAL && $query->getAttribute() === 'userId') { + return $this->paginate($this->userTargets, $queries); + } + } + + return $this->paginate($this->directTargets, $queries); + } + + /** + * @param array $queries + */ + private function limitOf(array $queries): int + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_LIMIT) { + return (int) ($query->getValues()[0] ?? 0); + } + } + + return 0; + } + + /** + * @param array $queries + */ + private function cursorOf(array $queries): ?Document + { + foreach ($queries as $query) { + if ($query->getMethod() === Query::TYPE_CURSOR_AFTER) { + $value = $query->getValues()[0] ?? null; + + return $value instanceof Document ? $value : null; + } + } + + return null; + } +}