diff --git a/.env b/.env index aa7438f6bb..57deb8cc0a 100644 --- a/.env +++ b/.env @@ -150,3 +150,6 @@ _APP_POOL_ADAPTER=stack _APP_WORKER_SCREENSHOTS_ROUTER=http://appwrite _TESTS_OAUTH2_GITHUB_CLIENT_ID= _TESTS_OAUTH2_GITHUB_CLIENT_SECRET= + +_APP_WORKERS_NUM=1 +_APP_WORKER_MAX_COROUTINES=1 diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 26f6bf21c4..8665fbf24c 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -236,6 +236,100 @@ Http::post('/v1/messaging/providers/sendgrid') ->dynamic($provider, Response::MODEL_PROVIDER); }); +Http::post('/v1/messaging/providers/ses') + ->desc('Create Amazon SES provider') + ->groups(['api', 'messaging']) + ->label('audits.event', 'provider.create') + ->label('audits.resource', 'provider/{response.$id}') + ->label('event', 'providers.[providerId].create') + ->label('scope', 'providers.write') + ->label('resourceType', RESOURCE_TYPE_PROVIDERS) + ->label('sdk', new Method( + namespace: 'messaging', + group: 'providers', + name: 'createSesProvider', + description: '/docs/references/messaging/create-ses-provider.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_CREATED, + model: Response::MODEL_PROVIDER, + ) + ] + )) + ->param('providerId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Provider ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', false, ['dbForProject']) + ->param('name', '', new Text(128), 'Provider name.') + ->param('accessKey', '', new Text(0), 'AWS access key ID.', true) + ->param('secretKey', '', new Text(0), 'AWS secret access key.', true) + ->param('region', '', new Text(128), 'AWS region, for example us-east-1.', true) + ->param('fromName', '', new Text(128, 0), 'Sender Name.', true) + ->param('fromEmail', '', new Email(), 'Sender email address.', true) + ->param('replyToName', '', new Text(128, 0), 'Name set in the reply to field for the mail. Default value is sender name.', true) + ->param('replyToEmail', '', new Email(), 'Email set in the reply to field for the mail. Default value is sender email.', true) + ->param('enabled', null, new Nullable(new Boolean()), 'Set as enabled.', true) + ->inject('queueForEvents') + ->inject('dbForProject') + ->inject('response') + ->action(function (string $providerId, string $name, string $accessKey, string $secretKey, string $region, string $fromName, string $fromEmail, string $replyToName, string $replyToEmail, ?bool $enabled, Event $queueForEvents, Database $dbForProject, Response $response) { + $providerId = $providerId == 'unique()' ? ID::unique() : $providerId; + + $credentials = []; + + if (!empty($accessKey)) { + $credentials['accessKey'] = $accessKey; + } + + if (!empty($secretKey)) { + $credentials['secretKey'] = $secretKey; + } + + if (!empty($region)) { + $credentials['region'] = $region; + } + + $options = [ + 'fromName' => $fromName, + 'fromEmail' => $fromEmail, + 'replyToName' => $replyToName, + 'replyToEmail' => $replyToEmail, + ]; + + if ( + $enabled === true + && !empty($fromEmail) + && !empty($credentials['accessKey']) + && !empty($credentials['secretKey']) + && !empty($credentials['region']) + ) { + $enabled = true; + } else { + $enabled = false; + } + + $provider = new Document([ + '$id' => $providerId, + 'name' => $name, + 'provider' => 'ses', + 'type' => MESSAGE_TYPE_EMAIL, + 'enabled' => $enabled, + 'credentials' => $credentials, + 'options' => $options, + ]); + + try { + $provider = $dbForProject->createDocument('providers', $provider); + } catch (DuplicateException) { + throw new Exception(Exception::PROVIDER_ALREADY_EXISTS); + } + + $queueForEvents + ->setParam('providerId', $provider->getId()); + + $response + ->setStatusCode(Response::STATUS_CODE_CREATED) + ->dynamic($provider, Response::MODEL_PROVIDER); + }); + Http::post('/v1/messaging/providers/resend') ->desc('Create Resend provider') ->groups(['api', 'messaging']) @@ -1458,6 +1552,119 @@ Http::patch('/v1/messaging/providers/sendgrid/:providerId') ->dynamic($provider, Response::MODEL_PROVIDER); }); +Http::patch('/v1/messaging/providers/ses/:providerId') + ->desc('Update Amazon SES provider') + ->groups(['api', 'messaging']) + ->label('audits.event', 'provider.update') + ->label('audits.resource', 'provider/{response.$id}') + ->label('event', 'providers.[providerId].update') + ->label('scope', 'providers.write') + ->label('resourceType', RESOURCE_TYPE_PROVIDERS) + ->label('sdk', new Method( + namespace: 'messaging', + group: 'providers', + name: 'updateSesProvider', + description: '/docs/references/messaging/update-ses-provider.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_PROVIDER, + ) + ] + )) + ->param('providerId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Provider ID.', false, ['dbForProject']) + ->param('name', '', new Text(128), 'Provider name.', true) + ->param('enabled', null, new Nullable(new Boolean()), 'Set as enabled.', true) + ->param('accessKey', '', new Text(0), 'AWS access key ID.', true) + ->param('secretKey', '', new Text(0), 'AWS secret access key.', true) + ->param('region', '', new Text(128), 'AWS region, for example us-east-1.', true) + ->param('fromName', '', new Text(128), 'Sender Name.', true) + ->param('fromEmail', '', new Email(), 'Sender email address.', true) + ->param('replyToName', '', new Text(128), 'Name set in the Reply To field for the mail. Default value is Sender Name.', true) + ->param('replyToEmail', '', new Text(128), 'Email set in the Reply To field for the mail. Default value is Sender Email.', true) + ->inject('queueForEvents') + ->inject('dbForProject') + ->inject('response') + ->action(function (string $providerId, string $name, ?bool $enabled, string $accessKey, string $secretKey, string $region, string $fromName, string $fromEmail, string $replyToName, string $replyToEmail, Event $queueForEvents, Database $dbForProject, Response $response) { + $provider = $dbForProject->getDocument('providers', $providerId); + + if ($provider->isEmpty()) { + throw new Exception(Exception::PROVIDER_NOT_FOUND); + } + + $providerAttr = $provider->getAttribute('provider'); + + if ($providerAttr !== 'ses') { + throw new Exception(Exception::PROVIDER_INCORRECT_TYPE); + } + + if (!empty($name)) { + $provider->setAttribute('name', $name); + } + + $options = $provider->getAttribute('options'); + + if (!empty($fromName)) { + $options['fromName'] = $fromName; + } + + if (!empty($fromEmail)) { + $options['fromEmail'] = $fromEmail; + } + + if (!empty($replyToName)) { + $options['replyToName'] = $replyToName; + } + + if (!empty($replyToEmail)) { + $options['replyToEmail'] = $replyToEmail; + } + + $provider->setAttribute('options', $options); + + $credentials = $provider->getAttribute('credentials'); + + if (!empty($accessKey)) { + $credentials['accessKey'] = $accessKey; + } + + if (!empty($secretKey)) { + $credentials['secretKey'] = $secretKey; + } + + if (!empty($region)) { + $credentials['region'] = $region; + } + + $provider->setAttribute('credentials', $credentials); + + if (!\is_null($enabled)) { + if ($enabled) { + if ( + !empty($credentials['accessKey']) && + !empty($credentials['secretKey']) && + !empty($credentials['region']) && + !empty($options['fromEmail']) + ) { + $provider->setAttribute('enabled', true); + } else { + throw new Exception(Exception::PROVIDER_MISSING_CREDENTIALS); + } + } else { + $provider->setAttribute('enabled', false); + } + } + + $provider = $dbForProject->updateDocument('providers', $provider->getId(), $provider); + + $queueForEvents + ->setParam('providerId', $provider->getId()); + + $response + ->dynamic($provider, Response::MODEL_PROVIDER); + }); + Http::patch('/v1/messaging/providers/resend/:providerId') ->desc('Update Resend provider') ->groups(['api', 'messaging']) diff --git a/app/init/registers.php b/app/init/registers.php index 9ea19eee24..50080f0538 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -254,6 +254,10 @@ $register->set('pools', function () { $workerCount = intval(System::getEnv('_APP_CPU_NUM', swoole_cpu_num())) * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); $poolSize = max(1, (int)($instanceConnections / $workerCount)); + // Queue workers consume jobs concurrently with coroutines; each in-flight + // job may hold a connection, so pools must cover the coroutine count. + $poolSize = max($poolSize, (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1)); + foreach ($connections as $key => $connection) { $type = $connection['type']; $multiple = $connection['multiple']; @@ -340,9 +344,13 @@ $register->set('pools', function () { // PubSub workers hold one long-lived subscribed connection and also need // spare capacity for publishes from the same process. - $connectionPoolSize = $type === 'pubsub' - ? max(2, $poolSize) - : $poolSize; + // Consumer pools lease one connection per consume coroutine for the + // lifetime of the worker, plus headroom for queue-size reads. + $connectionPoolSize = match ($type) { + 'pubsub' => max(2, $poolSize), + 'consumer' => $poolSize + 2, + default => $poolSize, + }; $pool = new Pool($poolAdapter, $name, $connectionPoolSize, function () use ($type, $resource, $dsn) { // Get Adapter diff --git a/app/worker.php b/app/worker.php index 169b8e9770..063d4746ac 100644 --- a/app/worker.php +++ b/app/worker.php @@ -83,17 +83,25 @@ if (\str_starts_with($workerName, 'databases')) { /** @var \Utopia\Pools\Group $pools */ $pools = $container->get('pools'); +// Each consume coroutine leases its own broker connection from the pool for +// the lifetime of its consume loop, so concurrent coroutines never share a +// connection. The publisher side backs the queue-depth telemetry gauge. $adapter = new Swoole( - $pools->get('consumer')->pop()->getResource(), + new BrokerPool( + publisher: $pools->get('consumer'), + consumer: $pools->get('consumer'), + ), System::getEnv('_APP_WORKERS_NUM', 1), - $queueName + $queueName, + maxCoroutines: (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1), + resources: $container, ); -$worker = new Server($adapter, $container); +$worker = new Server($adapter); try { $worker->init()->action(function () use ($worker, $registerWorkerMessageResources, $queueName) { - $registerWorkerMessageResources($worker->getContainer()); + $registerWorkerMessageResources($worker->context()); Span::init("worker.{$queueName}"); }); @@ -103,7 +111,7 @@ try { $container->set('bus', function ($register) use ($worker) { return $register->get('bus')->setResolver( - fn (string $name) => $worker->getContainer()->get($name) + fn (string $name) => $worker->context()->get($name) ); }, ['register']); diff --git a/composer.json b/composer.json index ad18771699..b78637ccd9 100644 --- a/composer.json +++ b/composer.json @@ -74,13 +74,13 @@ "utopia-php/locale": "0.8.*", "utopia-php/lock": "0.2.*", "utopia-php/logger": "0.8.*", - "utopia-php/messaging": "0.22.*", + "utopia-php/messaging": "1.0.*", "utopia-php/migration": "1.*", - "utopia-php/platform": "1.0.0-rc3", + "utopia-php/platform": "1.0.0-rc4", "utopia-php/pools": "1.*", "utopia-php/span": "1.1.*", "utopia-php/preloader": "0.2.*", - "utopia-php/queue": "0.18.*", + "utopia-php/queue": "0.19.*", "utopia-php/servers": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/storage": "2.*", diff --git a/composer.lock b/composer.lock index 70c843efe4..b456b34ec7 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ec80b1465e57bc223903fa102249f216", + "content-hash": "0c23c30bba86ad84377d4d1abce0992a", "packages": [ { "name": "adhocore/jwt", @@ -4606,16 +4606,16 @@ }, { "name": "utopia-php/messaging", - "version": "0.22.3", + "version": "1.0.0", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "67366d5f45cc92efe7adb6aab5d6dcd2342f2f9e" + "reference": "1aca8968c3bf3c78c428a399dfe97cbe9b366ec4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/67366d5f45cc92efe7adb6aab5d6dcd2342f2f9e", - "reference": "67366d5f45cc92efe7adb6aab5d6dcd2342f2f9e", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/1aca8968c3bf3c78c428a399dfe97cbe9b366ec4", + "reference": "1aca8968c3bf3c78c428a399dfe97cbe9b366ec4", "shasum": "" }, "require": { @@ -4651,9 +4651,9 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.22.3" + "source": "https://github.com/utopia-php/messaging/tree/1.0.0" }, - "time": "2026-05-19T05:31:20+00:00" + "time": "2026-06-02T14:54:22+00:00" }, { "name": "utopia-php/migration", @@ -4773,16 +4773,16 @@ }, { "name": "utopia-php/platform", - "version": "1.0.0-rc3", + "version": "1.0.0-rc4", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "674fd3cea6d1f8db660b01c5be89e8333dfb6de9" + "reference": "455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/platform/zipball/674fd3cea6d1f8db660b01c5be89e8333dfb6de9", - "reference": "674fd3cea6d1f8db660b01c5be89e8333dfb6de9", + "url": "https://api.github.com/repos/utopia-php/platform/zipball/455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0", + "reference": "455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0", "shasum": "" }, "require": { @@ -4791,7 +4791,7 @@ "php": ">=8.3", "utopia-php/cli": "0.23.*", "utopia-php/http": "^2.0@RC", - "utopia-php/queue": "0.18.*", + "utopia-php/queue": "^0.19", "utopia-php/servers": "0.4.*" }, "require-dev": { @@ -4818,9 +4818,9 @@ ], "support": { "issues": "https://github.com/utopia-php/platform/issues", - "source": "https://github.com/utopia-php/platform/tree/1.0.0-rc3" + "source": "https://github.com/utopia-php/platform/tree/1.0.0-rc4" }, - "time": "2026-05-25T11:48:18+00:00" + "time": "2026-06-02T20:19:26+00:00" }, { "name": "utopia-php/pools", @@ -4976,16 +4976,16 @@ }, { "name": "utopia-php/queue", - "version": "0.18.6", + "version": "0.19.0", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "954cd3788b37697b83fd2757adcd93422f21a846" + "reference": "cd4248af69006b2568c3486a675f27cb21191859" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/954cd3788b37697b83fd2757adcd93422f21a846", - "reference": "954cd3788b37697b83fd2757adcd93422f21a846", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/cd4248af69006b2568c3486a675f27cb21191859", + "reference": "cd4248af69006b2568c3486a675f27cb21191859", "shasum": "" }, "require": { @@ -5037,9 +5037,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.18.6" + "source": "https://github.com/utopia-php/queue/tree/0.19.0" }, - "time": "2026-06-01T19:47:57+00:00" + "time": "2026-06-02T20:12:18+00:00" }, { "name": "utopia-php/registry", diff --git a/docker-compose.yml b/docker-compose.yml index ab51ad32c0..90bea1dc5d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -349,6 +349,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -383,6 +385,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -424,6 +428,8 @@ services: environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -484,6 +490,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -504,7 +512,6 @@ services: - _APP_DB_USER_VECTORSDB - _APP_DB_PASS_VECTORSDB - _APP_LOGGING_CONFIG - - _APP_WORKERS_NUM - _APP_QUEUE_NAME - _APP_DATABASE_SHARED_TABLES @@ -527,6 +534,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -605,6 +614,8 @@ services: - _APP_OPTIONS_FORCE_HTTPS # Basic - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_LOGGING_CONFIG @@ -665,6 +676,8 @@ services: - ./src:/usr/src/code/src environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -705,6 +718,8 @@ services: - ${_APP_DB_HOST:-mongodb} environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -736,6 +751,8 @@ services: - redis environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -782,6 +799,8 @@ services: # - smtp environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -823,6 +842,8 @@ services: - redis environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -883,6 +904,8 @@ services: - ${_APP_DB_HOST:-mongodb} environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1044,6 +1067,8 @@ services: - ${_APP_DB_HOST:-mongodb} environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1078,6 +1103,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 diff --git a/docs/references/messaging/create-ses-provider.md b/docs/references/messaging/create-ses-provider.md new file mode 100644 index 0000000000..c612414f99 --- /dev/null +++ b/docs/references/messaging/create-ses-provider.md @@ -0,0 +1 @@ +Create a new Amazon SES provider. \ No newline at end of file diff --git a/docs/references/messaging/update-ses-provider.md b/docs/references/messaging/update-ses-provider.md new file mode 100644 index 0000000000..79947186e8 --- /dev/null +++ b/docs/references/messaging/update-ses-provider.md @@ -0,0 +1 @@ +Update an Amazon SES provider by its unique ID. \ No newline at end of file diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index 69b27af25c..11f388d3c0 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -260,8 +260,8 @@ class Create extends Action }; try { - $locks($lockKey, 600, function () use ($bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $response, &$completed): void { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + $locks($lockKey, 600, function () use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $response, &$completed): void { + $file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); if (!$file->isEmpty()) { $chunks = $file->getAttribute('chunksTotal', 1); $uploaded = $file->getAttribute('chunksUploaded', 0); @@ -325,7 +325,7 @@ class Create extends Action } $finalizeUpload = function (int $chunksUploaded) use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $mergeUploadMetadata, $path, $permissions, $queueForEvents, $response): void { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + $file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); $uploaded = 0; if (!$file->isEmpty()) { diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 911db304dd..ddd7edf641 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -22,6 +22,7 @@ use Utopia\Messaging\Adapter\Email as EmailAdapter; use Utopia\Messaging\Adapter\Email\Mailgun; use Utopia\Messaging\Adapter\Email\Resend; use Utopia\Messaging\Adapter\Email\Sendgrid; +use Utopia\Messaging\Adapter\Email\SES; use Utopia\Messaging\Adapter\Email\SMTP; use Utopia\Messaging\Adapter\Push\APNS; use Utopia\Messaging\Adapter\Push as PushAdapter; @@ -53,8 +54,6 @@ use function Swoole\Coroutine\batch; class Messaging extends Action { - private ?Local $localDevice = null; - private ?SMSAdapter $adapter = null; public static function getName(): string @@ -894,6 +893,12 @@ class Messaging extends Action ), 'sendgrid' => new Sendgrid($apiKey), 'resend' => new Resend($apiKey), + 'ses' => new SES( + $credentials['accessKey'] ?? '', + $credentials['secretKey'] ?? '', + $credentials['region'] ?? '', + $credentials['sessionToken'] ?? null, + ), default => null }; } @@ -1066,11 +1071,9 @@ class Messaging extends Action private function getLocalDevice($project): Local { - if ($this->localDevice === null) { - $this->localDevice = new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); - } - - return $this->localDevice; + // Not cached: the path is project-scoped and the worker handles + // messages from many projects (and coroutines run them concurrently). + return new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); } private function createInternalSMSAdapter(): ?SMSAdapter diff --git a/tests/e2e/Services/Messaging/MessagingBase.php b/tests/e2e/Services/Messaging/MessagingBase.php index 43dc58d44c..c291dd94c9 100644 --- a/tests/e2e/Services/Messaging/MessagingBase.php +++ b/tests/e2e/Services/Messaging/MessagingBase.php @@ -1047,6 +1047,95 @@ trait MessagingBase } } + public function testSesProvider(): void + { + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]; + + // Create with full credentials but no fromEmail, so the provider stays disabled. + $response = $this->client->call(Client::METHOD_POST, '/messaging/providers/ses', $headers, [ + 'providerId' => ID::unique(), + 'name' => 'SES1', + 'accessKey' => 'my-access-key', + 'secretKey' => 'my-secret-key', + 'region' => 'us-east-1', + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals('SES1', $response['body']['name']); + $this->assertEquals('ses', $response['body']['provider']); + $this->assertEquals('email', $response['body']['type']); + $this->assertEquals(false, $response['body']['enabled']); + $this->assertEquals('my-access-key', $response['body']['credentials']['accessKey']); + $this->assertEquals('my-secret-key', $response['body']['credentials']['secretKey']); + $this->assertEquals('us-east-1', $response['body']['credentials']['region']); + $this->assertArrayHasKey('fromEmail', $response['body']['options']); + $this->assertArrayNotHasKey('accessKey', $response['body']['options']); + + $providerId = $response['body']['$id']; + + // Create enabled: all credentials plus fromEmail present. + $enabledResponse = $this->client->call(Client::METHOD_POST, '/messaging/providers/ses', $headers, [ + 'providerId' => ID::unique(), + 'name' => 'SES-enabled', + 'accessKey' => 'my-access-key', + 'secretKey' => 'my-secret-key', + 'region' => 'us-east-1', + 'fromName' => 'Sender Name', + 'fromEmail' => 'sender-email@my-domain.com', + 'enabled' => true, + ]); + + $this->assertEquals(201, $enabledResponse['headers']['status-code']); + $this->assertEquals(true, $enabledResponse['body']['enabled']); + $this->assertEquals('sender-email@my-domain.com', $enabledResponse['body']['options']['fromEmail']); + + // Sparse update: change only the region, credentials must be preserved. + $updateResponse = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/ses/' . $providerId, $headers, [ + 'name' => 'SES2', + 'region' => 'eu-west-1', + ]); + + $this->assertEquals(200, $updateResponse['headers']['status-code']); + $this->assertEquals('SES2', $updateResponse['body']['name']); + $this->assertEquals('eu-west-1', $updateResponse['body']['credentials']['region']); + $this->assertEquals('my-access-key', $updateResponse['body']['credentials']['accessKey']); + $this->assertEquals('my-secret-key', $updateResponse['body']['credentials']['secretKey']); + + // Regression: enabling while fromEmail is still empty must be rejected, not silently enabled. + $enableWithoutFromEmail = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/ses/' . $providerId, $headers, [ + 'enabled' => true, + ]); + + $this->assertEquals(400, $enableWithoutFromEmail['headers']['status-code']); + $this->assertEquals('provider_missing_credentials', $enableWithoutFromEmail['body']['type']); + + // Enable the first provider once fromEmail is supplied. + $enableResponse = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/ses/' . $providerId, $headers, [ + 'fromEmail' => 'sender-email@my-domain.com', + 'enabled' => true, + ]); + + $this->assertEquals(200, $enableResponse['headers']['status-code']); + $this->assertEquals(true, $enableResponse['body']['enabled']); + + // Updating an SES provider through a different provider route is rejected. + $wrongTypeResponse = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/sendgrid/' . $providerId, $headers, [ + 'name' => 'Wrong', + ]); + + $this->assertEquals(400, $wrongTypeResponse['headers']['status-code']); + + $deleteResponse = $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, $headers); + $this->assertEquals(204, $deleteResponse['headers']['status-code']); + + $deleteEnabledResponse = $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $enabledResponse['body']['$id'], $headers); + $this->assertEquals(204, $deleteEnabledResponse['headers']['status-code']); + } + public function testCreateTopic(): void { $response1 = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 375e526fcf..f654083d80 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -1289,6 +1289,100 @@ trait StorageBase $this->assertEquals(204, $deleteBucketResponse['headers']['status-code']); } + public function testCreateBucketFileChunkedUploadWithoutReadPermission(): void + { + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket Chunked Upload No Read Permission', + 'fileSecurity' => true, + 'permissions' => [ + Permission::create(Role::any()), + ], + ]); + + $this->assertEquals(201, $bucket['headers']['status-code']); + + $bucketId = $bucket['body']['$id']; + $source = __DIR__ . "/../../../resources/disk-a/large-file.mp4"; + $totalSize = \filesize($source); + $chunkSize = 5 * 1024 * 1024; + $mimeType = mime_content_type($source); + $fileId = ID::unique(); + $uploadedFileId = null; + $chunksTotal = (int) ceil($totalSize / $chunkSize); + + $this->assertGreaterThan($chunkSize, $totalSize, 'Test file must span at least 2 chunks'); + + try { + $handle = fopen($source, 'rb'); + $this->assertNotFalse($handle, "Could not open test resource: $source"); + + $permissions = [ + Permission::delete(Role::user($this->getUser()['$id'])), + ]; + $upload = null; + + for ($chunk = 0; $chunk < $chunksTotal; $chunk++) { + $start = $chunk * $chunkSize; + $chunkData = fread($handle, min($chunkSize, $totalSize - $start)); + $end = $start + strlen($chunkData) - 1; + + $headers = [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'content-range' => 'bytes ' . $start . '-' . $end . '/' . $totalSize, + ]; + + if (!empty($uploadedFileId)) { + $headers['x-appwrite-id'] = $uploadedFileId; + } + + $upload = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge($headers, $this->getHeaders()), [ + 'fileId' => $fileId, + 'file' => new CURLFile('data://' . $mimeType . ';base64,' . base64_encode($chunkData), $mimeType, 'large-file.mp4'), + 'permissions' => $permissions, + ]); + + $this->assertEquals(201, $upload['headers']['status-code'], $upload['body']['message'] ?? ''); + $this->assertEquals($chunk + 1, $upload['body']['chunksUploaded']); + + $uploadedFileId ??= $upload['body']['$id']; + } + + fclose($handle); + + if ($upload === null) { + $this->fail('Expected at least one chunk upload response.'); + } + + $this->assertNotContains(Permission::read(Role::user($this->getUser()['$id'])), $upload['body']['$permissions']); + $this->assertEquals($chunksTotal, $upload['body']['chunksTotal']); + $this->assertEquals($chunksTotal, $upload['body']['chunksUploaded']); + } finally { + if (isset($handle) && \is_resource($handle)) { + fclose($handle); + } + + if (!empty($uploadedFileId)) { + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $uploadedFileId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + } + public function testCreateBucketFileOutOfOrder(): void { // Create a bucket for this test