From b28633086bfd305c51ca0615ca8c0ba3f76d7185 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Jun 2026 03:09:45 +1200 Subject: [PATCH 01/14] feat(messaging): add Amazon SES email provider Wire the Amazon SES adapter shipped in utopia-php/messaging 1.0.0 into the messaging service, mirroring the existing Sendgrid/Mailgun providers. - Bump utopia-php/messaging 0.22.* -> 1.0.* (the 1.0.0 src is byte-identical to 0.22.3 apart from the new SES adapter, so the bump is non-breaking) - Add POST/PATCH /v1/messaging/providers/ses routes (createSesProvider / updateSesProvider), storing accessKey/secretKey/region under credentials and fromName/fromEmail/replyToName/replyToEmail under options, with the same enable-guard and sparse-update semantics as the other email providers - Map the 'ses' provider to the SES adapter in the messaging worker - Add SES reference docs and isolated e2e create/update/delete coverage Co-Authored-By: Claude Opus 4.8 (1M context) --- app/controllers/api/messaging.php | 207 ++++++++++++++++++ composer.json | 2 +- composer.lock | 14 +- .../messaging/create-ses-provider.md | 1 + .../messaging/update-ses-provider.md | 1 + src/Appwrite/Platform/Workers/Messaging.php | 7 + .../e2e/Services/Messaging/MessagingBase.php | 81 +++++++ 7 files changed, 305 insertions(+), 8 deletions(-) create mode 100644 docs/references/messaging/create-ses-provider.md create mode 100644 docs/references/messaging/update-ses-provider.md diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 26f6bf21c4..b1adf60c33 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -236,6 +236,100 @@ Http::post('/v1/messaging/providers/sendgrid') ->dynamic($provider, Response::MODEL_PROVIDER); }); +Http::post('/v1/messaging/providers/ses') + ->desc('Create Amazon SES provider') + ->groups(['api', 'messaging']) + ->label('audits.event', 'provider.create') + ->label('audits.resource', 'provider/{response.$id}') + ->label('event', 'providers.[providerId].create') + ->label('scope', 'providers.write') + ->label('resourceType', RESOURCE_TYPE_PROVIDERS) + ->label('sdk', new Method( + namespace: 'messaging', + group: 'providers', + name: 'createSesProvider', + description: '/docs/references/messaging/create-ses-provider.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_CREATED, + model: Response::MODEL_PROVIDER, + ) + ] + )) + ->param('providerId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Provider ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', false, ['dbForProject']) + ->param('name', '', new Text(128), 'Provider name.') + ->param('accessKey', '', new Text(0), 'AWS access key ID.', true) + ->param('secretKey', '', new Text(0), 'AWS secret access key.', true) + ->param('region', '', new Text(128), 'AWS region, for example us-east-1.', true) + ->param('fromName', '', new Text(128, 0), 'Sender Name.', true) + ->param('fromEmail', '', new Email(), 'Sender email address.', true) + ->param('replyToName', '', new Text(128, 0), 'Name set in the reply to field for the mail. Default value is sender name.', true) + ->param('replyToEmail', '', new Email(), 'Email set in the reply to field for the mail. Default value is sender email.', true) + ->param('enabled', null, new Nullable(new Boolean()), 'Set as enabled.', true) + ->inject('queueForEvents') + ->inject('dbForProject') + ->inject('response') + ->action(function (string $providerId, string $name, string $accessKey, string $secretKey, string $region, string $fromName, string $fromEmail, string $replyToName, string $replyToEmail, ?bool $enabled, Event $queueForEvents, Database $dbForProject, Response $response) { + $providerId = $providerId == 'unique()' ? ID::unique() : $providerId; + + $credentials = []; + + if (!empty($accessKey)) { + $credentials['accessKey'] = $accessKey; + } + + if (!empty($secretKey)) { + $credentials['secretKey'] = $secretKey; + } + + if (!empty($region)) { + $credentials['region'] = $region; + } + + $options = [ + 'fromName' => $fromName, + 'fromEmail' => $fromEmail, + 'replyToName' => $replyToName, + 'replyToEmail' => $replyToEmail, + ]; + + if ( + $enabled === true + && !empty($fromEmail) + && \array_key_exists('accessKey', $credentials) + && \array_key_exists('secretKey', $credentials) + && \array_key_exists('region', $credentials) + ) { + $enabled = true; + } else { + $enabled = false; + } + + $provider = new Document([ + '$id' => $providerId, + 'name' => $name, + 'provider' => 'ses', + 'type' => MESSAGE_TYPE_EMAIL, + 'enabled' => $enabled, + 'credentials' => $credentials, + 'options' => $options, + ]); + + try { + $provider = $dbForProject->createDocument('providers', $provider); + } catch (DuplicateException) { + throw new Exception(Exception::PROVIDER_ALREADY_EXISTS); + } + + $queueForEvents + ->setParam('providerId', $provider->getId()); + + $response + ->setStatusCode(Response::STATUS_CODE_CREATED) + ->dynamic($provider, Response::MODEL_PROVIDER); + }); + Http::post('/v1/messaging/providers/resend') ->desc('Create Resend provider') ->groups(['api', 'messaging']) @@ -1458,6 +1552,119 @@ Http::patch('/v1/messaging/providers/sendgrid/:providerId') ->dynamic($provider, Response::MODEL_PROVIDER); }); +Http::patch('/v1/messaging/providers/ses/:providerId') + ->desc('Update Amazon SES provider') + ->groups(['api', 'messaging']) + ->label('audits.event', 'provider.update') + ->label('audits.resource', 'provider/{response.$id}') + ->label('event', 'providers.[providerId].update') + ->label('scope', 'providers.write') + ->label('resourceType', RESOURCE_TYPE_PROVIDERS) + ->label('sdk', new Method( + namespace: 'messaging', + group: 'providers', + name: 'updateSesProvider', + description: '/docs/references/messaging/update-ses-provider.md', + auth: [AuthType::ADMIN, AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_PROVIDER, + ) + ] + )) + ->param('providerId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Provider ID.', false, ['dbForProject']) + ->param('name', '', new Text(128), 'Provider name.', true) + ->param('enabled', null, new Nullable(new Boolean()), 'Set as enabled.', true) + ->param('accessKey', '', new Text(0), 'AWS access key ID.', true) + ->param('secretKey', '', new Text(0), 'AWS secret access key.', true) + ->param('region', '', new Text(128), 'AWS region, for example us-east-1.', true) + ->param('fromName', '', new Text(128), 'Sender Name.', true) + ->param('fromEmail', '', new Email(), 'Sender email address.', true) + ->param('replyToName', '', new Text(128), 'Name set in the Reply To field for the mail. Default value is Sender Name.', true) + ->param('replyToEmail', '', new Text(128), 'Email set in the Reply To field for the mail. Default value is Sender Email.', true) + ->inject('queueForEvents') + ->inject('dbForProject') + ->inject('response') + ->action(function (string $providerId, string $name, ?bool $enabled, string $accessKey, string $secretKey, string $region, string $fromName, string $fromEmail, string $replyToName, string $replyToEmail, Event $queueForEvents, Database $dbForProject, Response $response) { + $provider = $dbForProject->getDocument('providers', $providerId); + + if ($provider->isEmpty()) { + throw new Exception(Exception::PROVIDER_NOT_FOUND); + } + + $providerAttr = $provider->getAttribute('provider'); + + if ($providerAttr !== 'ses') { + throw new Exception(Exception::PROVIDER_INCORRECT_TYPE); + } + + if (!empty($name)) { + $provider->setAttribute('name', $name); + } + + $options = $provider->getAttribute('options'); + + if (!empty($fromName)) { + $options['fromName'] = $fromName; + } + + if (!empty($fromEmail)) { + $options['fromEmail'] = $fromEmail; + } + + if (!empty($replyToName)) { + $options['replyToName'] = $replyToName; + } + + if (!empty($replyToEmail)) { + $options['replyToEmail'] = $replyToEmail; + } + + $provider->setAttribute('options', $options); + + $credentials = $provider->getAttribute('credentials'); + + if (!empty($accessKey)) { + $credentials['accessKey'] = $accessKey; + } + + if (!empty($secretKey)) { + $credentials['secretKey'] = $secretKey; + } + + if (!empty($region)) { + $credentials['region'] = $region; + } + + $provider->setAttribute('credentials', $credentials); + + if (!\is_null($enabled)) { + if ($enabled) { + if ( + \array_key_exists('accessKey', $credentials) && + \array_key_exists('secretKey', $credentials) && + \array_key_exists('region', $credentials) && + \array_key_exists('fromEmail', $options) + ) { + $provider->setAttribute('enabled', true); + } else { + throw new Exception(Exception::PROVIDER_MISSING_CREDENTIALS); + } + } else { + $provider->setAttribute('enabled', false); + } + } + + $provider = $dbForProject->updateDocument('providers', $provider->getId(), $provider); + + $queueForEvents + ->setParam('providerId', $provider->getId()); + + $response + ->dynamic($provider, Response::MODEL_PROVIDER); + }); + Http::patch('/v1/messaging/providers/resend/:providerId') ->desc('Update Resend provider') ->groups(['api', 'messaging']) diff --git a/composer.json b/composer.json index ad18771699..1e3ce344b8 100644 --- a/composer.json +++ b/composer.json @@ -74,7 +74,7 @@ "utopia-php/locale": "0.8.*", "utopia-php/lock": "0.2.*", "utopia-php/logger": "0.8.*", - "utopia-php/messaging": "0.22.*", + "utopia-php/messaging": "1.0.*", "utopia-php/migration": "1.*", "utopia-php/platform": "1.0.0-rc3", "utopia-php/pools": "1.*", diff --git a/composer.lock b/composer.lock index 70c843efe4..11711529ba 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ec80b1465e57bc223903fa102249f216", + "content-hash": "551924e300a646e6f433ea0a0f3d6c9f", "packages": [ { "name": "adhocore/jwt", @@ -4606,16 +4606,16 @@ }, { "name": "utopia-php/messaging", - "version": "0.22.3", + "version": "1.0.0", "source": { "type": "git", "url": "https://github.com/utopia-php/messaging.git", - "reference": "67366d5f45cc92efe7adb6aab5d6dcd2342f2f9e" + "reference": "1aca8968c3bf3c78c428a399dfe97cbe9b366ec4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/messaging/zipball/67366d5f45cc92efe7adb6aab5d6dcd2342f2f9e", - "reference": "67366d5f45cc92efe7adb6aab5d6dcd2342f2f9e", + "url": "https://api.github.com/repos/utopia-php/messaging/zipball/1aca8968c3bf3c78c428a399dfe97cbe9b366ec4", + "reference": "1aca8968c3bf3c78c428a399dfe97cbe9b366ec4", "shasum": "" }, "require": { @@ -4651,9 +4651,9 @@ ], "support": { "issues": "https://github.com/utopia-php/messaging/issues", - "source": "https://github.com/utopia-php/messaging/tree/0.22.3" + "source": "https://github.com/utopia-php/messaging/tree/1.0.0" }, - "time": "2026-05-19T05:31:20+00:00" + "time": "2026-06-02T14:54:22+00:00" }, { "name": "utopia-php/migration", diff --git a/docs/references/messaging/create-ses-provider.md b/docs/references/messaging/create-ses-provider.md new file mode 100644 index 0000000000..c612414f99 --- /dev/null +++ b/docs/references/messaging/create-ses-provider.md @@ -0,0 +1 @@ +Create a new Amazon SES provider. \ No newline at end of file diff --git a/docs/references/messaging/update-ses-provider.md b/docs/references/messaging/update-ses-provider.md new file mode 100644 index 0000000000..79947186e8 --- /dev/null +++ b/docs/references/messaging/update-ses-provider.md @@ -0,0 +1 @@ +Update an Amazon SES provider by its unique ID. \ No newline at end of file diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 7af079e914..759d78f81e 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -21,6 +21,7 @@ use Utopia\Messaging\Adapter\Email as EmailAdapter; use Utopia\Messaging\Adapter\Email\Mailgun; use Utopia\Messaging\Adapter\Email\Resend; use Utopia\Messaging\Adapter\Email\Sendgrid; +use Utopia\Messaging\Adapter\Email\SES; use Utopia\Messaging\Adapter\Email\SMTP; use Utopia\Messaging\Adapter\Push\APNS; use Utopia\Messaging\Adapter\Push as PushAdapter; @@ -537,6 +538,12 @@ class Messaging extends Action ), 'sendgrid' => new Sendgrid($apiKey), 'resend' => new Resend($apiKey), + 'ses' => new SES( + $credentials['accessKey'] ?? '', + $credentials['secretKey'] ?? '', + $credentials['region'] ?? '', + $credentials['sessionToken'] ?? null, + ), default => null }; } diff --git a/tests/e2e/Services/Messaging/MessagingBase.php b/tests/e2e/Services/Messaging/MessagingBase.php index 43dc58d44c..a18fecf5d2 100644 --- a/tests/e2e/Services/Messaging/MessagingBase.php +++ b/tests/e2e/Services/Messaging/MessagingBase.php @@ -1047,6 +1047,87 @@ trait MessagingBase } } + public function testSesProvider(): void + { + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]; + + // Create disabled: missing fromEmail and partial credentials. + $response = $this->client->call(Client::METHOD_POST, '/messaging/providers/ses', $headers, [ + 'providerId' => ID::unique(), + 'name' => 'SES1', + 'accessKey' => 'my-access-key', + 'secretKey' => 'my-secret-key', + 'region' => 'us-east-1', + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + $this->assertEquals('SES1', $response['body']['name']); + $this->assertEquals('ses', $response['body']['provider']); + $this->assertEquals('email', $response['body']['type']); + $this->assertEquals(false, $response['body']['enabled']); + $this->assertEquals('my-access-key', $response['body']['credentials']['accessKey']); + $this->assertEquals('my-secret-key', $response['body']['credentials']['secretKey']); + $this->assertEquals('us-east-1', $response['body']['credentials']['region']); + $this->assertArrayHasKey('fromEmail', $response['body']['options']); + $this->assertArrayNotHasKey('accessKey', $response['body']['options']); + + $providerId = $response['body']['$id']; + + // Create enabled: all credentials plus fromEmail present. + $enabledResponse = $this->client->call(Client::METHOD_POST, '/messaging/providers/ses', $headers, [ + 'providerId' => ID::unique(), + 'name' => 'SES-enabled', + 'accessKey' => 'my-access-key', + 'secretKey' => 'my-secret-key', + 'region' => 'us-east-1', + 'fromName' => 'Sender Name', + 'fromEmail' => 'sender-email@my-domain.com', + 'enabled' => true, + ]); + + $this->assertEquals(201, $enabledResponse['headers']['status-code']); + $this->assertEquals(true, $enabledResponse['body']['enabled']); + $this->assertEquals('sender-email@my-domain.com', $enabledResponse['body']['options']['fromEmail']); + + // Sparse update: change only the region, credentials must be preserved. + $updateResponse = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/ses/' . $providerId, $headers, [ + 'name' => 'SES2', + 'region' => 'eu-west-1', + ]); + + $this->assertEquals(200, $updateResponse['headers']['status-code']); + $this->assertEquals('SES2', $updateResponse['body']['name']); + $this->assertEquals('eu-west-1', $updateResponse['body']['credentials']['region']); + $this->assertEquals('my-access-key', $updateResponse['body']['credentials']['accessKey']); + $this->assertEquals('my-secret-key', $updateResponse['body']['credentials']['secretKey']); + + // Enable the first provider once fromEmail is supplied. + $enableResponse = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/ses/' . $providerId, $headers, [ + 'fromEmail' => 'sender-email@my-domain.com', + 'enabled' => true, + ]); + + $this->assertEquals(200, $enableResponse['headers']['status-code']); + $this->assertEquals(true, $enableResponse['body']['enabled']); + + // Updating an SES provider through a different provider route is rejected. + $wrongTypeResponse = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/sendgrid/' . $providerId, $headers, [ + 'name' => 'Wrong', + ]); + + $this->assertEquals(400, $wrongTypeResponse['headers']['status-code']); + + $deleteResponse = $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, $headers); + $this->assertEquals(204, $deleteResponse['headers']['status-code']); + + $deleteEnabledResponse = $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $enabledResponse['body']['$id'], $headers); + $this->assertEquals(204, $deleteEnabledResponse['headers']['status-code']); + } + public function testCreateTopic(): void { $response1 = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ From 1bfb0b115c2b09ba3230c465f858c216af3f2dd8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Jun 2026 03:20:17 +1200 Subject: [PATCH 02/14] fix(messaging): require non-empty SES fromEmail and credentials to enable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The enable-guard used array_key_exists, which is true even for empty-string values — options['fromEmail'] is always stored on create, so a provider could be enabled with an empty sender address. Use !empty() on the values in both the create and update guards. Also corrects a misleading test comment. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/controllers/api/messaging.php | 14 +++++++------- tests/e2e/Services/Messaging/MessagingBase.php | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index b1adf60c33..8665fbf24c 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -297,9 +297,9 @@ Http::post('/v1/messaging/providers/ses') if ( $enabled === true && !empty($fromEmail) - && \array_key_exists('accessKey', $credentials) - && \array_key_exists('secretKey', $credentials) - && \array_key_exists('region', $credentials) + && !empty($credentials['accessKey']) + && !empty($credentials['secretKey']) + && !empty($credentials['region']) ) { $enabled = true; } else { @@ -1642,10 +1642,10 @@ Http::patch('/v1/messaging/providers/ses/:providerId') if (!\is_null($enabled)) { if ($enabled) { if ( - \array_key_exists('accessKey', $credentials) && - \array_key_exists('secretKey', $credentials) && - \array_key_exists('region', $credentials) && - \array_key_exists('fromEmail', $options) + !empty($credentials['accessKey']) && + !empty($credentials['secretKey']) && + !empty($credentials['region']) && + !empty($options['fromEmail']) ) { $provider->setAttribute('enabled', true); } else { diff --git a/tests/e2e/Services/Messaging/MessagingBase.php b/tests/e2e/Services/Messaging/MessagingBase.php index a18fecf5d2..163f11ff21 100644 --- a/tests/e2e/Services/Messaging/MessagingBase.php +++ b/tests/e2e/Services/Messaging/MessagingBase.php @@ -1055,7 +1055,7 @@ trait MessagingBase 'x-appwrite-key' => $this->getProject()['apiKey'], ]; - // Create disabled: missing fromEmail and partial credentials. + // Create with full credentials but no fromEmail, so the provider stays disabled. $response = $this->client->call(Client::METHOD_POST, '/messaging/providers/ses', $headers, [ 'providerId' => ID::unique(), 'name' => 'SES1', From 5e07e9fc6e6f83903b606631f6aa7fc10141227e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 3 Jun 2026 03:23:26 +1200 Subject: [PATCH 03/14] test(messaging): cover enabling SES provider with empty fromEmail Regression test for the enable-guard fix: PATCH {enabled:true} on an SES provider whose fromEmail is still empty must return 400 provider_missing_credentials rather than enabling a provider that would fail every send at the AWS API. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/e2e/Services/Messaging/MessagingBase.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/e2e/Services/Messaging/MessagingBase.php b/tests/e2e/Services/Messaging/MessagingBase.php index 163f11ff21..c291dd94c9 100644 --- a/tests/e2e/Services/Messaging/MessagingBase.php +++ b/tests/e2e/Services/Messaging/MessagingBase.php @@ -1105,6 +1105,14 @@ trait MessagingBase $this->assertEquals('my-access-key', $updateResponse['body']['credentials']['accessKey']); $this->assertEquals('my-secret-key', $updateResponse['body']['credentials']['secretKey']); + // Regression: enabling while fromEmail is still empty must be rejected, not silently enabled. + $enableWithoutFromEmail = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/ses/' . $providerId, $headers, [ + 'enabled' => true, + ]); + + $this->assertEquals(400, $enableWithoutFromEmail['headers']['status-code']); + $this->assertEquals('provider_missing_credentials', $enableWithoutFromEmail['body']['type']); + // Enable the first provider once fromEmail is supplied. $enableResponse = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/ses/' . $providerId, $headers, [ 'fromEmail' => 'sender-email@my-domain.com', From ccd1a073b6752b81942714e3eb906024512c897a Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 2 Jun 2026 20:08:26 +0400 Subject: [PATCH 04/14] fix(storage): resume chunked uploads without file read permission --- .../Storage/Http/Buckets/Files/Create.php | 6 +- tests/e2e/Services/Storage/StorageBase.php | 85 +++++++++++++++++++ 2 files changed, 88 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index 69b27af25c..11f388d3c0 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -260,8 +260,8 @@ class Create extends Action }; try { - $locks($lockKey, 600, function () use ($bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $response, &$completed): void { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + $locks($lockKey, 600, function () use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $path, $permissions, $response, &$completed): void { + $file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); if (!$file->isEmpty()) { $chunks = $file->getAttribute('chunksTotal', 1); $uploaded = $file->getAttribute('chunksUploaded', 0); @@ -325,7 +325,7 @@ class Create extends Action } $finalizeUpload = function (int $chunksUploaded) use ($authorization, $bucket, &$chunks, $contentRange, $dbForProject, $deviceForFiles, $fileId, $fileName, $fileSize, &$metadata, $mergeUploadMetadata, $path, $permissions, $queueForEvents, $response): void { - $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); + $file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); $uploaded = 0; if (!$file->isEmpty()) { diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 375e526fcf..96e96f96ea 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -1289,6 +1289,91 @@ trait StorageBase $this->assertEquals(204, $deleteBucketResponse['headers']['status-code']); } + public function testCreateBucketFileChunkedUploadWithoutReadPermission(): void + { + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket Chunked Upload No Read Permission', + 'fileSecurity' => true, + 'permissions' => [ + Permission::create(Role::any()), + ], + ]); + + $this->assertEquals(201, $bucket['headers']['status-code']); + + $bucketId = $bucket['body']['$id']; + $source = __DIR__ . "/../../../resources/disk-a/large-file.mp4"; + $totalSize = \filesize($source); + $chunkSize = 5 * 1024 * 1024; + $mimeType = mime_content_type($source); + $fileId = ID::unique(); + $uploadedFileId = null; + + $this->assertGreaterThan($chunkSize, $totalSize, 'Test file must span at least 2 chunks'); + + try { + $handle = fopen($source, 'rb'); + $this->assertNotFalse($handle, "Could not open test resource: $source"); + + $firstChunk = fread($handle, $chunkSize); + $secondChunk = fread($handle, min($chunkSize, $totalSize - $chunkSize)); + fclose($handle); + + $permissions = [ + Permission::delete(Role::user($this->getUser()['$id'])), + ]; + + $firstUpload = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'content-range' => 'bytes 0-' . ($chunkSize - 1) . '/' . $totalSize, + ], $this->getHeaders()), [ + 'fileId' => $fileId, + 'file' => new CURLFile('data://' . $mimeType . ';base64,' . base64_encode($firstChunk), $mimeType, 'large-file.mp4'), + 'permissions' => $permissions, + ]); + + $this->assertEquals(201, $firstUpload['headers']['status-code']); + $this->assertNotContains(Permission::read(Role::user($this->getUser()['$id'])), $firstUpload['body']['$permissions']); + + $uploadedFileId = $firstUpload['body']['$id']; + $secondChunkEnd = $chunkSize + strlen($secondChunk) - 1; + + $secondUpload = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'content-range' => 'bytes ' . $chunkSize . '-' . $secondChunkEnd . '/' . $totalSize, + 'x-appwrite-id' => $uploadedFileId, + ], $this->getHeaders()), [ + 'fileId' => $fileId, + 'file' => new CURLFile('data://' . $mimeType . ';base64,' . base64_encode($secondChunk), $mimeType, 'large-file.mp4'), + 'permissions' => $permissions, + ]); + + $this->assertEquals(201, $secondUpload['headers']['status-code'], $secondUpload['body']['message'] ?? ''); + $this->assertEquals(2, $secondUpload['body']['chunksUploaded']); + } finally { + if (!empty($uploadedFileId)) { + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $uploadedFileId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + + $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + } + } + public function testCreateBucketFileOutOfOrder(): void { // Create a bucket for this test From 06c722e1e450e7d63bc5f370792e660108372e3c Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 2 Jun 2026 20:16:57 +0400 Subject: [PATCH 05/14] test(storage): cover final chunk upload without read permission --- tests/e2e/Services/Storage/StorageBase.php | 62 ++++++++++++---------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 96e96f96ea..036afdfba4 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -1313,6 +1313,7 @@ trait StorageBase $mimeType = mime_content_type($source); $fileId = ID::unique(); $uploadedFileId = null; + $chunksTotal = (int) ceil($totalSize / $chunkSize); $this->assertGreaterThan($chunkSize, $totalSize, 'Test file must span at least 2 chunks'); @@ -1320,44 +1321,47 @@ trait StorageBase $handle = fopen($source, 'rb'); $this->assertNotFalse($handle, "Could not open test resource: $source"); - $firstChunk = fread($handle, $chunkSize); - $secondChunk = fread($handle, min($chunkSize, $totalSize - $chunkSize)); - fclose($handle); - $permissions = [ Permission::delete(Role::user($this->getUser()['$id'])), ]; - $firstUpload = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ - 'content-type' => 'multipart/form-data', - 'x-appwrite-project' => $this->getProject()['$id'], - 'content-range' => 'bytes 0-' . ($chunkSize - 1) . '/' . $totalSize, - ], $this->getHeaders()), [ - 'fileId' => $fileId, - 'file' => new CURLFile('data://' . $mimeType . ';base64,' . base64_encode($firstChunk), $mimeType, 'large-file.mp4'), - 'permissions' => $permissions, - ]); + for ($chunk = 0; $chunk < $chunksTotal; $chunk++) { + $start = $chunk * $chunkSize; + $chunkData = fread($handle, min($chunkSize, $totalSize - $start)); + $end = $start + strlen($chunkData) - 1; - $this->assertEquals(201, $firstUpload['headers']['status-code']); - $this->assertNotContains(Permission::read(Role::user($this->getUser()['$id'])), $firstUpload['body']['$permissions']); + $headers = [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'content-range' => 'bytes ' . $start . '-' . $end . '/' . $totalSize, + ]; - $uploadedFileId = $firstUpload['body']['$id']; - $secondChunkEnd = $chunkSize + strlen($secondChunk) - 1; + if (!empty($uploadedFileId)) { + $headers['x-appwrite-id'] = $uploadedFileId; + } - $secondUpload = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ - 'content-type' => 'multipart/form-data', - 'x-appwrite-project' => $this->getProject()['$id'], - 'content-range' => 'bytes ' . $chunkSize . '-' . $secondChunkEnd . '/' . $totalSize, - 'x-appwrite-id' => $uploadedFileId, - ], $this->getHeaders()), [ - 'fileId' => $fileId, - 'file' => new CURLFile('data://' . $mimeType . ';base64,' . base64_encode($secondChunk), $mimeType, 'large-file.mp4'), - 'permissions' => $permissions, - ]); + $upload = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge($headers, $this->getHeaders()), [ + 'fileId' => $fileId, + 'file' => new CURLFile('data://' . $mimeType . ';base64,' . base64_encode($chunkData), $mimeType, 'large-file.mp4'), + 'permissions' => $permissions, + ]); - $this->assertEquals(201, $secondUpload['headers']['status-code'], $secondUpload['body']['message'] ?? ''); - $this->assertEquals(2, $secondUpload['body']['chunksUploaded']); + $this->assertEquals(201, $upload['headers']['status-code'], $upload['body']['message'] ?? ''); + $this->assertEquals($chunk + 1, $upload['body']['chunksUploaded']); + + $uploadedFileId ??= $upload['body']['$id']; + } + + fclose($handle); + + $this->assertNotContains(Permission::read(Role::user($this->getUser()['$id'])), $upload['body']['$permissions']); + $this->assertEquals($chunksTotal, $upload['body']['chunksTotal']); + $this->assertEquals($chunksTotal, $upload['body']['chunksUploaded']); } finally { + if (isset($handle) && \is_resource($handle)) { + fclose($handle); + } + if (!empty($uploadedFileId)) { $this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId . '/files/' . $uploadedFileId, [ 'content-type' => 'application/json', From 2d7f3d4cba3dbcaee2d4036708b9d3224cace6d1 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 2 Jun 2026 20:22:44 +0400 Subject: [PATCH 06/14] test(storage): guard final chunk upload assertion --- tests/e2e/Services/Storage/StorageBase.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 036afdfba4..f654083d80 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -1324,6 +1324,7 @@ trait StorageBase $permissions = [ Permission::delete(Role::user($this->getUser()['$id'])), ]; + $upload = null; for ($chunk = 0; $chunk < $chunksTotal; $chunk++) { $start = $chunk * $chunkSize; @@ -1354,6 +1355,10 @@ trait StorageBase fclose($handle); + if ($upload === null) { + $this->fail('Expected at least one chunk upload response.'); + } + $this->assertNotContains(Permission::read(Role::user($this->getUser()['$id'])), $upload['body']['$permissions']); $this->assertEquals($chunksTotal, $upload['body']['chunksTotal']); $this->assertEquals($chunksTotal, $upload['body']['chunksUploaded']); From 2191a332ef5cfe93c82e0921a31f211c04cfb83b Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 21:23:31 +0100 Subject: [PATCH 07/14] Add coroutine concurrency to queue workers Upgrade utopia-php/queue to 0.19.0 and utopia-php/platform to 1.0.0-rc4, enabling workers to consume multiple jobs concurrently via Swoole coroutines within a single worker process. - Workers pass a maxCoroutines option to the queue adapter, configured via the new _APP_WORKER_MAX_COROUTINES env var (default 1) - Per-message DI resources now live in a coroutine-scoped context container (Server::context()) instead of the shared container - Consumer Redis connections are wrapped in the Locking decorator so concurrent coroutines can safely share one connection - Add a functions worker load test: 30 async 1s executions drain in 31.1s at 1 coroutine, 10.4s at 5, 7.3s at 10 Co-Authored-By: Claude Opus 4.8 (1M context) --- .env | 2 + app/init/registers.php | 8 +- app/worker.php | 10 +- composer.json | 4 +- composer.lock | 28 ++-- docker-compose.yml | 2 + .../Services/Functions/FunctionsLoadTest.php | 126 ++++++++++++++++++ tests/resources/functions/sleep/index.js | 4 + 8 files changed, 163 insertions(+), 21 deletions(-) create mode 100644 tests/e2e/Services/Functions/FunctionsLoadTest.php create mode 100644 tests/resources/functions/sleep/index.js diff --git a/.env b/.env index aa7438f6bb..61de326ee5 100644 --- a/.env +++ b/.env @@ -150,3 +150,5 @@ _APP_POOL_ADAPTER=stack _APP_WORKER_SCREENSHOTS_ROUTER=http://appwrite _TESTS_OAUTH2_GITHUB_CLIENT_ID= _TESTS_OAUTH2_GITHUB_CLIENT_SECRET= + +_APP_WORKER_MAX_COROUTINES=3 diff --git a/app/init/registers.php b/app/init/registers.php index 9ea19eee24..881aadd249 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -364,11 +364,17 @@ $register->set('pools', function () { default => null }; case 'publisher': - case 'consumer': return match ($dsn->getScheme()) { 'redis' => new Queue\Broker\Redis(new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort())), default => null }; + case 'consumer': + // Consumers are shared by multiple coroutines within a worker + // process, so serialize commands on the single connection. + return match ($dsn->getScheme()) { + 'redis' => new Queue\Broker\Redis(new Queue\Connection\Locking(new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()))), + default => null + }; case 'cache': $adapter = match ($dsn->getScheme()) { 'redis' => new RedisCache($resource()), diff --git a/app/worker.php b/app/worker.php index 169b8e9770..e65655c2de 100644 --- a/app/worker.php +++ b/app/worker.php @@ -86,14 +86,16 @@ $pools = $container->get('pools'); $adapter = new Swoole( $pools->get('consumer')->pop()->getResource(), System::getEnv('_APP_WORKERS_NUM', 1), - $queueName + $queueName, + maxCoroutines: (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1), + resources: $container, ); -$worker = new Server($adapter, $container); +$worker = new Server($adapter); try { $worker->init()->action(function () use ($worker, $registerWorkerMessageResources, $queueName) { - $registerWorkerMessageResources($worker->getContainer()); + $registerWorkerMessageResources($worker->context()); Span::init("worker.{$queueName}"); }); @@ -103,7 +105,7 @@ try { $container->set('bus', function ($register) use ($worker) { return $register->get('bus')->setResolver( - fn (string $name) => $worker->getContainer()->get($name) + fn (string $name) => $worker->context()->get($name) ); }, ['register']); diff --git a/composer.json b/composer.json index 1e3ce344b8..b78637ccd9 100644 --- a/composer.json +++ b/composer.json @@ -76,11 +76,11 @@ "utopia-php/logger": "0.8.*", "utopia-php/messaging": "1.0.*", "utopia-php/migration": "1.*", - "utopia-php/platform": "1.0.0-rc3", + "utopia-php/platform": "1.0.0-rc4", "utopia-php/pools": "1.*", "utopia-php/span": "1.1.*", "utopia-php/preloader": "0.2.*", - "utopia-php/queue": "0.18.*", + "utopia-php/queue": "0.19.*", "utopia-php/servers": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/storage": "2.*", diff --git a/composer.lock b/composer.lock index 11711529ba..b456b34ec7 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "551924e300a646e6f433ea0a0f3d6c9f", + "content-hash": "0c23c30bba86ad84377d4d1abce0992a", "packages": [ { "name": "adhocore/jwt", @@ -4773,16 +4773,16 @@ }, { "name": "utopia-php/platform", - "version": "1.0.0-rc3", + "version": "1.0.0-rc4", "source": { "type": "git", "url": "https://github.com/utopia-php/platform.git", - "reference": "674fd3cea6d1f8db660b01c5be89e8333dfb6de9" + "reference": "455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/platform/zipball/674fd3cea6d1f8db660b01c5be89e8333dfb6de9", - "reference": "674fd3cea6d1f8db660b01c5be89e8333dfb6de9", + "url": "https://api.github.com/repos/utopia-php/platform/zipball/455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0", + "reference": "455c84f8d1cdaec6fc2c39587a78aa5ef263ddd0", "shasum": "" }, "require": { @@ -4791,7 +4791,7 @@ "php": ">=8.3", "utopia-php/cli": "0.23.*", "utopia-php/http": "^2.0@RC", - "utopia-php/queue": "0.18.*", + "utopia-php/queue": "^0.19", "utopia-php/servers": "0.4.*" }, "require-dev": { @@ -4818,9 +4818,9 @@ ], "support": { "issues": "https://github.com/utopia-php/platform/issues", - "source": "https://github.com/utopia-php/platform/tree/1.0.0-rc3" + "source": "https://github.com/utopia-php/platform/tree/1.0.0-rc4" }, - "time": "2026-05-25T11:48:18+00:00" + "time": "2026-06-02T20:19:26+00:00" }, { "name": "utopia-php/pools", @@ -4976,16 +4976,16 @@ }, { "name": "utopia-php/queue", - "version": "0.18.6", + "version": "0.19.0", "source": { "type": "git", "url": "https://github.com/utopia-php/queue.git", - "reference": "954cd3788b37697b83fd2757adcd93422f21a846" + "reference": "cd4248af69006b2568c3486a675f27cb21191859" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/queue/zipball/954cd3788b37697b83fd2757adcd93422f21a846", - "reference": "954cd3788b37697b83fd2757adcd93422f21a846", + "url": "https://api.github.com/repos/utopia-php/queue/zipball/cd4248af69006b2568c3486a675f27cb21191859", + "reference": "cd4248af69006b2568c3486a675f27cb21191859", "shasum": "" }, "require": { @@ -5037,9 +5037,9 @@ ], "support": { "issues": "https://github.com/utopia-php/queue/issues", - "source": "https://github.com/utopia-php/queue/tree/0.18.6" + "source": "https://github.com/utopia-php/queue/tree/0.19.0" }, - "time": "2026-06-01T19:47:57+00:00" + "time": "2026-06-02T20:12:18+00:00" }, { "name": "utopia-php/registry", diff --git a/docker-compose.yml b/docker-compose.yml index ab51ad32c0..dca86166e0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -350,6 +350,7 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_WORKER_MAX_COROUTINES - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 - _APP_REDIS_HOST @@ -737,6 +738,7 @@ services: environment: - _APP_ENV - _APP_WORKER_PER_CORE + - _APP_WORKER_MAX_COROUTINES - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN diff --git a/tests/e2e/Services/Functions/FunctionsLoadTest.php b/tests/e2e/Services/Functions/FunctionsLoadTest.php new file mode 100644 index 0000000000..81d861fc6b --- /dev/null +++ b/tests/e2e/Services/Functions/FunctionsLoadTest.php @@ -0,0 +1,126 @@ +setupFunction([ + 'functionId' => ID::unique(), + 'name' => 'Load Test Sleep', + 'runtime' => 'node-22', + 'entrypoint' => 'index.js', + // Must exceed worst-case queue drain time: executions waiting in the + // queue longer than the function timeout are reported as failed. + 'timeout' => 300, + ]); + + $this->setupDeployment($functionId, [ + 'code' => $this->packageFunction('sleep'), + 'activate' => true, + ]); + + // Warm up the runtime so cold start does not skew the measurement. + $execution = $this->createExecution($functionId, ['async' => false]); + $this->assertEquals(201, $execution['headers']['status-code']); + $this->assertEquals('completed', $execution['body']['status']); + + // Fire N async executions. + $start = \microtime(true); + + for ($i = 0; $i < self::EXECUTIONS; $i++) { + $execution = $this->createExecution($functionId, ['async' => true]); + $this->assertEquals(202, $execution['headers']['status-code']); + } + + $enqueued = \microtime(true) - $start; + + // Poll until all executions reach a terminal state. + // The warmup execution is already completed, so expect N + 1. + $expected = self::EXECUTIONS + 1; + $completed = 0; + $failed = 0; + $drained = null; + + while (\microtime(true) - $start < self::DRAIN_TIMEOUT_SECONDS) { + $completed = $this->countExecutions($functionId, 'completed'); + $failed = $this->countExecutions($functionId, 'failed'); + + if ($completed + $failed >= $expected) { + $drained = \microtime(true) - $start; + break; + } + + \usleep(250_000); + } + + $this->assertNotNull($drained, "Queue did not drain in " . self::DRAIN_TIMEOUT_SECONDS . "s ({$completed} completed, {$failed} failed)"); + + if ($failed > 0) { + $samples = $this->listExecutions($functionId, [ + 'queries' => [ + Query::equal('status', ['failed'])->toString(), + Query::limit(3)->toString(), + ], + ]); + + $details = \array_map(fn ($e) => [ + 'responseStatusCode' => $e['responseStatusCode'] ?? null, + 'errors' => $e['errors'] ?? null, + 'logs' => $e['logs'] ?? null, + 'duration' => $e['duration'] ?? null, + ], $samples['body']['executions'] ?? []); + + $this->fail("{$failed} executions failed. Samples: " . \json_encode($details, JSON_PRETTY_PRINT)); + } + $this->assertEquals($expected, $completed); + + $throughput = self::EXECUTIONS / $drained; + + \fwrite(STDERR, \sprintf( + "\n[load] executions=%d enqueue=%.2fs drain=%.2fs throughput=%.2f exec/s\n", + self::EXECUTIONS, + $enqueued, + $drained, + $throughput, + )); + + $this->cleanupFunction($functionId); + } + + private function countExecutions(string $functionId, string $status): int + { + $executions = $this->listExecutions($functionId, [ + 'queries' => [ + Query::equal('status', [$status])->toString(), + Query::limit(1)->toString(), + ], + ]); + + $this->assertEquals(200, $executions['headers']['status-code']); + + return $executions['body']['total']; + } +} diff --git a/tests/resources/functions/sleep/index.js b/tests/resources/functions/sleep/index.js new file mode 100644 index 0000000000..416bb512ec --- /dev/null +++ b/tests/resources/functions/sleep/index.js @@ -0,0 +1,4 @@ +module.exports = async (context) => { + await new Promise((resolve) => setTimeout(resolve, 1000)); + return context.res.send('OK'); +}; From 2e49713194fbf53b48be29ac475c360ee795d814 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 21:31:17 +0100 Subject: [PATCH 08/14] Set explicit per-worker process and coroutine counts in compose Workers run a single process (_APP_WORKERS_NUM=1) with concurrency provided by coroutines instead. I/O-bound workers get 8 coroutines; CPU-heavy, stateful, or ordering-sensitive workers (databases, builds, migrations, stats-resources, stats-usage) stay serial at 1. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env | 3 ++- docker-compose.yml | 31 ++++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 61de326ee5..57deb8cc0a 100644 --- a/.env +++ b/.env @@ -151,4 +151,5 @@ _APP_WORKER_SCREENSHOTS_ROUTER=http://appwrite _TESTS_OAUTH2_GITHUB_CLIENT_ID= _TESTS_OAUTH2_GITHUB_CLIENT_SECRET= -_APP_WORKER_MAX_COROUTINES=3 +_APP_WORKERS_NUM=1 +_APP_WORKER_MAX_COROUTINES=1 diff --git a/docker-compose.yml b/docker-compose.yml index dca86166e0..981835b866 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -349,8 +349,9 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - - _APP_WORKER_MAX_COROUTINES - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 - _APP_REDIS_HOST @@ -384,6 +385,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -425,6 +428,8 @@ services: environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -485,6 +490,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -505,7 +512,6 @@ services: - _APP_DB_USER_VECTORSDB - _APP_DB_PASS_VECTORSDB - _APP_LOGGING_CONFIG - - _APP_WORKERS_NUM - _APP_QUEUE_NAME - _APP_DATABASE_SHARED_TABLES @@ -528,6 +534,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -606,6 +614,8 @@ services: - _APP_OPTIONS_FORCE_HTTPS # Basic - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_LOGGING_CONFIG @@ -666,6 +676,8 @@ services: - ./src:/usr/src/code/src environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -706,6 +718,8 @@ services: - ${_APP_DB_HOST:-mongodb} environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -737,8 +751,9 @@ services: - redis environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - - _APP_WORKER_MAX_COROUTINES - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN @@ -784,6 +799,8 @@ services: # - smtp environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -825,6 +842,8 @@ services: - redis environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -885,6 +904,8 @@ services: - ${_APP_DB_HOST:-mongodb} environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1046,6 +1067,8 @@ services: - ${_APP_DB_HOST:-mongodb} environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1080,6 +1103,8 @@ services: - appwrite-embedding environment: - _APP_ENV + - _APP_WORKERS_NUM=1 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 From e3afc66bac38fa64ceeb40ac2f07e19bd0ee7fc8 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 21:33:16 +0100 Subject: [PATCH 09/14] Run all workers at 8 coroutines except databases Only the databases worker is ordering-sensitive (schema operations must apply in sequence); every other worker handles independent jobs. Co-Authored-By: Claude Opus 4.8 (1M context) --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 981835b866..5bada8bb6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -535,7 +535,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -905,7 +905,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1068,7 +1068,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1104,7 +1104,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=1 + - _APP_WORKER_MAX_COROUTINES=8 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 From ead860457ca6eb47975ed3af0df6097ee16c85f2 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 21:35:08 +0100 Subject: [PATCH 10/14] Keep stats workers serial: shared accumulator state is not coroutine-safe StatsUsage batches metrics across messages on shared instance state ($stats/$keys/$projects/$statDocuments); concurrent consume coroutines race the flush in commitToDB, losing aggregates on reset and double-applying upsertDocumentsWithIncrease on overlapping flushes. StatsResources resets its shared $documents buffer at the start of each job, so a second coroutine starting mid-count wipes the first one's batch and the projects' metrics interleave into one write. Both stay at 1 coroutine until accumulation moves to per-message state. Co-Authored-By: Claude Opus 4.8 (1M context) --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5bada8bb6b..2a175751a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1068,7 +1068,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=8 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -1104,7 +1104,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=8 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 From 7889a1448f27858830a4d513d263a4378782c6c3 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 21:38:26 +0100 Subject: [PATCH 11/14] Serialize workers with shared accumulator state; fix Messaging local device cache Coroutine-safety audit of all worker actions (instance state mutated during action() is shared across consume coroutines): - audits: batches logs across messages ($logs/$lastTriggeredTime) with Commit/NoCommit ack semantics tied to the shared batch; concurrent flushes lose or duplicate audit entries - webhooks: $errors buffer is reset per job and drives the retry/disable path; concurrent jobs wipe or steal each other's errors - migrations: project DB callables are rebound on the instance per message; a concurrent message swaps database handles mid-migration Messaging cached a project-scoped Local device unkeyed, reusing the first project's storage path for every subsequent message; create it per call instead. Deletes, mails, certificates, executions, functions, builds and screenshots keep all per-message state in locals and stay at 8. Co-Authored-By: Claude Opus 4.8 (1M context) --- docker-compose.yml | 6 +++--- src/Appwrite/Platform/Workers/Messaging.php | 10 +++------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2a175751a4..a065408197 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -350,7 +350,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=8 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -386,7 +386,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=8 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -905,7 +905,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=8 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 759d78f81e..82548b83a7 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -53,8 +53,6 @@ use function Swoole\Coroutine\batch; class Messaging extends Action { - private ?Local $localDevice = null; - private ?SMSAdapter $adapter = null; public static function getName(): string @@ -716,11 +714,9 @@ class Messaging extends Action private function getLocalDevice($project): Local { - if ($this->localDevice === null) { - $this->localDevice = new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); - } - - return $this->localDevice; + // Not cached: the path is project-scoped and the worker handles + // messages from many projects (and coroutines run them concurrently). + return new Local(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); } private function createInternalSMSAdapter(): ?SMSAdapter From b53268a4ddaf2c237d69e3e36aebab0fae24303f Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 21:40:48 +0100 Subject: [PATCH 12/14] Remove functions load test harness The throughput numbers live in the PR; the harness itself is not CI material. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Services/Functions/FunctionsLoadTest.php | 126 ------------------ tests/resources/functions/sleep/index.js | 4 - 2 files changed, 130 deletions(-) delete mode 100644 tests/e2e/Services/Functions/FunctionsLoadTest.php delete mode 100644 tests/resources/functions/sleep/index.js diff --git a/tests/e2e/Services/Functions/FunctionsLoadTest.php b/tests/e2e/Services/Functions/FunctionsLoadTest.php deleted file mode 100644 index 81d861fc6b..0000000000 --- a/tests/e2e/Services/Functions/FunctionsLoadTest.php +++ /dev/null @@ -1,126 +0,0 @@ -setupFunction([ - 'functionId' => ID::unique(), - 'name' => 'Load Test Sleep', - 'runtime' => 'node-22', - 'entrypoint' => 'index.js', - // Must exceed worst-case queue drain time: executions waiting in the - // queue longer than the function timeout are reported as failed. - 'timeout' => 300, - ]); - - $this->setupDeployment($functionId, [ - 'code' => $this->packageFunction('sleep'), - 'activate' => true, - ]); - - // Warm up the runtime so cold start does not skew the measurement. - $execution = $this->createExecution($functionId, ['async' => false]); - $this->assertEquals(201, $execution['headers']['status-code']); - $this->assertEquals('completed', $execution['body']['status']); - - // Fire N async executions. - $start = \microtime(true); - - for ($i = 0; $i < self::EXECUTIONS; $i++) { - $execution = $this->createExecution($functionId, ['async' => true]); - $this->assertEquals(202, $execution['headers']['status-code']); - } - - $enqueued = \microtime(true) - $start; - - // Poll until all executions reach a terminal state. - // The warmup execution is already completed, so expect N + 1. - $expected = self::EXECUTIONS + 1; - $completed = 0; - $failed = 0; - $drained = null; - - while (\microtime(true) - $start < self::DRAIN_TIMEOUT_SECONDS) { - $completed = $this->countExecutions($functionId, 'completed'); - $failed = $this->countExecutions($functionId, 'failed'); - - if ($completed + $failed >= $expected) { - $drained = \microtime(true) - $start; - break; - } - - \usleep(250_000); - } - - $this->assertNotNull($drained, "Queue did not drain in " . self::DRAIN_TIMEOUT_SECONDS . "s ({$completed} completed, {$failed} failed)"); - - if ($failed > 0) { - $samples = $this->listExecutions($functionId, [ - 'queries' => [ - Query::equal('status', ['failed'])->toString(), - Query::limit(3)->toString(), - ], - ]); - - $details = \array_map(fn ($e) => [ - 'responseStatusCode' => $e['responseStatusCode'] ?? null, - 'errors' => $e['errors'] ?? null, - 'logs' => $e['logs'] ?? null, - 'duration' => $e['duration'] ?? null, - ], $samples['body']['executions'] ?? []); - - $this->fail("{$failed} executions failed. Samples: " . \json_encode($details, JSON_PRETTY_PRINT)); - } - $this->assertEquals($expected, $completed); - - $throughput = self::EXECUTIONS / $drained; - - \fwrite(STDERR, \sprintf( - "\n[load] executions=%d enqueue=%.2fs drain=%.2fs throughput=%.2f exec/s\n", - self::EXECUTIONS, - $enqueued, - $drained, - $throughput, - )); - - $this->cleanupFunction($functionId); - } - - private function countExecutions(string $functionId, string $status): int - { - $executions = $this->listExecutions($functionId, [ - 'queries' => [ - Query::equal('status', [$status])->toString(), - Query::limit(1)->toString(), - ], - ]); - - $this->assertEquals(200, $executions['headers']['status-code']); - - return $executions['body']['total']; - } -} diff --git a/tests/resources/functions/sleep/index.js b/tests/resources/functions/sleep/index.js deleted file mode 100644 index 416bb512ec..0000000000 --- a/tests/resources/functions/sleep/index.js +++ /dev/null @@ -1,4 +0,0 @@ -module.exports = async (context) => { - await new Promise((resolve) => setTimeout(resolve, 1000)); - return context.res.send('OK'); -}; From 4baaa7d93fd544b8fc82faa08372c8f745c366da Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 22:03:10 +0100 Subject: [PATCH 13/14] Fix coroutine starvation: scale pools to coroutine count, serialize SMTP workers Two CI failure causes: - Connection pools are sized for HTTP-server worker counts and came out at 1 per worker process; 8 consume coroutines exhausted them (Pool 'cache_redis_main' is empty). Floor the pool size at _APP_WORKER_MAX_COROUTINES. - The mails and messaging workers disable the Swoole TCP hook at runtime to work around the TLS handshake crash (swoole-src#4909, #7914). The flag is process-global, so after the first job every blocking Redis pop freezes the coroutine scheduler and deliveries arrive minutes late. Both workers stay at 1 coroutine, matching pre-coroutine behavior. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/init/registers.php | 4 ++++ docker-compose.yml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/init/registers.php b/app/init/registers.php index 881aadd249..2f5f39ceb0 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -254,6 +254,10 @@ $register->set('pools', function () { $workerCount = intval(System::getEnv('_APP_CPU_NUM', swoole_cpu_num())) * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); $poolSize = max(1, (int)($instanceConnections / $workerCount)); + // Queue workers consume jobs concurrently with coroutines; each in-flight + // job may hold a connection, so pools must cover the coroutine count. + $poolSize = max($poolSize, (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1)); + foreach ($connections as $key => $connection) { $type = $connection['type']; $multiple = $connection['multiple']; diff --git a/docker-compose.yml b/docker-compose.yml index a065408197..90bea1dc5d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -800,7 +800,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=8 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 @@ -843,7 +843,7 @@ services: environment: - _APP_ENV - _APP_WORKERS_NUM=1 - - _APP_WORKER_MAX_COROUTINES=8 + - _APP_WORKER_MAX_COROUTINES=1 - _APP_WORKER_PER_CORE - _APP_POOL_ADAPTER - _APP_OPENSSL_KEY_V1 From 78769b7f13e6286b7d8cc04dfbab2e2ad0e2b840 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 2 Jun 2026 22:37:37 +0100 Subject: [PATCH 14/14] Lease one broker connection per consume coroutine instead of sharing one The Locking decorator serialized every Redis command on the single consumer connection shared by all consume coroutines. On an idle queue each waiting coroutine holds the mutex for a full 2s blocking pop, so a finished job's completion ops (remove job key, increment stats) starve behind the poppers: a 10ms cache purge took 22-24s to ack, and Storage's testFilePreviewCache observed stale previews because purges landed tens of seconds after enqueue. Pass a Broker\Pool as the adapter consumer instead: each consume coroutine leases its own connection from the consumer pool for the lifetime of its loop, eliminating the shared-connection mutex entirely. The Locking wrapper in the pool factory is reverted, and consumer pools get +2 headroom above the coroutine count for queue-size telemetry reads. Purge ack latency drops to 4-58ms. Known tradeoff: Broker\Pool::close() is a no-op, so SIGTERM no longer breaks the blocking pop early; workers rely on the process kill that already follows. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/init/registers.php | 18 ++++++++---------- app/worker.php | 8 +++++++- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/app/init/registers.php b/app/init/registers.php index 2f5f39ceb0..50080f0538 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -344,9 +344,13 @@ $register->set('pools', function () { // PubSub workers hold one long-lived subscribed connection and also need // spare capacity for publishes from the same process. - $connectionPoolSize = $type === 'pubsub' - ? max(2, $poolSize) - : $poolSize; + // Consumer pools lease one connection per consume coroutine for the + // lifetime of the worker, plus headroom for queue-size reads. + $connectionPoolSize = match ($type) { + 'pubsub' => max(2, $poolSize), + 'consumer' => $poolSize + 2, + default => $poolSize, + }; $pool = new Pool($poolAdapter, $name, $connectionPoolSize, function () use ($type, $resource, $dsn) { // Get Adapter @@ -368,17 +372,11 @@ $register->set('pools', function () { default => null }; case 'publisher': + case 'consumer': return match ($dsn->getScheme()) { 'redis' => new Queue\Broker\Redis(new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort())), default => null }; - case 'consumer': - // Consumers are shared by multiple coroutines within a worker - // process, so serialize commands on the single connection. - return match ($dsn->getScheme()) { - 'redis' => new Queue\Broker\Redis(new Queue\Connection\Locking(new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()))), - default => null - }; case 'cache': $adapter = match ($dsn->getScheme()) { 'redis' => new RedisCache($resource()), diff --git a/app/worker.php b/app/worker.php index e65655c2de..063d4746ac 100644 --- a/app/worker.php +++ b/app/worker.php @@ -83,8 +83,14 @@ if (\str_starts_with($workerName, 'databases')) { /** @var \Utopia\Pools\Group $pools */ $pools = $container->get('pools'); +// Each consume coroutine leases its own broker connection from the pool for +// the lifetime of its consume loop, so concurrent coroutines never share a +// connection. The publisher side backs the queue-depth telemetry gauge. $adapter = new Swoole( - $pools->get('consumer')->pop()->getResource(), + new BrokerPool( + publisher: $pools->get('consumer'), + consumer: $pools->get('consumer'), + ), System::getEnv('_APP_WORKERS_NUM', 1), $queueName, maxCoroutines: (int) System::getEnv('_APP_WORKER_MAX_COROUTINES', 1),