From 43fc29dc759787d58f79ef588f574981bad359b9 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 18 Aug 2026 14:02:09 +0300 Subject: [PATCH 1/8] run --- app/controllers/api/messaging.php | 32 +++++--- app/init/constants.php | 2 +- app/init/database/filters.php | 30 +++++--- src/Appwrite/Platform/Workers/Messaging.php | 17 ++-- .../unit/Utopia/Response/Model/TopicTest.php | 77 +++++++++++++++++++ 5 files changed, 129 insertions(+), 29 deletions(-) create mode 100644 tests/unit/Utopia/Response/Model/TopicTest.php diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 4be28c614e..3bd68d97c9 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -2617,7 +2617,11 @@ Http::get('/v1/messaging/topics') $cursor->setValue($cursorDocument[0]); } try { - $topics = $dbForProject->find('topics', $queries); + // Safe to skip subquery, Does not return in Resonse + $topics = $dbForProject->skipFilters( + fn () => $dbForProject->find('topics', $queries), + ['subQueryTopicTargets'] + ); $total = $includeTotal ? $dbForProject->count('topics', $queries, APP_LIMIT_COUNT) : 0; } catch (OrderException $e) { throw new Exception(Exception::DATABASE_QUERY_ORDER_NULL, "The order attribute '{$e->getAttribute()}' had a null value. Cursor pagination requires all documents order attribute values are non-null."); @@ -2840,10 +2844,15 @@ Http::post('/v1/messaging/topics/:topicId/subscribers') default => throw new Exception(Exception::TARGET_PROVIDER_INVALID_TYPE), }; - $authorization->skip(fn () => $dbForProject->increaseDocumentAttribute( - 'topics', - $topicId, - $totalAttribute, + // increaseDocumentAttribute reloads the topic internally, so it pays the same + // sub-query cost as the read above unless the filter is skipped here too. + $authorization->skip(fn () => $dbForProject->skipFilters( + fn () => $dbForProject->increaseDocumentAttribute( + 'topics', + $topicId, + $totalAttribute, + ), + ['subQueryTopicTargets'] )); } catch (DuplicateException) { throw new Exception(Exception::SUBSCRIBER_ALREADY_EXISTS); @@ -3047,11 +3056,14 @@ Http::delete('/v1/messaging/topics/:topicId/subscribers/:subscriberId') default => throw new Exception(Exception::TARGET_PROVIDER_INVALID_TYPE), }; - $authorization->skip(fn () => $dbForProject->decreaseDocumentAttribute( - 'topics', - $topicId, - $totalAttribute, - min: 0 + $authorization->skip(fn () => $dbForProject->skipFilters( + fn () => $dbForProject->decreaseDocumentAttribute( + 'topics', + $topicId, + $totalAttribute, + min: 0 + ), + ['subQueryTopicTargets'] )); $queueForEvents diff --git a/app/init/constants.php b/app/init/constants.php index 61d861936b..9974199c34 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -42,7 +42,7 @@ const APP_LIMIT_ARRAY_LABELS_SIZE = 1000; // Default maximum of how many labels const APP_LIMIT_ARRAY_SCOPES_SIZE = 200; // Default maximum of how many scope elements can there be in API parameter that expects array value const APP_LIMIT_ARRAY_ELEMENT_SIZE = 4096; // Default maximum length of element in array parameter represented by maximum URL length. const APP_LIMIT_SUBQUERY = 1000; -const APP_LIMIT_SUBSCRIBERS_SUBQUERY = 1_000_000; +const APP_LIMIT_SUBSCRIBERS_SUBQUERY = 25; const APP_LIMIT_WRITE_RATE_DEFAULT = 60; // Default maximum write rate per rate period const APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT = 60; // Default maximum write rate period in seconds const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return in list API calls diff --git a/app/init/database/filters.php b/app/init/database/filters.php index cba0c0cad6..60bf1204d3 100644 --- a/app/init/database/filters.php +++ b/app/init/database/filters.php @@ -357,19 +357,25 @@ Database::addFilter( return; }, function (mixed $value, Document $document, Database $database) { - $targetIds = $database->getAuthorization()->skip(fn () => \array_map( - fn ($document) => $document->getAttribute('targetInternalId'), - $database->find('subscribers', [ - Query::equal('topicInternalId', [$document->getSequence()]), - Query::limit(APP_LIMIT_SUBSCRIBERS_SUBQUERY) - ]) - )); - if (\count($targetIds) > 0) { - return $database->skipValidation(fn () => $database->find('targets', [ - Query::equal('$sequence', $targetIds) - ])); + $subscribers = $database->getAuthorization()->skip(fn () => $database->find('subscribers', [ + Query::select(['targetInternalId']), + Query::equal('topicInternalId', [$document->getSequence()]), + Query::limit(APP_LIMIT_SUBSCRIBERS_SUBQUERY) + ])); + + $targetIds = \array_map( + fn (Document $subscriber) => $subscriber->getAttribute('targetInternalId'), + $subscribers + ); + + if (\count($targetIds) === 0) { + return []; } - return []; + + return $database->skipValidation(fn () => $database->find('targets', [ + Query::equal('$sequence', $targetIds), + Query::limit(\count($targetIds)) + ])); } ); diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 528ed2fee7..9ba33a15f4 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -365,7 +365,9 @@ class Messaging extends Action * * Peak memory is O(MESSAGE_RECIPIENTS_PAGE_SIZE), never O(topic size): topics are walked through the * subscribers collection with cursor pagination rather than reading the topic's `targets` attribute, which - * triggers the subQueryTopicTargets filter and loads up to APP_LIMIT_SUBSCRIBERS_SUBQUERY rows at once. + * is capped at APP_LIMIT_SUBSCRIBERS_SUBQUERY and would silently drop the rest of the recipients. + * Decode filters run regardless of Query::select, so the topic lookup below skips that filter explicitly + * rather than relying on selecting only $sequence. * * @param array $topicIds * @param array $userIds @@ -382,11 +384,14 @@ class Messaging extends Action Document $default ): \Generator { if (\count($topicIds) > 0) { - $topics = $dbForProject->find('topics', [ - Query::select(['$sequence']), - Query::equal('$id', $topicIds), - Query::limit(\count($topicIds)), - ]); + $topics = $dbForProject->skipFilters( + fn () => $dbForProject->find('topics', [ + Query::select(['$sequence']), + Query::equal('$id', $topicIds), + Query::limit(\count($topicIds)), + ]), + ['subQueryTopicTargets'] + ); foreach ($topics as $topic) { $cursor = null; diff --git a/tests/unit/Utopia/Response/Model/TopicTest.php b/tests/unit/Utopia/Response/Model/TopicTest.php new file mode 100644 index 0000000000..61eb4e3011 --- /dev/null +++ b/tests/unit/Utopia/Response/Model/TopicTest.php @@ -0,0 +1,77 @@ +response = new Response(new SwooleResponse()); + $this->response->setModel(new Topic()); + $this->response->setModel(new BaseList('Topic list', Response::MODEL_TOPIC_LIST, 'topics', Response::MODEL_TOPIC)); + } + + public function testTargetsAreNotPartOfThePayload(): void + { + $output = $this->response->output($this->topic([ + 'targets' => [new Document(['$id' => 'target-id', 'identifier' => 'user@example.com'])], + ]), Response::MODEL_TOPIC); + + $this->assertArrayNotHasKey('targets', $output); + } + + public function testPayloadIsIdenticalWithAndWithoutTargets(): void + { + $withTargets = $this->response->output($this->topic([ + 'targets' => [new Document(['$id' => 'target-id'])], + ]), Response::MODEL_TOPIC); + + $withoutTargets = $this->response->output($this->topic(), Response::MODEL_TOPIC); + + $this->assertSame($withoutTargets, $withTargets); + } + + public function testListPayloadIsIdenticalWithAndWithoutTargets(): void + { + $withTargets = $this->response->output(new Document([ + 'topics' => [$this->topic(['targets' => [new Document(['$id' => 'target-id'])]])], + 'total' => 1, + ]), Response::MODEL_TOPIC_LIST); + + $withoutTargets = $this->response->output(new Document([ + 'topics' => [$this->topic()], + 'total' => 1, + ]), Response::MODEL_TOPIC_LIST); + + $this->assertSame($withoutTargets, $withTargets); + $this->assertArrayNotHasKey('targets', $withTargets['topics'][0]); + } + + /** + * @param array $extra + */ + private function topic(array $extra = []): Document + { + return new Document(\array_merge([ + '$id' => 'topic-id', + '$createdAt' => '2026-01-01T00:00:00.000+00:00', + '$updatedAt' => '2026-01-01T00:00:00.000+00:00', + 'name' => 'events', + 'emailTotal' => 7000, + 'smsTotal' => 0, + 'pushTotal' => 0, + 'subscribe' => ['users'], + ], $extra)); + } +} From e4eaa3d9ed782b94e34c2283f66b87bf6279c984 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 18 Aug 2026 14:23:16 +0300 Subject: [PATCH 2/8] Update --- app/controllers/api/messaging.php | 8 +- .../unit/Utopia/Response/Model/TopicTest.php | 77 ------------------- 2 files changed, 6 insertions(+), 79 deletions(-) delete mode 100644 tests/unit/Utopia/Response/Model/TopicTest.php diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index 3bd68d97c9..b11cced8c5 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -2844,8 +2844,10 @@ Http::post('/v1/messaging/topics/:topicId/subscribers') default => throw new Exception(Exception::TARGET_PROVIDER_INVALID_TYPE), }; - // increaseDocumentAttribute reloads the topic internally, so it pays the same - // sub-query cost as the read above unless the filter is skipped here too. + // increaseDocumentAttribute reloads the topic internally and pays the same sub-query + // cost as the read above. Safe to skip because that reload is forUpdate, which neither + // reads nor writes the document cache, so no reader can observe a topic decoded + // without `targets`. $authorization->skip(fn () => $dbForProject->skipFilters( fn () => $dbForProject->increaseDocumentAttribute( 'topics', @@ -3056,6 +3058,8 @@ Http::delete('/v1/messaging/topics/:topicId/subscribers/:subscriberId') default => throw new Exception(Exception::TARGET_PROVIDER_INVALID_TYPE), }; + // Mirrors the skip in createSubscriber: the reload inside is forUpdate, so nothing it + // decodes reaches the document cache. $authorization->skip(fn () => $dbForProject->skipFilters( fn () => $dbForProject->decreaseDocumentAttribute( 'topics', diff --git a/tests/unit/Utopia/Response/Model/TopicTest.php b/tests/unit/Utopia/Response/Model/TopicTest.php deleted file mode 100644 index 61eb4e3011..0000000000 --- a/tests/unit/Utopia/Response/Model/TopicTest.php +++ /dev/null @@ -1,77 +0,0 @@ -response = new Response(new SwooleResponse()); - $this->response->setModel(new Topic()); - $this->response->setModel(new BaseList('Topic list', Response::MODEL_TOPIC_LIST, 'topics', Response::MODEL_TOPIC)); - } - - public function testTargetsAreNotPartOfThePayload(): void - { - $output = $this->response->output($this->topic([ - 'targets' => [new Document(['$id' => 'target-id', 'identifier' => 'user@example.com'])], - ]), Response::MODEL_TOPIC); - - $this->assertArrayNotHasKey('targets', $output); - } - - public function testPayloadIsIdenticalWithAndWithoutTargets(): void - { - $withTargets = $this->response->output($this->topic([ - 'targets' => [new Document(['$id' => 'target-id'])], - ]), Response::MODEL_TOPIC); - - $withoutTargets = $this->response->output($this->topic(), Response::MODEL_TOPIC); - - $this->assertSame($withoutTargets, $withTargets); - } - - public function testListPayloadIsIdenticalWithAndWithoutTargets(): void - { - $withTargets = $this->response->output(new Document([ - 'topics' => [$this->topic(['targets' => [new Document(['$id' => 'target-id'])]])], - 'total' => 1, - ]), Response::MODEL_TOPIC_LIST); - - $withoutTargets = $this->response->output(new Document([ - 'topics' => [$this->topic()], - 'total' => 1, - ]), Response::MODEL_TOPIC_LIST); - - $this->assertSame($withoutTargets, $withTargets); - $this->assertArrayNotHasKey('targets', $withTargets['topics'][0]); - } - - /** - * @param array $extra - */ - private function topic(array $extra = []): Document - { - return new Document(\array_merge([ - '$id' => 'topic-id', - '$createdAt' => '2026-01-01T00:00:00.000+00:00', - '$updatedAt' => '2026-01-01T00:00:00.000+00:00', - 'name' => 'events', - 'emailTotal' => 7000, - 'smsTotal' => 0, - 'pushTotal' => 0, - 'subscribe' => ['users'], - ], $extra)); - } -} From b64fb4e71cf6dbe1d837564fdb98e1759b22428c Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 18 Aug 2026 14:27:46 +0300 Subject: [PATCH 3/8] Update --- app/controllers/api/messaging.php | 2 +- .../e2e/Services/Messaging/MessagingBase.php | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index b11cced8c5..bf29a240e2 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -2617,7 +2617,7 @@ Http::get('/v1/messaging/topics') $cursor->setValue($cursorDocument[0]); } try { - // Safe to skip subquery, Does not return in Resonse + // Safe to skip subquery, Does not return in Response $topics = $dbForProject->skipFilters( fn () => $dbForProject->find('topics', $queries), ['subQueryTopicTargets'] diff --git a/tests/e2e/Services/Messaging/MessagingBase.php b/tests/e2e/Services/Messaging/MessagingBase.php index dbf2e34aa1..57900097f2 100644 --- a/tests/e2e/Services/Messaging/MessagingBase.php +++ b/tests/e2e/Services/Messaging/MessagingBase.php @@ -1208,6 +1208,10 @@ trait MessagingBase $this->assertEquals(200, $response['headers']['status-code']); $this->assertGreaterThanOrEqual(2, \count($response['body']['topics'])); + foreach ($response['body']['topics'] as $topic) { + $this->assertArrayNotHasKey('targets', $topic); + } + $response = $this->client->call(Client::METHOD_GET, '/messaging/topics', [ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -1240,6 +1244,7 @@ trait MessagingBase $this->assertEquals(0, $response['body']['emailTotal']); $this->assertEquals(0, $response['body']['smsTotal']); $this->assertEquals(0, $response['body']['pushTotal']); + $this->assertArrayNotHasKey('targets', $response['body']); } public function testCreateSubscriber(): void @@ -1373,6 +1378,33 @@ trait MessagingBase $this->assertEquals(201, $response['headers']['status-code']); } + + // The topic's `targets` sub-query is internal and capped far below the subscriber count, + // so it must never surface. Endpoints that load a topic without running the filter have to + // stay indistinguishable from those that run it. + $response = $this->client->call(Client::METHOD_GET, '/messaging/topics/' . $topic['$id'], [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(101, $response['body']['emailTotal']); + $this->assertArrayNotHasKey('targets', $response['body']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'queries' => [ + Query::equal('$id', [$topic['$id']])->toString(), + ], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertSame(1, \count($response['body']['topics'])); + $this->assertArrayNotHasKey('targets', $response['body']['topics'][0]); } public function testGetSubscriber(): void From 6be0925c668480eb7515a7faa6c6f6ac8ea7cf5c Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 19 Aug 2026 18:32:05 +0300 Subject: [PATCH 4/8] skip filters --- src/Appwrite/Deletes/Targets.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Deletes/Targets.php b/src/Appwrite/Deletes/Targets.php index 5ba667d823..958992ea7f 100644 --- a/src/Appwrite/Deletes/Targets.php +++ b/src/Appwrite/Deletes/Targets.php @@ -36,6 +36,10 @@ class Targets function (Document $subscriber) use ($database, $target) { $topicId = $subscriber->getAttribute('topicId'); $topicInternalId = $subscriber->getAttribute('topicInternalId'); + + // Thinking what to do here , We have a bug + // Query::select('$sequence') Will still make the subquery + $topic = $database->getDocument('topics', $topicId); if (!$topic->isEmpty() && $topic->getSequence() === $topicInternalId) { $totalAttribute = match ($target->getAttribute('providerType')) { @@ -46,12 +50,12 @@ class Targets }; try { - $database->decreaseDocumentAttribute( + $database->skipFilters(fn () => $database->decreaseDocumentAttribute( 'topics', $topicId, $totalAttribute, min: 0 - ); + ), ['subQueryTopicTargets']); } catch (LimitException $e) { Console::error("Delete subscribers decreaseDocumentAttribute (topicId={$topicId}): {$e->getMessage()}"); } From bec10a2ffc82f0d44401d1435e332bbd8bd4d48a Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 20 Aug 2026 09:50:05 +0300 Subject: [PATCH 5/8] Use select --- src/Appwrite/Deletes/Targets.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Appwrite/Deletes/Targets.php b/src/Appwrite/Deletes/Targets.php index 958992ea7f..e73c0e0a6d 100644 --- a/src/Appwrite/Deletes/Targets.php +++ b/src/Appwrite/Deletes/Targets.php @@ -37,10 +37,8 @@ class Targets $topicId = $subscriber->getAttribute('topicId'); $topicInternalId = $subscriber->getAttribute('topicInternalId'); - // Thinking what to do here , We have a bug - // Query::select('$sequence') Will still make the subquery + $topic = $database->getDocument('topics', $topicId, [Query::select(['$id', '$sequence'])]); - $topic = $database->getDocument('topics', $topicId); if (!$topic->isEmpty() && $topic->getSequence() === $topicInternalId) { $totalAttribute = match ($target->getAttribute('providerType')) { MESSAGE_TYPE_EMAIL => 'emailTotal', From c8d7e0b59130662fb2acf95f3f4bb358ae9162a0 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 20 Aug 2026 10:02:03 +0300 Subject: [PATCH 6/8] Straight forward comments --- app/controllers/api/messaging.php | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/controllers/api/messaging.php b/app/controllers/api/messaging.php index bf29a240e2..22527a6a59 100644 --- a/app/controllers/api/messaging.php +++ b/app/controllers/api/messaging.php @@ -2844,10 +2844,6 @@ Http::post('/v1/messaging/topics/:topicId/subscribers') default => throw new Exception(Exception::TARGET_PROVIDER_INVALID_TYPE), }; - // increaseDocumentAttribute reloads the topic internally and pays the same sub-query - // cost as the read above. Safe to skip because that reload is forUpdate, which neither - // reads nor writes the document cache, so no reader can observe a topic decoded - // without `targets`. $authorization->skip(fn () => $dbForProject->skipFilters( fn () => $dbForProject->increaseDocumentAttribute( 'topics', @@ -3058,8 +3054,6 @@ Http::delete('/v1/messaging/topics/:topicId/subscribers/:subscriberId') default => throw new Exception(Exception::TARGET_PROVIDER_INVALID_TYPE), }; - // Mirrors the skip in createSubscriber: the reload inside is forUpdate, so nothing it - // decodes reaches the document cache. $authorization->skip(fn () => $dbForProject->skipFilters( fn () => $dbForProject->decreaseDocumentAttribute( 'topics', From 2a147fe92ee4b7685a6141067d94e21e5c9a4449 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 20 Aug 2026 10:21:05 +0300 Subject: [PATCH 7/8] Use sequence --- src/Appwrite/Deletes/Targets.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Deletes/Targets.php b/src/Appwrite/Deletes/Targets.php index e73c0e0a6d..a260a342a5 100644 --- a/src/Appwrite/Deletes/Targets.php +++ b/src/Appwrite/Deletes/Targets.php @@ -37,9 +37,12 @@ class Targets $topicId = $subscriber->getAttribute('topicId'); $topicInternalId = $subscriber->getAttribute('topicInternalId'); - $topic = $database->getDocument('topics', $topicId, [Query::select(['$id', '$sequence'])]); + $topic = $database->findOne('topics', [ + Query::select(['$id', '$sequence']), + Query::equal('$sequence', [$topicInternalId]), + ]); - if (!$topic->isEmpty() && $topic->getSequence() === $topicInternalId) { + if (!$topic->isEmpty()) { $totalAttribute = match ($target->getAttribute('providerType')) { MESSAGE_TYPE_EMAIL => 'emailTotal', MESSAGE_TYPE_SMS => 'smsTotal', From b91ca40922bee1b9cb5438f3e9e90deb76a95ff5 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 20 Aug 2026 10:51:23 +0300 Subject: [PATCH 8/8] Add skipFilters --- src/Appwrite/Deletes/Targets.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Deletes/Targets.php b/src/Appwrite/Deletes/Targets.php index a260a342a5..ebd915fc9b 100644 --- a/src/Appwrite/Deletes/Targets.php +++ b/src/Appwrite/Deletes/Targets.php @@ -37,10 +37,10 @@ class Targets $topicId = $subscriber->getAttribute('topicId'); $topicInternalId = $subscriber->getAttribute('topicInternalId'); - $topic = $database->findOne('topics', [ + $topic = $database->skipFilters(fn () => $database->findOne('topics', [ Query::select(['$id', '$sequence']), Query::equal('$sequence', [$topicInternalId]), - ]); + ]), ['subQueryTopicTargets']); if (!$topic->isEmpty()) { $totalAttribute = match ($target->getAttribute('providerType')) {