From 77905e241f6dba5cc986861c292fc247d2f7cd83 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 18 Nov 2025 16:26:56 +0200 Subject: [PATCH] cached documents list --- composer.json | 2 +- composer.lock | 41 ++++--- .../Databases/Collections/Documents/XList.php | 81 +++++++++++--- .../Databases/Legacy/DatabasesBase.php | 101 ++++++++++++++++++ 4 files changed, 196 insertions(+), 29 deletions(-) diff --git a/composer.json b/composer.json index 1cff497d08..c24806bfc3 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.13.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "4.*", + "utopia-php/database": "dev-cache-documents-xlist as 4.3.0", "utopia-php/detector": "0.2.*", "utopia-php/domains": "0.9.*", "utopia-php/emails": "0.6.*", diff --git a/composer.lock b/composer.lock index 163bdb5bd9..1449590f7f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "3b502f78f5e31f2ea7b4c69e3301283a", + "content-hash": "e4f1bcba84d1ab408e70876ca2e60526", "packages": [ { "name": "adhocore/jwt", @@ -3844,16 +3844,16 @@ }, { "name": "utopia-php/database", - "version": "4.3.0", + "version": "dev-cache-documents-xlist", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "fe7a1326ad623609e65587fe8c01a630a7075fee" + "reference": "02c3b242549ce4d252df3db4fc9a10113fa65d5e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/fe7a1326ad623609e65587fe8c01a630a7075fee", - "reference": "fe7a1326ad623609e65587fe8c01a630a7075fee", + "url": "https://api.github.com/repos/utopia-php/database/zipball/02c3b242549ce4d252df3db4fc9a10113fa65d5e", + "reference": "02c3b242549ce4d252df3db4fc9a10113fa65d5e", "shasum": "" }, "require": { @@ -3896,9 +3896,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/4.3.0" + "source": "https://github.com/utopia-php/database/tree/cache-documents-xlist" }, - "time": "2025-11-14T03:43:10+00:00" + "time": "2025-11-18T14:05:50+00:00" }, { "name": "utopia-php/detector", @@ -4212,16 +4212,16 @@ }, { "name": "utopia-php/framework", - "version": "0.33.29", + "version": "0.33.30", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "6e63939fdb33b847f92839499cd6e8df626c278d" + "reference": "07cf699a7c47bd1a03b4da1812f1719a66b3c924" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/6e63939fdb33b847f92839499cd6e8df626c278d", - "reference": "6e63939fdb33b847f92839499cd6e8df626c278d", + "url": "https://api.github.com/repos/utopia-php/http/zipball/07cf699a7c47bd1a03b4da1812f1719a66b3c924", + "reference": "07cf699a7c47bd1a03b4da1812f1719a66b3c924", "shasum": "" }, "require": { @@ -4253,9 +4253,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.29" + "source": "https://github.com/utopia-php/http/tree/0.33.30" }, - "time": "2025-11-14T06:33:29+00:00" + "time": "2025-11-18T12:18:00+00:00" }, { "name": "utopia-php/image", @@ -8895,9 +8895,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-cache-documents-xlist", + "alias": "4.3.0", + "alias_normalized": "4.3.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -8921,5 +8930,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.2.0" } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php index a5a3ac0eb4..a3d614eb52 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php @@ -24,6 +24,7 @@ use Utopia\Swoole\Response as SwooleResponse; use Utopia\Validator\ArrayList; use Utopia\Validator\Boolean; use Utopia\Validator\Nullable; +use Utopia\Validator\Range; use Utopia\Validator\Text; class XList extends Action @@ -70,25 +71,26 @@ class XList extends Action ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) ->param('transactionId', null, new Nullable(new UID()), 'Transaction ID to read uncommitted changes within the transaction.', true) ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) + ->param('useCache', false, new Boolean(true), 'Opt-in to cached responses for select queries. Disabled by default.', true) + ->param('ttl', 60, new Range(min: 1, max: 86400), 'TTL (seconds) for cached respnses when caching is enabled. Must be between 1 and 86400 (24 hours).', true) ->inject('response') ->inject('dbForProject') ->inject('queueForStatsUsage') ->inject('transactionState') - ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, array $queries, ?string $transactionId, bool $includeTotal, UtopiaResponse $response, Database $dbForProject, StatsUsage $queueForStatsUsage, TransactionState $transactionState, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, array $queries, ?string $transactionId, bool $includeTotal, bool $useCache, int $ttl, UtopiaResponse $response, Database $dbForProject, StatsUsage $queueForStatsUsage, TransactionState $transactionState): void { - $isAPIKey = Auth::isAppUser($authorization->getRoles()); - $isPrivilegedUser = Auth::isPrivilegedUser($authorization->getRoles()); + $isAPIKey = Auth::isAppUser(Authorization::getRoles()); + $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); - $database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); if ($database->isEmpty() || (!$database->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = $authorization->skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId)); + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence(), $collectionId)); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception($this->getParentNotFoundException()); } @@ -116,7 +118,7 @@ class XList extends Action $documentId = $cursor->getValue(); - $cursorDocument = $authorization->skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence() . '_collection_' . $collection->getSequence(), $documentId)); + $cursorDocument = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getSequence() . '_collection_' . $collection->getSequence(), $documentId)); if ($cursorDocument->isEmpty()) { $type = ucfirst($this->getContext()); @@ -135,9 +137,62 @@ class XList extends Action $documents = $transactionState->listDocuments($collectionTableId, $transactionId, $queries); $total = $includeTotal ? $transactionState->countDocuments($collectionTableId, $transactionId, $queries) : 0; } elseif (! empty($selectQueries)) { - // has selects, allow relationship on documents - $documents = $dbForProject->find($collectionTableId, $queries); - $total = $includeTotal ? $dbForProject->count($collectionTableId, $queries, APP_LIMIT_COUNT) : 0; + + if ($useCache) { + $serializedQueries = []; + foreach ($queries as $query) { + $serializedQueries[] = $query instanceof Query ? $query->toArray() : $query; + } + + $hostname = $dbForProject->getAdapter()->getHostname(); + $cacheKeyBase = \sprintf( + '%s-cache-%s:%s:%s:collection:%s:%s', + $dbForProject->cacheName, + $hostname ?? '', + $dbForProject->getNamespace(), + $dbForProject->getTenant(), + $collectionId, + \md5(\implode($serializedQueries)) + ); + + $documentsCacheKey = $cacheKeyBase . ':documents'; + $totalCacheKey = $cacheKeyBase . ':total'; + + $documentsCacheHit = $totalDocumentsCacheHit = false; + + $cachedDocuments = $dbForProject->cache->load($documentsCacheKey, $ttl); + if ($cachedDocuments !== null && $cachedDocuments !== false) { + $documents = $cachedDocuments; + $documentsCacheHit = true; + } else { + $documents = $dbForProject->find($collectionTableId, $queries); + $dbForProject->cache->save($documentsCacheKey, $documents, $ttl); + } + + $response->addHeader('X-Appwrite-Cache-Documents', $documentsCacheHit ? 'hit' : 'miss'); + + + if ($includeTotal) { + $cachedTotal = $dbForProject->cache->load($totalCacheKey, $ttl); + if ($cachedTotal !== null && $cachedTotal !== false) { + $total = $cachedTotal; + $totalDocumentsCacheHit = true; + } else { + $total = $dbForProject->count($collectionTableId, $queries, APP_LIMIT_COUNT); + $dbForProject->cache->save($totalCacheKey, $total, $ttl); + } + } else { + $total = 0; + } + + $response->addHeader('X-Appwrite-Cache-Documents-Total', $totalDocumentsCacheHit ? 'hit' : 'miss'); + + } else { + // has selects, allow relationship on documents + $documents = $dbForProject->find($collectionTableId, $queries); + $total = $includeTotal ? $dbForProject->count($collectionTableId, $queries, APP_LIMIT_COUNT) : 0; + } + } else { // has no selects, disable relationship loading on documents /* @type Document[] $documents */ @@ -162,8 +217,7 @@ class XList extends Action document: $document, dbForProject: $dbForProject, collectionsCache: $collectionsCache, - authorization: $authorization, - operations: $operations + operations: $operations, ); } @@ -171,6 +225,9 @@ class XList extends Action ->addMetric(METRIC_DATABASES_OPERATIONS_READS, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); + + + $response->dynamic(new Document([ 'total' => $total, // rows or documents diff --git a/tests/e2e/Services/Databases/Legacy/DatabasesBase.php b/tests/e2e/Services/Databases/Legacy/DatabasesBase.php index 409668dc46..7c75d20aba 100644 --- a/tests/e2e/Services/Databases/Legacy/DatabasesBase.php +++ b/tests/e2e/Services/Databases/Legacy/DatabasesBase.php @@ -2375,6 +2375,107 @@ trait DatabasesBase return ['documents' => $documents['body']['documents'], 'databaseId' => $databaseId]; } + /** + * @depends testCreateDocument + */ + public function testListDocumentsWithCache(array $data): void + { + $databaseId = $data['databaseId']; + + // useCache=true + select queries + $documents1 = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $data['moviesId'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ + Query::select(['title', 'releaseYear', '$id'])->toString(), + Query::orderAsc('releaseYear')->toString(), + ], + 'useCache' => true, + 'ttl' => 60, + ]); + + $this->assertEquals(200, $documents1['headers']['status-code']); + $this->assertCount(3, $documents1['body']['documents']); + $this->assertEquals(1944, $documents1['body']['documents'][0]['releaseYear']); + $this->assertEquals(2017, $documents1['body']['documents'][1]['releaseYear']); + $this->assertEquals(2019, $documents1['body']['documents'][2]['releaseYear']); + + $this->assertArrayHasKey('title', $documents1['body']['documents'][0]); + $this->assertArrayHasKey('releaseYear', $documents1['body']['documents'][0]); + $this->assertArrayHasKey('$id', $documents1['body']['documents'][0]); + + // First request should be MISS + $this->assertArrayHasKey('X-Appwrite-Cache-Documents', $documents1['headers']); + $this->assertEquals('miss', $documents1['headers']['X-Appwrite-Cache-Documents']); + $this->assertArrayHasKey('X-Appwrite-Cache-Documents-Total', $documents1['headers']); + $this->assertEquals('miss', $documents1['headers']['X-Appwrite-Cache-Documents-Total']); + + // Should return cached results + $documents2 = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $data['moviesId'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ + Query::select(['title', 'releaseYear', '$id'])->toString(), + Query::orderAsc('releaseYear')->toString(), + ], + 'useCache' => true, + 'ttl' => 60, + ]); + + $this->assertEquals(200, $documents2['headers']['status-code']); + $this->assertCount(3, $documents2['body']['documents']); + + // Cached results match the first request + $this->assertEquals($documents1['body']['documents'][0]['$id'], $documents2['body']['documents'][0]['$id']); + $this->assertEquals($documents1['body']['documents'][0]['title'], $documents2['body']['documents'][0]['title']); + $this->assertEquals($documents1['body']['documents'][0]['releaseYear'], $documents2['body']['documents'][0]['releaseYear']); + + // Second request should hit cache + $this->assertArrayHasKey('X-Appwrite-Cache-Documents', $documents2['headers']); + $this->assertEquals('hit', $documents2['headers']['X-Appwrite-Cache-Documents']); + $this->assertArrayHasKey('X-Appwrite-Cache-Documents-Total', $documents2['headers']); + $this->assertEquals('hit', $documents2['headers']['X-Appwrite-Cache-Documents-Total']); + + // useCache=false - should still work and no cache headers + $documents3 = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $data['moviesId'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ + Query::select(['title', 'releaseYear', '$id'])->toString(), + Query::orderAsc('releaseYear')->toString(), + ], + 'useCache' => false, + ]); + + $this->assertEquals(200, $documents3['headers']['status-code']); + $this->assertCount(3, $documents3['body']['documents']); + $this->assertEquals(1944, $documents3['body']['documents'][0]['releaseYear']); + + // Verify cache headers are not present when useCache=false + $this->assertArrayNotHasKey('X-Appwrite-Cache-Documents', $documents3['headers']); + $this->assertArrayNotHasKey('X-Appwrite-Cache-Documents-Total', $documents3['headers']); + + // Test with total=false + $documents4 = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $data['moviesId'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => [ + Query::select(['title', 'releaseYear', '$id'])->toString(), + Query::orderAsc('releaseYear')->toString(), + ], + 'useCache' => true, + 'total' => false, + ]); + + $this->assertEquals(200, $documents4['headers']['status-code']); + $this->assertEquals(0, $documents4['body']['total']); + $this->assertCount(3, $documents4['body']['documents']); + } + /** * @depends testListDocuments */