From c5b8ed9cc1d92af07a9d48e37fc4bde1aec6c8ec Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 8 Apr 2026 23:02:34 +1200 Subject: [PATCH] feat(databases): cache list responses without requiring a select query --- .../Databases/Collections/Documents/XList.php | 90 +++++++++---------- .../Http/TablesDB/Tables/Rows/XList.php | 2 +- .../e2e/Services/Databases/DatabasesBase.php | 47 ++++++++++ 3 files changed, 91 insertions(+), 48 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php index 716638ab14..97588630d5 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php @@ -72,7 +72,7 @@ class XList extends Action ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID to read uncommitted changes within the transaction.', true, ['dbForProject']) ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) - ->param('ttl', 0, new Range(min: 0, max: 86400), 'TTL (seconds) for caching list responses that include a select query. Responses are stored in an in-memory key-value cache, keyed per project, collection, schema version (attributes and indexes), caller authorization roles, and the exact query — so users with different permissions never share cached entries. Schema changes invalidate cached entries automatically; document writes do not, so choose a TTL you are comfortable serving as stale data. Set to 0 to disable caching. Must be between 0 and 86400 (24 hours).', true) + ->param('ttl', 0, new Range(min: 0, max: 86400), 'TTL (seconds) for caching list responses. Responses are stored in an in-memory key-value cache, keyed per project, collection, schema version (attributes and indexes), caller authorization roles, and the exact query — so users with different permissions never share cached entries. Schema changes invalidate cached entries automatically; document writes do not, so choose a TTL you are comfortable serving as stale data. Set to 0 to disable caching. Must be between 0 and 86400 (24 hours).', true) ->inject('response') ->inject('dbForProject') ->inject('user') @@ -127,63 +127,59 @@ class XList extends Action } try { - $selectQueries = Query::groupByType($queries)['selections'] ?? []; + $hasSelects = ! empty(Query::groupByType($queries)['selections'] ?? []); $collectionTableId = 'database_' . $database->getSequence() . '_collection_' . $collection->getSequence(); + // When there are no select queries, relationship loading is skipped on the + // underlying find() to avoid pulling related documents the caller did not ask for. + $find = $hasSelects + ? fn () => $dbForDatabases->find($collectionTableId, $queries) + : fn () => $dbForDatabases->skipRelationships(fn () => $dbForDatabases->find($collectionTableId, $queries)); + // Use transaction-aware document retrieval if transactionId is provided if ($transactionId !== null) { $documents = $transactionState->listDocuments($database, $collectionTableId, $transactionId, $queries); $total = $includeTotal ? $transactionState->countDocuments($database, $collectionTableId, $transactionId, $queries) : 0; - } elseif (! empty($selectQueries)) { + } elseif ((int)$ttl > 0) { + $cacheKey = $this->getListCacheKey($dbForProject, $collectionId); + $roles = $dbForProject->getAuthorization()->getRoles(); + $documentsField = $this->getListCacheField($collection, $roles, $queries, self::LIST_CACHE_FIELD_DOCUMENTS); - if ((int)$ttl > 0) { - $cacheKey = $this->getListCacheKey($dbForProject, $collectionId); - $roles = $dbForProject->getAuthorization()->getRoles(); - $documentsField = $this->getListCacheField($collection, $roles, $queries, self::LIST_CACHE_FIELD_DOCUMENTS); + $documentsCacheHit = false; + $cachedDocuments = $dbForProject->getCache()->load($cacheKey, $ttl, $documentsField); - $documentsCacheHit = false; - $cachedDocuments = $dbForProject->getCache()->load($cacheKey, $ttl, $documentsField); - - if ($cachedDocuments !== null && - $cachedDocuments !== false && - \is_array($cachedDocuments)) { - $documents = \array_map(function ($doc) { - return new Document($doc); - }, $cachedDocuments); - $documentsCacheHit = true; - } else { - $documents = $dbForDatabases->find($collectionTableId, $queries); - - // Convert Document objects to arrays for caching - $documentsArray = \array_map(function ($doc) { - return $doc->getArrayCopy(); - }, $documents); - $dbForProject->getCache()->save($cacheKey, $documentsArray, $documentsField); - } - - if ($includeTotal) { - $totalField = $this->getListCacheField($collection, $roles, $queries, self::LIST_CACHE_FIELD_TOTAL); - $cachedTotal = $dbForProject->getCache()->load($cacheKey, $ttl, $totalField); - if ($cachedTotal !== null && $cachedTotal !== false) { - $total = $cachedTotal; - } else { - $total = $dbForProject->count($collectionTableId, $queries, APP_LIMIT_COUNT); - $dbForProject->getCache()->save($cacheKey, $total, $totalField); - } - } else { - $total = 0; - } - - $response->addHeader('X-Appwrite-Cache', $documentsCacheHit ? 'hit' : 'miss'); + if ($cachedDocuments !== null && + $cachedDocuments !== false && + \is_array($cachedDocuments)) { + $documents = \array_map(function ($doc) { + return new Document($doc); + }, $cachedDocuments); + $documentsCacheHit = true; } else { - // has selects, allow relationship on documents - $documents = $dbForDatabases->find($collectionTableId, $queries); - $total = $includeTotal ? $dbForDatabases->count($collectionTableId, $queries, APP_LIMIT_COUNT) : 0; + $documents = $find(); + + // Convert Document objects to arrays for caching + $documentsArray = \array_map(function ($doc) { + return $doc->getArrayCopy(); + }, $documents); + $dbForProject->getCache()->save($cacheKey, $documentsArray, $documentsField); } + if ($includeTotal) { + $totalField = $this->getListCacheField($collection, $roles, $queries, self::LIST_CACHE_FIELD_TOTAL); + $cachedTotal = $dbForProject->getCache()->load($cacheKey, $ttl, $totalField); + if ($cachedTotal !== null && $cachedTotal !== false) { + $total = $cachedTotal; + } else { + $total = $dbForProject->count($collectionTableId, $queries, APP_LIMIT_COUNT); + $dbForProject->getCache()->save($cacheKey, $total, $totalField); + } + } else { + $total = 0; + } + + $response->addHeader('X-Appwrite-Cache', $documentsCacheHit ? 'hit' : 'miss'); } else { - // has no selects, disable relationship loading on documents - /* @type Document[] $documents */ - $documents = $dbForDatabases->skipRelationships(fn () => $dbForDatabases->find($collectionTableId, $queries)); + $documents = $find(); $total = $includeTotal ? $dbForDatabases->count($collectionTableId, $queries, APP_LIMIT_COUNT) : 0; } } catch (OrderException $e) { diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php index 617081439d..91c62aea05 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php @@ -57,7 +57,7 @@ class XList extends DocumentXList ->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true) ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID to read uncommitted changes within the transaction.', true, ['dbForProject']) ->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true) - ->param('ttl', 0, new Range(min: 0, max: 86400), 'TTL (seconds) for caching list responses that include a select query. Responses are stored in an in-memory key-value cache, keyed per project, table, schema version (columns and indexes), caller authorization roles, and the exact query — so users with different permissions never share cached entries. Schema changes invalidate cached entries automatically; row writes do not, so choose a TTL you are comfortable serving as stale data. Set to 0 to disable caching. Must be between 0 and 86400 (24 hours).', true) + ->param('ttl', 0, new Range(min: 0, max: 86400), 'TTL (seconds) for caching list responses. Responses are stored in an in-memory key-value cache, keyed per project, table, schema version (columns and indexes), caller authorization roles, and the exact query — so users with different permissions never share cached entries. Schema changes invalidate cached entries automatically; row writes do not, so choose a TTL you are comfortable serving as stale data. Set to 0 to disable caching. Must be between 0 and 86400 (24 hours).', true) ->inject('response') ->inject('dbForProject') ->inject('user') diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index cc06da37ef..2c5e587fc2 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -3527,6 +3527,53 @@ trait DatabasesBase $this->assertEquals('miss', $documents3['headers']['x-appwrite-cache']); } + public function testListDocumentsCachedWithoutSelectQuery(): void + { + if (!$this->getSupportForAttributes()) { + $this->markTestSkipped('Attributes are not supported by this database adapter'); + return; + } + $data = $this->setupDocuments(); + $databaseId = $data['databaseId']; + $docIds = $data['documentIds']; + + // No Query::select(...) at all — ttl alone should enable caching. + $queries = [ + Query::equal('$id', $docIds)->toString(), + Query::orderAsc('releaseYear')->toString(), + ]; + + // 1. First request populates the cache. + $documents1 = $this->client->call(Client::METHOD_GET, $this->getRecordUrl($databaseId, $data['moviesId']), array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => $queries, + 'ttl' => 60, + ]); + + $this->assertEquals(200, $documents1['headers']['status-code']); + $this->assertArrayHasKey('x-appwrite-cache', $documents1['headers']); + $this->assertEquals('miss', $documents1['headers']['x-appwrite-cache']); + + // 2. Same request hits cache — proves the gate is ttl > 0, not the presence of a select query. + $documents2 = $this->client->call(Client::METHOD_GET, $this->getRecordUrl($databaseId, $data['moviesId']), array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => $queries, + 'ttl' => 60, + ]); + + $this->assertEquals(200, $documents2['headers']['status-code']); + $this->assertArrayHasKey('x-appwrite-cache', $documents2['headers']); + $this->assertEquals('hit', $documents2['headers']['x-appwrite-cache']); + $this->assertSame( + $documents1['body'][$this->getRecordResource()], + $documents2['body'][$this->getRecordResource()] + ); + } + public function testListDocumentsCachePurgedByUpdate(): void { if (!$this->getSupportForAttributes()) {