Merge pull request #11820 from appwrite/fix-ttl-message

This commit is contained in:
Jake Barnby
2026-04-09 03:39:48 +12:00
committed by GitHub
8 changed files with 275 additions and 88 deletions
@@ -7,9 +7,13 @@ use Appwrite\Platform\Action as AppwriteAction;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Operator;
use Utopia\Database\Query;
class Action extends AppwriteAction
{
public const LIST_CACHE_FIELD_DOCUMENTS = 'documents';
public const LIST_CACHE_FIELD_TOTAL = 'total';
private string $context = DATABASE_TYPE_LEGACY;
public function getDatabaseType(): string
@@ -101,4 +105,67 @@ class Action extends AppwriteAction
return $data;
}
/**
* Stable Redis key for a collection's cached list responses.
*
* All variations (schema × roles × queries) for a single collection live as
* fields inside this one Redis hash, so purging every cached entry for a
* collection is a single O(1) DEL regardless of how many variations have
* been cached.
*/
protected function getListCacheKey(Database $dbForProject, string $collectionId): string
{
return \sprintf(
'%s-cache:%s:%s:%s:collection:%s',
$dbForProject->getCacheName(),
$dbForProject->getAdapter()->getHostname(),
$dbForProject->getNamespace(),
$dbForProject->getTenant(),
$collectionId,
);
}
/**
* Hash field for a single variation of a cached list response.
*
* Scoped by the collection schema (attributes + indexes), the caller's
* authorization roles, the exact query set, and the field type — so users
* with different permissions never share entries.
*
* @param Document $collection Collection document (for schema hash)
* @param array<mixed> $roles Caller authorization roles
* @param array<Query|string> $queries Queries for this list call
* @param string $type LIST_CACHE_FIELD_DOCUMENTS or LIST_CACHE_FIELD_TOTAL
*/
protected function getListCacheField(Document $collection, array $roles, array $queries, string $type): string
{
$schemaHash = \md5(
\json_encode($collection->getAttribute('attributes', []))
. \json_encode($collection->getAttribute('indexes', []))
);
$serialized = \array_map(
static fn ($query) => $query instanceof Query ? $query->toArray() : $query,
$queries,
);
return \sprintf(
'%s:%s:%s:%s',
$schemaHash,
\md5(\json_encode($roles)),
\md5(\json_encode($serialized)),
$type,
);
}
/**
* Purge every cached list response for a collection.
*
* One DEL on the collection's Redis hash, clearing all variations at once.
*/
protected function purgeListCache(Database $dbForProject, string $collectionId): bool
{
return $dbForProject->getCache()->purge($this->getListCacheKey($dbForProject, $collectionId));
}
}
@@ -3,34 +3,29 @@
namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections;
use Appwrite\Extend\Exception;
use Appwrite\Platform\Modules\Databases\Http\Databases\Action as DatabasesAction;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Platform\Action as UtopiaAction;
use Utopia\Platform\Scope\HTTP;
abstract class Action extends UtopiaAction
abstract class Action extends DatabasesAction
{
/**
* The current API context (either 'table' or 'collection').
*/
private ?string $context = COLLECTIONS;
private ?string $databaseType = LEGACY;
/**
* Get the response model used in the SDK and HTTP responses.
*/
abstract protected function getResponseModel(): string;
public function setHttpPath(string $path): UtopiaAction
public function setHttpPath(string $path): self
{
if (\str_contains($path, '/tablesdb')) {
$this->context = TABLES;
$this->databaseType = TABLESDB;
} elseif (\str_contains($path, '/vectorsdb')) {
$this->databaseType = VECTORSDB;
}
return parent::setHttpPath($path);
parent::setHttpPath($path);
return $this;
}
/**
@@ -41,14 +36,6 @@ abstract class Action extends UtopiaAction
return $this->context;
}
/**
* Get the current API database type.
*/
protected function getDatabaseType(): string
{
return $this->databaseType;
}
/**
* Get the key used in event parameters (e.g., 'collectionId' or 'tableId').
*/
@@ -72,7 +72,7 @@ class XList extends Action
->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true)
->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID to read uncommitted changes within the transaction.', true, ['dbForProject'])
->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true)
->param('ttl', 0, new Range(min: 0, max: 86400), 'TTL (seconds) for cached responses when caching is enabled for select queries. Must be between 0 and 86400 (24 hours).', true)
->param('ttl', 0, new Range(min: 0, max: 86400), 'TTL (seconds) for caching list responses. Responses are stored in an in-memory key-value cache, keyed per project, collection, schema version (attributes and indexes), caller authorization roles, and the exact query — so users with different permissions never share cached entries. Schema changes invalidate cached entries automatically; document writes do not, so choose a TTL you are comfortable serving as stale data. Set to 0 to disable caching. Must be between 0 and 86400 (24 hours).', true)
->inject('response')
->inject('dbForProject')
->inject('user')
@@ -127,84 +127,59 @@ class XList extends Action
}
try {
$selectQueries = Query::groupByType($queries)['selections'] ?? [];
$hasSelects = ! empty(Query::groupByType($queries)['selections'] ?? []);
$collectionTableId = 'database_' . $database->getSequence() . '_collection_' . $collection->getSequence();
// When there are no select queries, relationship loading is skipped on the
// underlying find() to avoid pulling related documents the caller did not ask for.
$find = $hasSelects
? fn () => $dbForDatabases->find($collectionTableId, $queries)
: fn () => $dbForDatabases->skipRelationships(fn () => $dbForDatabases->find($collectionTableId, $queries));
// Use transaction-aware document retrieval if transactionId is provided
if ($transactionId !== null) {
$documents = $transactionState->listDocuments($database, $collectionTableId, $transactionId, $queries);
$total = $includeTotal ? $transactionState->countDocuments($database, $collectionTableId, $transactionId, $queries) : 0;
} elseif (! empty($selectQueries)) {
} elseif ((int)$ttl > 0) {
$cacheKey = $this->getListCacheKey($dbForProject, $collectionId);
$roles = $dbForProject->getAuthorization()->getRoles();
$documentsField = $this->getListCacheField($collection, $roles, $queries, self::LIST_CACHE_FIELD_DOCUMENTS);
if ((int)$ttl > 0) {
$serializedQueries = [];
foreach ($queries as $query) {
$serializedQueries[] = $query instanceof Query ? $query->toArray() : $query;
}
$hostname = $dbForProject->getAdapter()->getHostname();
$roles = $dbForProject->getAuthorization()->getRoles();
$schemaHash = \md5(\json_encode($collection->getAttribute('attributes', [])) . \json_encode($collection->getAttribute('indexes', [])));
$cacheKeyBase = \sprintf(
'%s-cache-%s:%s:%s:collection:%s:%s:user:%s:%s',
$dbForProject->getCacheName(),
$hostname,
$dbForProject->getNamespace(),
$dbForProject->getTenant(),
$collectionId,
$schemaHash,
\md5(\json_encode($roles)),
\md5(\json_encode($serializedQueries))
);
$documentsCacheKey = $cacheKeyBase . ':documents';
$totalCacheKey = $cacheKeyBase . ':total';
$documentsCacheHit = $totalDocumentsCacheHit = false;
$cachedDocuments = $dbForProject->getCache()->load($documentsCacheKey, $ttl);
if ($cachedDocuments !== null &&
$cachedDocuments !== false &&
\is_array($cachedDocuments)) {
$documents = \array_map(function ($doc) {
return new Document($doc);
}, $cachedDocuments);
$documentsCacheHit = true;
} else {
$documents = $dbForDatabases->find($collectionTableId, $queries);
// Convert Document objects to arrays for caching
$documentsArray = \array_map(function ($doc) {
return $doc->getArrayCopy();
}, $documents);
$dbForProject->getCache()->save($documentsCacheKey, $documentsArray);
}
if ($includeTotal) {
$cachedTotal = $dbForProject->getCache()->load($totalCacheKey, $ttl);
if ($cachedTotal !== null && $cachedTotal !== false) {
$total = $cachedTotal;
$totalDocumentsCacheHit = true;
} else {
$total = $dbForProject->count($collectionTableId, $queries, APP_LIMIT_COUNT);
$dbForProject->getCache()->save($totalCacheKey, $total);
}
} else {
$total = 0;
}
$response->addHeader('X-Appwrite-Cache', $documentsCacheHit ? 'hit' : 'miss');
$documentsCacheHit = false;
$cachedDocuments = $dbForProject->getCache()->load($cacheKey, $ttl, $documentsField);
if ($cachedDocuments !== null &&
$cachedDocuments !== false &&
\is_array($cachedDocuments)) {
$documents = \array_map(function ($doc) {
return new Document($doc);
}, $cachedDocuments);
$documentsCacheHit = true;
} else {
// has selects, allow relationship on documents
$documents = $dbForDatabases->find($collectionTableId, $queries);
$total = $includeTotal ? $dbForDatabases->count($collectionTableId, $queries, APP_LIMIT_COUNT) : 0;
$documents = $find();
// Convert Document objects to arrays for caching
$documentsArray = \array_map(function ($doc) {
return $doc->getArrayCopy();
}, $documents);
$dbForProject->getCache()->save($cacheKey, $documentsArray, $documentsField);
}
if ($includeTotal) {
$totalField = $this->getListCacheField($collection, $roles, $queries, self::LIST_CACHE_FIELD_TOTAL);
$cachedTotal = $dbForProject->getCache()->load($cacheKey, $ttl, $totalField);
if ($cachedTotal !== null && $cachedTotal !== false) {
$total = $cachedTotal;
} else {
$total = $dbForDatabases->count($collectionTableId, $queries, APP_LIMIT_COUNT);
$dbForProject->getCache()->save($cacheKey, $total, $totalField);
}
} else {
$total = 0;
}
$response->addHeader('X-Appwrite-Cache', $documentsCacheHit ? 'hit' : 'miss');
} else {
// has no selects, disable relationship loading on documents
/* @type Document[] $documents */
$documents = $dbForDatabases->skipRelationships(fn () => $dbForDatabases->find($collectionTableId, $queries));
$documents = $find();
$total = $includeTotal ? $dbForDatabases->count($collectionTableId, $queries, APP_LIMIT_COUNT) : 0;
}
} catch (OrderException $e) {
@@ -68,6 +68,7 @@ class Update extends Action
->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true)
->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true)
->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true)
->param('purge', false, new Boolean(true), 'When true, purge all cached list responses for this collection as part of the update. Use this to force readers to see fresh data immediately instead of waiting for the cache TTL to expire.', true)
->inject('response')
->inject('dbForProject')
->inject('getDatabasesDB')
@@ -76,7 +77,7 @@ class Update extends Action
->callback($this->action(...));
}
public function action(string $databaseId, string $collectionId, ?string $name, ?array $permissions, bool $documentSecurity, bool $enabled, UtopiaResponse $response, Database $dbForProject, callable $getDatabasesDB, Event $queueForEvents, Authorization $authorization): void
public function action(string $databaseId, string $collectionId, ?string $name, ?array $permissions, bool $documentSecurity, bool $enabled, bool $purge, UtopiaResponse $response, Database $dbForProject, callable $getDatabasesDB, Event $queueForEvents, Authorization $authorization): void
{
$database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
if ($database->isEmpty()) {
@@ -117,6 +118,10 @@ class Update extends Action
->setParam('databaseId', $databaseId)
->setParam($this->getEventsParamKey(), $collection->getId());
if ($purge) {
$this->purgeListCache($dbForProject, $collectionId);
}
$this->addRowBytesInfo($collection, $dbForProject);
$response->dynamic($collection, $this->getResponseModel());
@@ -58,6 +58,7 @@ class Update extends CollectionUpdate
->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true)
->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](https://appwrite.io/docs/permissions).', true)
->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true)
->param('purge', false, new Boolean(true), 'When true, purge all cached list responses for this collection as part of the update. Use this to force readers to see fresh data immediately instead of waiting for the cache TTL to expire.', true)
->inject('response')
->inject('dbForProject')
->inject('getDatabasesDB')
@@ -57,7 +57,7 @@ class XList extends DocumentXList
->param('queries', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true)
->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID to read uncommitted changes within the transaction.', true, ['dbForProject'])
->param('total', true, new Boolean(true), 'When set to false, the total count returned will be 0 and will not be calculated.', true)
->param('ttl', 0, new Range(min: 0, max: 86400), 'TTL (seconds) for cached responses when caching is enabled for select queries. Must be between 0 and 86400 (24 hours).', true)
->param('ttl', 0, new Range(min: 0, max: 86400), 'TTL (seconds) for caching list responses. Responses are stored in an in-memory key-value cache, keyed per project, table, schema version (columns and indexes), caller authorization roles, and the exact query — so users with different permissions never share cached entries. Schema changes invalidate cached entries automatically; row writes do not, so choose a TTL you are comfortable serving as stale data. Set to 0 to disable caching. Must be between 0 and 86400 (24 hours).', true)
->inject('response')
->inject('dbForProject')
->inject('user')
@@ -60,6 +60,7 @@ class Update extends CollectionUpdate
->param('permissions', null, new Nullable(new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](https://appwrite.io/docs/permissions).', true)
->param('rowSecurity', false, new Boolean(true), 'Enables configuring permissions for individual rows. A user needs one of row or table-level permissions to access a row. [Learn more about permissions](https://appwrite.io/docs/permissions).', true)
->param('enabled', true, new Boolean(), 'Is table enabled? When set to \'disabled\', users cannot access the table but Server SDKs with and API key can still read and write to the table. No data is lost when this is toggled.', true)
->param('purge', false, new Boolean(true), 'When true, purge all cached list responses for this table as part of the update. Use this to force readers to see fresh data immediately instead of waiting for the cache TTL to expire.', true)
->inject('response')
->inject('dbForProject')
->inject('getDatabasesDB')
@@ -3527,6 +3527,157 @@ trait DatabasesBase
$this->assertEquals('miss', $documents3['headers']['x-appwrite-cache']);
}
public function testListDocumentsCachedWithoutSelectQuery(): void
{
if (!$this->getSupportForAttributes()) {
$this->markTestSkipped('Attributes are not supported by this database adapter');
return;
}
$data = $this->setupDocuments();
$databaseId = $data['databaseId'];
$docIds = $data['documentIds'];
// No Query::select(...) at all — ttl alone should enable caching.
$queries = [
Query::equal('$id', $docIds)->toString(),
Query::orderAsc('releaseYear')->toString(),
];
// 1. First request populates the cache.
$documents1 = $this->client->call(Client::METHOD_GET, $this->getRecordUrl($databaseId, $data['moviesId']), array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => $queries,
'ttl' => 60,
]);
$this->assertEquals(200, $documents1['headers']['status-code']);
$this->assertArrayHasKey('x-appwrite-cache', $documents1['headers']);
$this->assertEquals('miss', $documents1['headers']['x-appwrite-cache']);
// 2. Same request hits cache — proves the gate is ttl > 0, not the presence of a select query.
$documents2 = $this->client->call(Client::METHOD_GET, $this->getRecordUrl($databaseId, $data['moviesId']), array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => $queries,
'ttl' => 60,
]);
$this->assertEquals(200, $documents2['headers']['status-code']);
$this->assertArrayHasKey('x-appwrite-cache', $documents2['headers']);
$this->assertEquals('hit', $documents2['headers']['x-appwrite-cache']);
$this->assertSame(
$documents1['body'][$this->getRecordResource()],
$documents2['body'][$this->getRecordResource()]
);
}
public function testListDocumentsCachePurgedByUpdate(): void
{
if (!$this->getSupportForAttributes()) {
$this->markTestSkipped('Attributes are not supported by this database adapter');
return;
}
$data = $this->setupDocuments();
$databaseId = $data['databaseId'];
$docIds = $data['documentIds'];
// Use different select queries from other cache tests to avoid cache key collision.
$queries = [
Query::equal('$id', $docIds)->toString(),
Query::select(['title', 'tagline', '$id'])->toString(),
Query::orderAsc('$createdAt')->toString(),
];
// 1. First request populates the cache.
$documents1 = $this->client->call(Client::METHOD_GET, $this->getRecordUrl($databaseId, $data['moviesId']), array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => $queries,
'ttl' => 300,
]);
$this->assertEquals(200, $documents1['headers']['status-code']);
$this->assertEquals('miss', $documents1['headers']['x-appwrite-cache']);
// 2. Same request hits cache.
$documents2 = $this->client->call(Client::METHOD_GET, $this->getRecordUrl($databaseId, $data['moviesId']), array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => $queries,
'ttl' => 300,
]);
$this->assertEquals(200, $documents2['headers']['status-code']);
$this->assertEquals('hit', $documents2['headers']['x-appwrite-cache']);
// 3. Update the collection/table with purge=true to invalidate all cached list responses.
$update = $this->client->call(Client::METHOD_PUT, $this->getContainerUrl($databaseId, $data['moviesId']), array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]), [
'name' => 'Movies',
'enabled' => true,
$this->getSecurityParam() => true,
'purge' => true,
]);
$this->assertEquals(200, $update['headers']['status-code']);
// 4. Same request should now miss cache because purge=true cleared the hash.
$documents3 = $this->client->call(Client::METHOD_GET, $this->getRecordUrl($databaseId, $data['moviesId']), array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => $queries,
'ttl' => 300,
]);
$this->assertEquals(200, $documents3['headers']['status-code']);
$this->assertEquals('miss', $documents3['headers']['x-appwrite-cache']);
// 5. Re-reading without purge should hit the freshly populated cache.
$documents4 = $this->client->call(Client::METHOD_GET, $this->getRecordUrl($databaseId, $data['moviesId']), array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => $queries,
'ttl' => 300,
]);
$this->assertEquals(200, $documents4['headers']['status-code']);
$this->assertEquals('hit', $documents4['headers']['x-appwrite-cache']);
// 6. Update without purge=true must NOT invalidate the cache.
$update2 = $this->client->call(Client::METHOD_PUT, $this->getContainerUrl($databaseId, $data['moviesId']), array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]), [
'name' => 'Movies',
'enabled' => true,
$this->getSecurityParam() => true,
]);
$this->assertEquals(200, $update2['headers']['status-code']);
$documents5 = $this->client->call(Client::METHOD_GET, $this->getRecordUrl($databaseId, $data['moviesId']), array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'queries' => $queries,
'ttl' => 300,
]);
$this->assertEquals(200, $documents5['headers']['status-code']);
$this->assertEquals('hit', $documents5['headers']['x-appwrite-cache']);
}
public function testGetDocument(): void
{
$data = $this->getDocumentsList();