diff --git a/app/config/collections.php b/app/config/collections.php index ee732d0856..0d9583b482 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2769,6 +2769,17 @@ $collections = [ '$id' => 'cache', 'name' => 'Cache', 'attributes' => [ + [ + '$id' => 'resource', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => 'accessedAt', 'type' => Database::VAR_INTEGER, @@ -2780,6 +2791,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'signature', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 274137b779..7c5502a064 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -803,8 +803,9 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->alias('/v1/storage/files/:fileId/preview', ['bucketId' => 'default']) ->desc('Get File Preview') ->groups(['api', 'storage']) - ->label('cache', true) ->label('scope', 'files.read') + ->label('cache', true) + ->label('cache.resource', 'file/{request.fileId}') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getFilePreview') @@ -1377,8 +1378,8 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') ->inject('usage') ->inject('mode') ->inject('deviceFiles') - ->inject('project') - ->action(function (string $bucketId, string $fileId, Response $response, Request $request, Database $dbForProject, Event $events, Audit $audits, Stats $usage, string $mode, Device $deviceFiles, Document $project) { + ->inject('deletes') + ->action(function (string $bucketId, string $fileId, Response $response, Request $request, Database $dbForProject, Event $events, Audit $audits, Stats $usage, string $mode, Device $deviceFiles, Delete $deletes) { $bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); if ( @@ -1417,9 +1418,11 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') } if ($deviceDeleted) { - $key = md5($request->getURI() . implode('*', $request->getParams())); - $cache = new Cache(new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId())); - $cache->purge($key); + $deletes + ->setType(DELETE_TYPE_CACHE_BY_RESOURCE) + ->setType(DELETE_TYPE_CACHE_BY_RESOURCE) + ->setResource('file/' . $fileId) + ; if ($bucket->getAttribute('permission') === 'bucket') { $deleted = Authorization::skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId)); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 61a5d54c23..821f0067ee 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -232,9 +232,11 @@ App::shutdown() ->inject('dbForProject') ->action(function (App $utopia, Request $request, Response $response, Document $project, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, string $mode, Database $dbForProject) { + $responsePayload = $response->getPayload(); + if (!empty($events->getEvent())) { if (empty($events->getPayload())) { - $events->setPayload($response->getPayload()); + $events->setPayload($responsePayload); } /** * Trigger functions. @@ -303,30 +305,70 @@ App::shutdown() } $route = $utopia->match($request); + $requestParams = $route->getParamsValues(); + $user = $audits->getUser(); + + $parseLabel = function ($label) use ($responsePayload, $requestParams, $user) { + preg_match_all('/{(.*?)}/', $label, $matches); + foreach ($matches[1] ?? [] as $pos => $match) { + $find = $matches[0][$pos]; + $parts = explode('.', $match); + + if (count($parts) !== 2) { + throw new Exception('Too less or too many parts', 400, Exception::GENERAL_ARGUMENT_INVALID); + } + + $namespace = $parts[0]; + $replace = $parts[1]; + + $params = match ($namespace) { + 'user' => (array)$user, + 'request' => $requestParams, + default => $responsePayload, + }; + + if (array_key_exists($replace, $params)) { + $label = \str_replace($find, $params[$replace], $label); + } + } + + return $label; + }; $useCache = $route->getLabel('cache', false); if ($useCache) { + $resource = null; $data = $response->getPayload(); if (!empty($data['payload'])) { + + $pattern = $route->getLabel('cache.resource', null); + if (!empty($pattern)) { + $resource = $parseLabel($pattern); + } + $key = md5($request->getURI() . implode('*', $request->getParams())); + + $data = json_encode([ + 'content-type' => $response->getContentType(), + 'payload' => base64_encode($data['payload']), + ]) ; + + $signature = md5($data); $cacheLog = $dbForProject->getDocument('cache', $key); if ($cacheLog->isEmpty()) { Authorization::skip(fn () => $dbForProject->createDocument('cache', new Document([ '$id' => $key, + 'resource' => $resource, 'accessedAt' => \time(), + 'signature' => $signature, ]))); } elseif (date('Y/m/d', \time()) > date('Y/m/d', $cacheLog->getAttribute('accessedAt'))) { $cacheLog->setAttribute('accessedAt', \time()); Authorization::skip(fn () => $dbForProject->updateDocument('cache', $cacheLog->getId(), $cacheLog)); } - $data = [ - 'content-type' => $response->getContentType(), - 'payload' => base64_encode($data['payload']), - ] ; - $cache = new Cache(new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId())); - $cache->save($key, json_encode($data)); + $cache->save($key, $data); } } diff --git a/app/init.php b/app/init.php index 8823dc564a..00ec1fc836 100644 --- a/app/init.php +++ b/app/init.php @@ -143,7 +143,8 @@ const DELETE_TYPE_USAGE = 'usage'; const DELETE_TYPE_REALTIME = 'realtime'; const DELETE_TYPE_BUCKETS = 'buckets'; const DELETE_TYPE_SESSIONS = 'sessions'; -const DELETE_TYPE_CACHE = 'cache'; +const DELETE_TYPE_CACHE_BY_TIMESTAMP = 'cacheByTimeStamp'; +const DELETE_TYPE_CACHE_BY_RESOURCE = 'cacheByResource'; // Mail Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_MAGIC_SESSION = 'magicSession'; diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 8f1a92e5a6..73df458854 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -131,7 +131,7 @@ $cli { (new Delete()) - ->setType(DELETE_TYPE_CACHE) + ->setType(DELETE_TYPE_CACHE_BY_TIMESTAMP) ->setTimestamp(time() - $interval) ->trigger(); } diff --git a/app/workers/deletes.php b/app/workers/deletes.php index c3ec31b82d..3d77cc3236 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -35,7 +35,6 @@ class DeletesV1 extends Worker { $project = new Document($this->args['project'] ?? []); $type = $this->args['type'] ?? ''; - switch (strval($type)) { case DELETE_TYPE_DOCUMENT: $document = new Document($this->args['document'] ?? []); @@ -110,10 +109,12 @@ class DeletesV1 extends Worker $this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']); break; - case DELETE_TYPE_CACHE: - $this->deleteCache($this->args['timestamp']); + case DELETE_TYPE_CACHE_BY_RESOURCE: + $this->deleteCacheByResource($project->getId()); + break; + case DELETE_TYPE_CACHE_BY_TIMESTAMP: + $this->deleteCacheByTimestamp(); break; - default: Console::error('No delete operation for type: ' . $type); break; @@ -124,13 +125,26 @@ class DeletesV1 extends Worker { } - /** - * @param int $timestamp + * @param string $projectId */ - protected function deleteCache(int $timestamp): void + protected function deleteCacheByResource(string $projectId): void { - $this->deleteForProjectIds(function (string $projectId) use ($timestamp) { + $this->deleteCacheFiles([ + new Query('resource', Query::TYPE_EQUAL, [$this->args['resource']]) + ]); + } + + protected function deleteCacheByTimestamp(): void + { + $this->deleteCacheFiles([ + new Query('accessedAt', Query::TYPE_LESSER, [$this->args['timestamp']]) + ]); + } + + protected function deleteCacheFiles($query): void + { + $this->deleteForProjectIds(function (string $projectId) use ($query) { $dbForProject = $this->getProjectDB($projectId); $cache = new Cache( @@ -139,7 +153,7 @@ class DeletesV1 extends Worker $this->deleteByGroup( 'cache', - [new Query('accessedAt', Query::TYPE_LESSER, [$timestamp])], + $query, $dbForProject, function (Document $document) use ($cache, $projectId) { $path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId(); diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index 057abe17f7..09cccd799c 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -12,6 +12,7 @@ class Delete extends Event protected ?int $timestamp1d = null; protected ?int $timestamp30m = null; protected ?Document $document = null; + protected ?string $resource = null; public function __construct() { @@ -93,6 +94,29 @@ class Delete extends Event return $this; } + /** + * Returns the resource for the delete event. + * + * @return string + */ + public function getResource(): string + { + return $this->resource; + } + + /** + * Sets the resource for the delete event. + * + * @param string $resource + * @return self + */ + public function setResource(string $resource): self + { + $this->resource = $resource; + + return $this; + } + /** * Returns the set document for the delete event. * @@ -103,6 +127,7 @@ class Delete extends Event return $this->document; } + /** * Executes this event and sends it to the deletes worker. * @@ -117,7 +142,8 @@ class Delete extends Event 'document' => $this->document, 'timestamp' => $this->timestamp, 'timestamp1d' => $this->timestamp1d, - 'timestamp30m' => $this->timestamp30m + 'timestamp30m' => $this->timestamp30m, + 'resource' => $this->resource, ]); } } diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index edcaa0f8a6..4937eddec2 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -490,7 +490,7 @@ trait StorageBase $this->assertEquals(204, $file['headers']['status-code']); $this->assertEmpty($file['body']); - + sleep(10); //upload again using the same ID $file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ 'content-type' => 'multipart/form-data',