some fixes

This commit is contained in:
shimon
2022-08-15 12:05:41 +03:00
parent cabe989555
commit 0dc485d8d7
8 changed files with 134 additions and 26 deletions
+22
View File
@@ -2769,6 +2769,17 @@ $collections = [
'$id' => 'cache',
'name' => 'Cache',
'attributes' => [
[
'$id' => 'resource',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => 'accessedAt',
'type' => Database::VAR_INTEGER,
@@ -2780,6 +2791,17 @@ $collections = [
'array' => false,
'filters' => [],
],
[
'$id' => 'signature',
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
+9 -6
View File
@@ -803,8 +803,9 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
->alias('/v1/storage/files/:fileId/preview', ['bucketId' => 'default'])
->desc('Get File Preview')
->groups(['api', 'storage'])
->label('cache', true)
->label('scope', 'files.read')
->label('cache', true)
->label('cache.resource', 'file/{request.fileId}')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFilePreview')
@@ -1377,8 +1378,8 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
->inject('usage')
->inject('mode')
->inject('deviceFiles')
->inject('project')
->action(function (string $bucketId, string $fileId, Response $response, Request $request, Database $dbForProject, Event $events, Audit $audits, Stats $usage, string $mode, Device $deviceFiles, Document $project) {
->inject('deletes')
->action(function (string $bucketId, string $fileId, Response $response, Request $request, Database $dbForProject, Event $events, Audit $audits, Stats $usage, string $mode, Device $deviceFiles, Delete $deletes) {
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
if (
@@ -1417,9 +1418,11 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
}
if ($deviceDeleted) {
$key = md5($request->getURI() . implode('*', $request->getParams()));
$cache = new Cache(new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()));
$cache->purge($key);
$deletes
->setType(DELETE_TYPE_CACHE_BY_RESOURCE)
->setType(DELETE_TYPE_CACHE_BY_RESOURCE)
->setResource('file/' . $fileId)
;
if ($bucket->getAttribute('permission') === 'bucket') {
$deleted = Authorization::skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getInternalId(), $fileId));
+49 -7
View File
@@ -232,9 +232,11 @@ App::shutdown()
->inject('dbForProject')
->action(function (App $utopia, Request $request, Response $response, Document $project, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, string $mode, Database $dbForProject) {
$responsePayload = $response->getPayload();
if (!empty($events->getEvent())) {
if (empty($events->getPayload())) {
$events->setPayload($response->getPayload());
$events->setPayload($responsePayload);
}
/**
* Trigger functions.
@@ -303,30 +305,70 @@ App::shutdown()
}
$route = $utopia->match($request);
$requestParams = $route->getParamsValues();
$user = $audits->getUser();
$parseLabel = function ($label) use ($responsePayload, $requestParams, $user) {
preg_match_all('/{(.*?)}/', $label, $matches);
foreach ($matches[1] ?? [] as $pos => $match) {
$find = $matches[0][$pos];
$parts = explode('.', $match);
if (count($parts) !== 2) {
throw new Exception('Too less or too many parts', 400, Exception::GENERAL_ARGUMENT_INVALID);
}
$namespace = $parts[0];
$replace = $parts[1];
$params = match ($namespace) {
'user' => (array)$user,
'request' => $requestParams,
default => $responsePayload,
};
if (array_key_exists($replace, $params)) {
$label = \str_replace($find, $params[$replace], $label);
}
}
return $label;
};
$useCache = $route->getLabel('cache', false);
if ($useCache) {
$resource = null;
$data = $response->getPayload();
if (!empty($data['payload'])) {
$pattern = $route->getLabel('cache.resource', null);
if (!empty($pattern)) {
$resource = $parseLabel($pattern);
}
$key = md5($request->getURI() . implode('*', $request->getParams()));
$data = json_encode([
'content-type' => $response->getContentType(),
'payload' => base64_encode($data['payload']),
]) ;
$signature = md5($data);
$cacheLog = $dbForProject->getDocument('cache', $key);
if ($cacheLog->isEmpty()) {
Authorization::skip(fn () => $dbForProject->createDocument('cache', new Document([
'$id' => $key,
'resource' => $resource,
'accessedAt' => \time(),
'signature' => $signature,
])));
} elseif (date('Y/m/d', \time()) > date('Y/m/d', $cacheLog->getAttribute('accessedAt'))) {
$cacheLog->setAttribute('accessedAt', \time());
Authorization::skip(fn () => $dbForProject->updateDocument('cache', $cacheLog->getId(), $cacheLog));
}
$data = [
'content-type' => $response->getContentType(),
'payload' => base64_encode($data['payload']),
] ;
$cache = new Cache(new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()));
$cache->save($key, json_encode($data));
$cache->save($key, $data);
}
}
+2 -1
View File
@@ -143,7 +143,8 @@ const DELETE_TYPE_USAGE = 'usage';
const DELETE_TYPE_REALTIME = 'realtime';
const DELETE_TYPE_BUCKETS = 'buckets';
const DELETE_TYPE_SESSIONS = 'sessions';
const DELETE_TYPE_CACHE = 'cache';
const DELETE_TYPE_CACHE_BY_TIMESTAMP = 'cacheByTimeStamp';
const DELETE_TYPE_CACHE_BY_RESOURCE = 'cacheByResource';
// Mail Types
const MAIL_TYPE_VERIFICATION = 'verification';
const MAIL_TYPE_MAGIC_SESSION = 'magicSession';
+1 -1
View File
@@ -131,7 +131,7 @@ $cli
{
(new Delete())
->setType(DELETE_TYPE_CACHE)
->setType(DELETE_TYPE_CACHE_BY_TIMESTAMP)
->setTimestamp(time() - $interval)
->trigger();
}
+23 -9
View File
@@ -35,7 +35,6 @@ class DeletesV1 extends Worker
{
$project = new Document($this->args['project'] ?? []);
$type = $this->args['type'] ?? '';
switch (strval($type)) {
case DELETE_TYPE_DOCUMENT:
$document = new Document($this->args['document'] ?? []);
@@ -110,10 +109,12 @@ class DeletesV1 extends Worker
$this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']);
break;
case DELETE_TYPE_CACHE:
$this->deleteCache($this->args['timestamp']);
case DELETE_TYPE_CACHE_BY_RESOURCE:
$this->deleteCacheByResource($project->getId());
break;
case DELETE_TYPE_CACHE_BY_TIMESTAMP:
$this->deleteCacheByTimestamp();
break;
default:
Console::error('No delete operation for type: ' . $type);
break;
@@ -124,13 +125,26 @@ class DeletesV1 extends Worker
{
}
/**
* @param int $timestamp
* @param string $projectId
*/
protected function deleteCache(int $timestamp): void
protected function deleteCacheByResource(string $projectId): void
{
$this->deleteForProjectIds(function (string $projectId) use ($timestamp) {
$this->deleteCacheFiles([
new Query('resource', Query::TYPE_EQUAL, [$this->args['resource']])
]);
}
protected function deleteCacheByTimestamp(): void
{
$this->deleteCacheFiles([
new Query('accessedAt', Query::TYPE_LESSER, [$this->args['timestamp']])
]);
}
protected function deleteCacheFiles($query): void
{
$this->deleteForProjectIds(function (string $projectId) use ($query) {
$dbForProject = $this->getProjectDB($projectId);
$cache = new Cache(
@@ -139,7 +153,7 @@ class DeletesV1 extends Worker
$this->deleteByGroup(
'cache',
[new Query('accessedAt', Query::TYPE_LESSER, [$timestamp])],
$query,
$dbForProject,
function (Document $document) use ($cache, $projectId) {
$path = APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId . DIRECTORY_SEPARATOR . $document->getId();
+27 -1
View File
@@ -12,6 +12,7 @@ class Delete extends Event
protected ?int $timestamp1d = null;
protected ?int $timestamp30m = null;
protected ?Document $document = null;
protected ?string $resource = null;
public function __construct()
{
@@ -93,6 +94,29 @@ class Delete extends Event
return $this;
}
/**
* Returns the resource for the delete event.
*
* @return string
*/
public function getResource(): string
{
return $this->resource;
}
/**
* Sets the resource for the delete event.
*
* @param string $resource
* @return self
*/
public function setResource(string $resource): self
{
$this->resource = $resource;
return $this;
}
/**
* Returns the set document for the delete event.
*
@@ -103,6 +127,7 @@ class Delete extends Event
return $this->document;
}
/**
* Executes this event and sends it to the deletes worker.
*
@@ -117,7 +142,8 @@ class Delete extends Event
'document' => $this->document,
'timestamp' => $this->timestamp,
'timestamp1d' => $this->timestamp1d,
'timestamp30m' => $this->timestamp30m
'timestamp30m' => $this->timestamp30m,
'resource' => $this->resource,
]);
}
}
+1 -1
View File
@@ -490,7 +490,7 @@ trait StorageBase
$this->assertEquals(204, $file['headers']['status-code']);
$this->assertEmpty($file['body']);
sleep(10);
//upload again using the same ID
$file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([
'content-type' => 'multipart/form-data',