From 1e9f3e38b200aec910774686b8e80bb20ed8757d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 13:10:20 +0545 Subject: [PATCH] storage stats --- app/controllers/api/storage.php | 64 ++++++++++++++++++++++++++++----- app/tasks/usage.php | 22 +++++++++++- src/Appwrite/Stats/Stats.php | 15 ++++++++ 3 files changed, 91 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 5a5c43de58..651985057a 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -54,7 +54,7 @@ App::post('/v1/storage/files') /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Stats\Stats $usage */ $file = $request->getFiles('file'); @@ -150,6 +150,8 @@ App::post('/v1/storage/files') $usage ->setParam('storage', $sizeActual) + ->setParam('storage.files.create', 1) + ->setParam('bucketId', 'default') ; $response->setStatusCode(Response::STATUS_CODE_CREATED); @@ -175,9 +177,11 @@ App::get('/v1/storage/files') ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, $search)] : []; @@ -189,6 +193,11 @@ App::get('/v1/storage/files') } } + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + $response->dynamic(new Document([ 'files' => $dbForInternal->find('files', $queries, $limit, $offset, [], [$orderType], $afterFile ?? null), 'sum' => $dbForInternal->count('files', $queries, APP_LIMIT_COUNT), @@ -209,16 +218,21 @@ App::get('/v1/storage/files/:fileId') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($fileId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); if (empty($file->getId())) { throw new Exception('File not found', 404); } - + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; $response->dynamic($file, Response::MODEL_FILE); }); @@ -249,11 +263,13 @@ App::get('/v1/storage/files/:fileId/preview') ->inject('response') ->inject('project') ->inject('dbForInternal') - ->action(function ($fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForInternal, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $stats */ $storage = 'files'; @@ -366,6 +382,11 @@ App::get('/v1/storage/files/:fileId/preview') $cache->save($key, $data); + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + $response ->setContentType($outputs[$output]) ->addHeader('Expires', $date) @@ -390,9 +411,11 @@ App::get('/v1/storage/files/:fileId/download') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($fileId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); @@ -424,6 +447,11 @@ App::get('/v1/storage/files/:fileId/download') $source = $compressor->decompress($source); + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + // Response $response ->setContentType($file->getAttribute('mimeType')) @@ -448,9 +476,11 @@ App::get('/v1/storage/files/:fileId/view') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($fileId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); $mimes = Config::getParam('storage-mimes'); @@ -490,6 +520,11 @@ App::get('/v1/storage/files/:fileId/view') $output = $compressor->decompress($source); $fileName = $file->getAttribute('name', ''); + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + // Response $response ->setContentType($contentType) @@ -520,7 +555,8 @@ App::put('/v1/storage/files/:fileId') ->inject('response') ->inject('dbForInternal') ->inject('audits') - ->action(function ($fileId, $read, $write, $response, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($fileId, $read, $write, $response, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ @@ -542,6 +578,11 @@ App::put('/v1/storage/files/:fileId') ->setParam('resource', 'storage/files/'.$file->getId()) ; + $usage + ->setParam('storage.files.update', 1) + ->setParam('bucketId', 'default') + ; + $response->dynamic($file, Response::MODEL_FILE); }); @@ -567,7 +608,7 @@ App::delete('/v1/storage/files/:fileId') /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); @@ -596,5 +637,10 @@ App::delete('/v1/storage/files/:fileId') ->setParam('eventData', $response->output($file, Response::MODEL_FILE)) ; + $usage + ->setParam('storage.files.delete', 1) + ->setParam('bucketId', 'default') + ; + $response->noContent(); }); \ No newline at end of file diff --git a/app/tasks/usage.php b/app/tasks/usage.php index fa8de4c86f..5230963098 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -32,7 +32,7 @@ use Utopia\Database\Validator\Authorization; * users.sessions.delete (project=x,provider=y) * * storage.buckets.CRUD (project=x) - * storage.files.CRUD (project=x,files=y) + * storage.files.CRUD (project=x,bucket=y) * * refactor later * - functions @@ -128,6 +128,26 @@ $cli 'table' => 'appwrite_usage_database_documents_delete', 'groupBy' => 'collectionId', ], + 'storage.buckets.bucketId.files.create' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_storage_files_create', + 'groupBy' => 'bucketId', + ], + 'storage.buckets.bucketId.files.read' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_storage_files_read', + 'groupBy' => 'bucketId', + ], + 'storage.buckets.bucketId.files.update' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_storage_files_update', + 'groupBy' => 'bucketId', + ], + 'storage.buckets.bucketId.files.delete' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_storage_files_delete', + 'groupBy' => 'bucketId', + ], ]; $attempts = 0; diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index 60609f1a4e..5c24e1ae94 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -131,6 +131,21 @@ class Stats } } + $storageMertics = [ + 'storage.files.create', + 'storage.files.read', + 'storage.files.update', + 'storage.files.delete', + ]; + + foreach ($storageMertics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $storageTags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? ''); + $this->statsd->increment($metric . $storageTags); + } + } + if ($storage >= 1) { $this->statsd->count('storage.all' . $tags, $storage); }