From cfe49b68927fb6b2be41e90dff2362d1071f4a05 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 13:54:45 +0530 Subject: [PATCH] feat(usage): refactored functions usage endpoint --- app/controllers/api/database.php | 261 ++++++++++++++++++++---------- app/controllers/api/functions.php | 95 +++++++++++ app/controllers/api/projects.php | 2 +- 3 files changed, 273 insertions(+), 85 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index db8a97a9ec..86f4029cdb 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1,5 +1,6 @@ isValid($default)) { throw new Exception('Length of default attribute exceeds attribute size', 400); } - } + } if (!\is_null($format)) { $name = \json_decode($format, true)['name']; @@ -110,16 +111,14 @@ $attributesCallback = function ($attribute, $response, $dbForExternal, $database $database ->setParam('type', DATABASE_TYPE_CREATE_ATTRIBUTE) - ->setParam('document', $attribute) - ; + ->setParam('document', $attribute); $usage->setParam('database.collections.update', 1); $audits ->setParam('event', 'database.attributes.create') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $attribute) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $attribute); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($attribute, Response::MODEL_ATTRIBUTE); @@ -167,9 +166,8 @@ App::post('/v1/database/collections') $audits ->setParam('event', 'database.collections.create') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $collection->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $collection->getArrayCopy()); $usage->setParam('database.collections.create', 1); @@ -250,6 +248,112 @@ App::get('/v1/database/collections/:collectionId') $response->dynamic($collection, Response::MODEL_COLLECTION); }); +App::get('/v1/database/usage') + ->desc('Get Database Usage') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'database') + ->label('sdk.method', 'getUsage') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForConsole') + ->inject('dbForInternal') + ->inject('register') + ->action(function ($range, $response, $dbForConsole, $dbForInternal, $register) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForConsole */ + /** @var Utopia\Database\Database $dbForInternal */ + /** @var Utopia\Registry\Registry $register */ + + $stats = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + + $metrics = [ + 'database.collections.create', + 'database.collections.read', + 'database.collections.update', + 'database.collections.delete', + 'database.documents.create', + 'database.documents.read', + 'database.documents.update', + 'database.documents.delete' + ]; + + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + } + } + + $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); + $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; + + $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['n + '])], 0, ['time'], [Database::ORDER_DESC]); + $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; + + Authorization::reset(); + + $response->json([ + 'range' => $range, + 'stats' => $stats, + 'requests' => [ + 'data' => $stats['requests'] ?? [], + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $stats['requests'] ?? [])), + ], + 'documents' => [ + 'data' => [], + 'total' => $documentsTotal, + ], + 'collections' => [ + 'data' => [], + 'total' => $collectionsTotal, + ] + ]); + }); + + +// :collectionId/usage +// 'reads', +// 'writes', +// 'updates', +// 'delete' + App::get('/v1/database/collections/:collectionId/logs') ->desc('List Collection Logs') ->groups(['api', 'database']) @@ -283,7 +387,7 @@ App::get('/v1/database/collections/:collectionId/logs') $audit = new Audit($dbForInternal); - $logs = $audit->getLogsByResource('database/collection/'.$collection->getId()); + $logs = $audit->getLogsByResource('database/collection/' . $collection->getId()); $output = []; @@ -335,8 +439,8 @@ App::get('/v1/database/collections/:collectionId/logs') $record = $geodb->get($log['ip']); if ($record) { - $output[$i]['countryCode'] = $locale->getText('countries.'.strtolower($record['country']['iso_code']), false) ? \strtolower($record['country']['iso_code']) : '--'; - $output[$i]['countryName'] = $locale->getText('countries.'.strtolower($record['country']['iso_code']), $locale->getText('locale.country.unknown')); + $output[$i]['countryCode'] = $locale->getText('countries.' . strtolower($record['country']['iso_code']), false) ? \strtolower($record['country']['iso_code']) : '--'; + $output[$i]['countryName'] = $locale->getText('countries.' . strtolower($record['country']['iso_code']), $locale->getText('locale.country.unknown')); } else { $output[$i]['countryCode'] = '--'; $output[$i]['countryName'] = $locale->getText('locale.country.unknown'); @@ -390,16 +494,15 @@ App::put('/v1/database/collections/:collectionId') } catch (AuthorizationException $exception) { throw new Exception('Unauthorized permissions', 401); } catch (StructureException $exception) { - throw new Exception('Bad structure. '.$exception->getMessage(), 400); - } + throw new Exception('Bad structure. ' . $exception->getMessage(), 400); + } $usage->setParam('database.collections.update', 1); $audits ->setParam('event', 'database.collections.update') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $collection->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $collection->getArrayCopy()); $response->dynamic($collection, Response::MODEL_COLLECTION); }); @@ -440,14 +543,12 @@ App::delete('/v1/database/collections/:collectionId') $usage->setParam('database.collections.delete', 1); $events - ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)) - ; + ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)); $audits ->setParam('event', 'database.collections.delete') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $collection->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $collection->getArrayCopy()); $response->noContent(); }); @@ -530,7 +631,7 @@ App::post('/v1/database/collections/:collectionId/attributes/email') 'required' => $required, 'default' => $default, 'array' => $array, - 'format' => \json_encode(['name'=>'email']), + 'format' => \json_encode(['name' => 'email']), ]), $response, $dbForExternal, $database, $audits, $usage); }); @@ -571,7 +672,7 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') 'required' => $required, 'default' => $default, 'array' => $array, - 'format' => \json_encode(['name'=>'ip']), + 'format' => \json_encode(['name' => 'ip']), ]), $response, $dbForExternal, $database, $audits, $usage); }); @@ -613,7 +714,7 @@ App::post('/v1/database/collections/:collectionId/attributes/url') 'required' => $required, 'default' => $default, 'array' => $array, - 'format' => \json_encode(['name'=>'url']), + 'format' => \json_encode(['name' => 'url']), ]), $response, $dbForExternal, $database, $audits, $usage); }); @@ -648,7 +749,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Stats\Stats $usage */ - + return $attributesCallback(new Document([ '$collection' => $collectionId, '$id' => $attributeId, @@ -658,7 +759,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') 'default' => $default, 'array' => $array, 'format' => \json_encode([ - 'name'=>'int-range', + 'name' => 'int-range', 'min' => $min, 'max' => $max, ]), @@ -705,7 +806,7 @@ App::post('/v1/database/collections/:collectionId/attributes/float') 'default' => $default, 'array' => $array, 'format' => \json_encode([ - 'name'=>'float-range', + 'name' => 'float-range', 'min' => $min, 'max' => $max, ]), @@ -835,7 +936,7 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId') ])]); $usage->setParam('database.collections.read', 1); - + $response->dynamic($attribute, Response::MODEL_ATTRIBUTE); }); @@ -890,20 +991,17 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') $database ->setParam('type', DATABASE_TYPE_DELETE_ATTRIBUTE) - ->setParam('document', $attribute) - ; + ->setParam('document', $attribute); $usage->setParam('database.collections.update', 1); $events - ->setParam('payload', $response->output($attribute, Response::MODEL_ATTRIBUTE)) - ; + ->setParam('payload', $response->output($attribute, Response::MODEL_ATTRIBUTE)); $audits ->setParam('event', 'database.attributes.delete') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $attribute->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $attribute->getArrayCopy()); $response->noContent(); }); @@ -984,20 +1082,17 @@ App::post('/v1/database/collections/:collectionId/indexes') $database ->setParam('type', DATABASE_TYPE_CREATE_INDEX) - ->setParam('document', $index) - ; + ->setParam('document', $index); $usage->setParam('database.collections.update', 1); $audits ->setParam('event', 'database.indexes.create') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $index->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $index->getArrayCopy()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($index, Response::MODEL_INDEX); - }); App::get('/v1/database/collections/:collectionId/indexes') @@ -1083,7 +1178,7 @@ App::get('/v1/database/collections/:collectionId/indexes/:indexId') ])]); $usage->setParam('database.collections.read', 1); - + $response->dynamic($index, Response::MODEL_INDEX); }); @@ -1124,7 +1219,7 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') $indexes = $collection->getAttribute('indexes'); // find attribute in collection - $index= null; + $index = null; foreach ($indexes as $i) { if ($i->getId() === $indexId) { $index = $i->setAttribute('$collection', $collectionId); // set the collectionId @@ -1138,20 +1233,17 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') $database ->setParam('type', DATABASE_TYPE_DELETE_INDEX) - ->setParam('document', $index) - ; + ->setParam('document', $index); $usage->setParam('database.collections.update', 1); $events - ->setParam('payload', $response->output($index, Response::MODEL_INDEX)) - ; + ->setParam('payload', $response->output($index, Response::MODEL_INDEX)); $audits ->setParam('event', 'database.indexes.delete') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $index->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $index->getArrayCopy()); $response->noContent(); }); @@ -1184,7 +1276,7 @@ App::post('/v1/database/collections/:collectionId/documents') /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Stats\Stats $usage */ - + $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array if (empty($data)) { @@ -1194,7 +1286,7 @@ App::post('/v1/database/collections/:collectionId/documents') if (isset($data['$id'])) { throw new Exception('$id is not allowed for creating new documents, try update instead', 400); } - + $collection = $dbForExternal->getCollection($collectionId); if ($collection->isEmpty()) { @@ -1203,26 +1295,23 @@ App::post('/v1/database/collections/:collectionId/documents') $data['$collection'] = $collection->getId(); // Adding this param to make API easier for developers $data['$id'] = $documentId == 'unique()' ? $dbForExternal->getId() : $documentId; - $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? []; // By default set read permissions for user - $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? []; // By default set write permissions for user + $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user + $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? []; // By default set write permissions for user try { $document = $dbForExternal->createDocument($collectionId, new Document($data)); - } - catch (StructureException $exception) { + } catch (StructureException $exception) { throw new Exception($exception->getMessage(), 400); } $usage ->setParam('database.documents.create', 1) - ->setParam('collectionId', $collectionId) - ; + ->setParam('collectionId', $collectionId); $audits ->setParam('event', 'database.documents.create') - ->setParam('resource', 'database/document/'.$document->getId()) - ->setParam('data', $document->getArrayCopy()) - ; + ->setParam('resource', 'database/document/' . $document->getId()) + ->setParam('data', $document->getArrayCopy()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($document, Response::MODEL_DOCUMENT); @@ -1286,9 +1375,8 @@ App::get('/v1/database/collections/:collectionId/documents') $usage ->setParam('database.documents.read', 1) - ->setParam('collectionId', $collectionId) - ; - + ->setParam('collectionId', $collectionId); + $response->dynamic(new Document([ 'sum' => $dbForExternal->count($collectionId, $queries, APP_LIMIT_COUNT), 'documents' => $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null), @@ -1329,8 +1417,7 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.read', 1) - ->setParam('collectionId', $collectionId) - ; + ->setParam('collectionId', $collectionId); $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -1379,7 +1466,7 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') if (empty($data)) { throw new Exception('Missing payload', 400); } - + if (!\is_array($data)) { throw new Exception('Data param should be a valid JSON object', 400); } @@ -1393,24 +1480,20 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') try { $document = $dbForExternal->updateDocument($collection->getId(), $document->getId(), new Document($data)); - } - catch (AuthorizationException $exception) { + } catch (AuthorizationException $exception) { throw new Exception('Unauthorized permissions', 401); + } catch (StructureException $exception) { + throw new Exception('Bad structure. ' . $exception->getMessage(), 400); } - catch (StructureException $exception) { - throw new Exception('Bad structure. '.$exception->getMessage(), 400); - } - + $usage ->setParam('database.documents.update', 1) - ->setParam('collectionId', $collectionId) - ; + ->setParam('collectionId', $collectionId); $audits ->setParam('event', 'database.documents.update') - ->setParam('resource', 'database/document/'.$document->getId()) - ->setParam('data', $document->getArrayCopy()) - ; + ->setParam('resource', 'database/document/' . $document->getId()) + ->setParam('data', $document->getArrayCopy()); $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -1456,18 +1539,28 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.delete', 1) - ->setParam('collectionId', $collectionId) - ; + ->setParam('collectionId', $collectionId); $events - ->setParam('eventData', $response->output($document, Response::MODEL_DOCUMENT)) - ; + ->setParam('eventData', $response->output($document, Response::MODEL_DOCUMENT)); $audits ->setParam('event', 'database.documents.delete') - ->setParam('resource', 'database/document/'.$document->getId()) + ->setParam('resource', 'database/document/' . $document->getId()) ->setParam('data', $document->getArrayCopy()) // Audit document in case of malicious or disastrous action ; $response->noContent(); }); + + +// Refactor usage endpoint in Functions API + + +// Create usage endpoint in the Database API + + +// Create usage endpoint in the users API + + +// Create usage endpoint in the Storage API \ No newline at end of file diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index fbd27c47f3..0f7a4f9a99 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -259,6 +259,101 @@ App::get('/v1/functions/:functionId/usage') } }); +App::get('/v1/functions/:functionId/usage2') + ->desc('Get Function Usage') + ->groups(['api', 'functions']) + ->label('scope', 'functions.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'functions') + ->label('sdk.method', 'getUsage') + ->param('functionId', '', new UID(), 'Function unique ID.') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true) + ->inject('response') + ->inject('project') + ->inject('dbForInternal') + ->inject('register') + ->action(function ($functionId, $range, $response, $project, $dbForInternal, $register) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Document $project */ + /** @var Utopia\Database\Database $dbForInternal */ + /** @var Utopia\Registry\Registry $register */ + + $function = $dbForInternal->getDocument('functions', $functionId); + + if ($function->isEmpty()) { + throw new Exception('Function not found', 404); + } + + if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + + $metrics = ["functions.$functionId.executions", "functions.$functionId.failures", "functions.$functionId.compute"]; + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + } + + $executions = $stats["functions.$functionId.executions"] ?? []; + $failures = $stats["functions.$functionId.failures"] ?? []; + $compute = $stats["functions.$functionId.compute"] ?? []; + + $response->json([ + 'range' => $range, + 'executions' => [ + 'data' => $executions, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $executions)), + ], + 'failures' => [ + 'data' => $failures, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $failures)), + ], + 'compute' => [ + 'data' => $compute, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $compute)), + ], + ]); + } else { + $response->json([]); + } + }); + App::put('/v1/functions/:functionId') ->groups(['api', 'functions']) ->desc('Update Function') diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 7d8f4ed54c..ba9d5627b5 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -217,7 +217,7 @@ App::get('/v1/projects/:projectId') }); App::get('/v1/projects/:projectId/usage') - ->desc('Get Project') + ->desc('Get Project Usage') ->groups(['api', 'projects']) ->label('scope', 'projects.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])