From c9b6b3bf8b965891957f82d485bfdb6659bef7bf Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 7 Dec 2022 12:48:18 +0200 Subject: [PATCH] storage api usage --- app/controllers/api/storage.php | 314 +++++++++++++++----------------- 1 file changed, 142 insertions(+), 172 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 45bb8148a8..1795c470c7 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1426,101 +1426,80 @@ App::get('/v1/storage/usage') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForProject') - ->action(function (string $range, Response $response, Database $dbForProject) { + ->inject('project') + ->action(function (string $range, Response $response, Database $dbForProject, Document $project) { - // $usage = []; - // if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { - // $periods = [ - // '24h' => [ - // 'period' => '1h', - // 'limit' => 24, - // ], - // '7d' => [ - // 'period' => '1d', - // 'limit' => 7, - // ], - // '30d' => [ - // 'period' => '1d', - // 'limit' => 30, - // ], - // '90d' => [ - // 'period' => '1d', - // 'limit' => 90, - // ], - // ]; + $periods = [ + '24h' => [ + 'period' => '1h', + 'limit' => 24, + 'factor' => 3600, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + 'factor' => 86400, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + 'factor' => 86400, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + 'factor' => 86400, + ], + ]; - // $metrics = [ - // 'project.$all.storage.size', - // 'buckets.$all.count.total', - // 'buckets.$all.requests.create', - // 'buckets.$all.requests.read', - // 'buckets.$all.requests.update', - // 'buckets.$all.requests.delete', - // 'files.$all.storage.size', - // 'files.$all.count.total', - // 'files.$all.requests.create', - // 'files.$all.requests.read', - // 'files.$all.requests.update', - // 'files.$all.requests.delete', - // ]; + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + $project->getId() . '.buckets', + ]; - // $stats = []; + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric] = []; + foreach ($results as $result) { + $stats[$metric][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; + } + } + }); - // Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - // foreach ($metrics as $metric) { - // $limit = $periods[$range]['limit']; - // $period = $periods[$range]['period']; + $format = match ($days['period']) { + '1h' => 'Y-m-d\TH:00:00.000P', + '1d' => 'Y-m-d\T00:00:00.000P', + }; - // $requestDocs = $dbForProject->find('stats', [ - // Query::equal('period', [$period]), - // Query::equal('metric', [$metric]), - // Query::limit($limit), - // Query::orderDesc('time'), - // ]); - - // $stats[$metric] = []; - // foreach ($requestDocs as $requestDoc) { - // $stats[$metric][] = [ - // 'value' => $requestDoc->getAttribute('value'), - // 'date' => $requestDoc->getAttribute('time'), - // ]; - // } - - // // backfill metrics with empty values for graphs - // $backfill = $limit - \count($requestDocs); - // while ($backfill > 0) { - // $last = $limit - $backfill - 1; // array index of last added metric - // $diff = match ($period) { // convert period to seconds for unix timestamp math - // '1h' => 3600, - // '1d' => 86400, - // }; - // $stats[$metric][] = [ - // 'value' => 0, - // 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), - // ]; - // $backfill--; - // } - // $stats[$metric] = array_reverse($stats[$metric]); - // } - // }); - - // $usage = new Document([ - // 'range' => $range, - // 'bucketsCount' => $stats['buckets.$all.count.total'], - // 'bucketsCreate' => $stats['buckets.$all.requests.create'], - // 'bucketsRead' => $stats['buckets.$all.requests.read'], - // 'bucketsUpdate' => $stats['buckets.$all.requests.update'], - // 'bucketsDelete' => $stats['buckets.$all.requests.delete'], - // 'storage' => $stats['project.$all.storage.size'], - // 'filesCount' => $stats['files.$all.count.total'], - // 'filesCreate' => $stats['files.$all.requests.create'], - // 'filesRead' => $stats['files.$all.requests.read'], - // 'filesUpdate' => $stats['files.$all.requests.update'], - // 'filesDelete' => $stats['files.$all.requests.delete'], - // ]); - // } - - // $response->dynamic($usage, Response::MODEL_USAGE_STORAGE); + foreach ($metrics as $metric) { + $usage[$metric] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric][] = [ + 'value' => $stats[$metric][$formatDate] ?? 0, + 'date' => $formatDate, + ]; + } + $usage[$metric] = array_reverse($usage[$metric]); + } + var_dump($usage); + $response->noContent(); +// $response->dynamic(new Document([ +// $usage +// ]), Response::MODEL_USAGE_STORAGE); }); App::get('/v1/storage/:bucketId/usage') @@ -1537,94 +1516,85 @@ App::get('/v1/storage/:bucketId/usage') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForProject') - ->action(function (string $bucketId, string $range, Response $response, Database $dbForProject) { + ->inject('project') + ->action(function (string $bucketId, string $range, Response $response, Database $dbForProject, Document $project) { - // $bucket = $dbForProject->getDocument('buckets', $bucketId); + $bucket = $dbForProject->getDocument('buckets', $bucketId); - // if ($bucket->isEmpty()) { - // throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); - // } + if ($bucket->isEmpty()) { + throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); + } - // $usage = []; - // if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { - // $periods = [ - // '24h' => [ - // 'period' => '1h', - // 'limit' => 24, - // ], - // '7d' => [ - // 'period' => '1d', - // 'limit' => 7, - // ], - // '30d' => [ - // 'period' => '1d', - // 'limit' => 30, - // ], - // '90d' => [ - // 'period' => '1d', - // 'limit' => 90, - // ], - // ]; + $periods = [ + '24h' => [ + 'period' => '1h', + 'limit' => 24, + 'factor' => 3600, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + 'factor' => 86400, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + 'factor' => 86400, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + 'factor' => 86400, + ], + ]; - // $metrics = [ - // "files.{$bucketId}.count.total", - // "files.{$bucketId}.storage.size", - // "files.{$bucketId}.requests.create", - // "files.{$bucketId}.requests.read", - // "files.{$bucketId}.requests.update", - // "files.{$bucketId}.requests.delete", - // ]; + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + $project->getId() . '.' . $bucket->getId() . '.files', + $project->getId() . '.' . $bucket->getId() . '.files.storage', + ]; - // $stats = []; + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric] = []; + foreach ($results as $result) { + $stats[$metric][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; + } + } + }); - // Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - // foreach ($metrics as $metric) { - // $limit = $periods[$range]['limit']; - // $period = $periods[$range]['period']; + $format = match ($days['period']) { + '1h' => 'Y-m-d\TH:00:00.000P', + '1d' => 'Y-m-d\T00:00:00.000P', + }; - // $requestDocs = $dbForProject->find('stats', [ - // Query::equal('period', [$period]), - // Query::equal('metric', [$metric]), - // Query::limit($limit), - // Query::orderDesc('time'), - // ]); + foreach ($metrics as $metric) { + $usage[$metric] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric][] = [ + 'value' => $stats[$metric][$formatDate] ?? 0, + 'date' => $formatDate, + ]; + } + $usage[$metric] = array_reverse($usage[$metric]); + } - // $stats[$metric] = []; - // foreach ($requestDocs as $requestDoc) { - // $stats[$metric][] = [ - // 'value' => $requestDoc->getAttribute('value'), - // 'date' => $requestDoc->getAttribute('time'), - // ]; - // } - - // // backfill metrics with empty values for graphs - // $backfill = $limit - \count($requestDocs); - // while ($backfill > 0) { - // $last = $limit - $backfill - 1; // array index of last added metric - // $diff = match ($period) { // convert period to seconds for unix timestamp math - // '1h' => 3600, - // '1d' => 86400, - // }; - // $stats[$metric][] = [ - // 'value' => 0, - // 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), - // ]; - // $backfill--; - // } - // $stats[$metric] = array_reverse($stats[$metric]); - // } - // }); - - // $usage = new Document([ - // 'range' => $range, - // 'filesCount' => $stats[$metrics[0]], - // 'filesStorage' => $stats[$metrics[1]], - // 'filesCreate' => $stats[$metrics[2]], - // 'filesRead' => $stats[$metrics[3]], - // 'filesUpdate' => $stats[$metrics[4]], - // 'filesDelete' => $stats[$metrics[5]], - // ]); - // } - - // $response->dynamic($usage, Response::MODEL_USAGE_BUCKETS); + var_dump($usage); + $response->dynamic(new Document([ + $usage + ]), Response::MODEL_USAGE_BUCKETS); });