From e87435dd617abdab00ac373685e1094a4a5391d4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 10 Aug 2021 14:29:31 +0545 Subject: [PATCH 001/120] wip usage tasks --- app/cli.php | 1 + app/tasks/usage.php | 74 +++++++++++++++++++++++++++++++++++++++++++++ bin/usage | 3 ++ 3 files changed, 78 insertions(+) create mode 100644 app/tasks/usage.php create mode 100755 bin/usage diff --git a/app/cli.php b/app/cli.php index ee6fe1a801..b48c853a24 100644 --- a/app/cli.php +++ b/app/cli.php @@ -15,6 +15,7 @@ include 'tasks/migrate.php'; include 'tasks/sdks.php'; include 'tasks/ssl.php'; include 'tasks/vars.php'; +include 'tasks/usage.php'; $cli ->task('version') diff --git a/app/tasks/usage.php b/app/tasks/usage.php new file mode 100644 index 0000000000..769d241042 --- /dev/null +++ b/app/tasks/usage.php @@ -0,0 +1,74 @@ +task('usage') + ->desc('Schedules syncing data from influxdb to Appwrite console db') + ->action(function () use ($register, $dbForConsole) { + Console::title('Usage Sync V1'); + Console::success(APP_NAME . ' usage sync process v1 has started'); + + $interval = (int) App::getEnv('_APP_USAGE_SYNC_INTERVAL', '10'); //10 seconds + + // Console::loop(function () use ($interval, $register, $dbForConsole) { + $time = date('d-m-Y H:i:s', time()); + Console::info("[{$time}] Syncing usage data from influxdb to Appwrite Console DB every {$interval} seconds"); + + $client = $register->get('influxdb'); + + $client = $register->get('influxdb'); + + $requests = []; + $network = []; + $functions = []; + + if ($client) { + // $start30 = DateTime::createFromFormat('U', \strtotime('-30 minutes'))->format(DateTime::RFC3339); + $start = DateTime::createFromFormat('U', \strtotime('-30 days'))->format(DateTime::RFC3339); + $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); + $database = $client->selectDB('telegraf'); + + // Requests + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . '1d' . ') FILL(null)'); + $points = $result->getPoints(); + + foreach ($points as $point) { + $requests[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + ]; + } + + // Network + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . '1d' . ') FILL(null)'); + $points = $result->getPoints(); + + foreach ($points as $point) { + $network[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + ]; + } + + // Functions + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . '1d' . ') FILL(null)'); + $points = $result->getPoints(); + + foreach ($points as $point) { + $functions[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + ]; + } + + var_dump($requests, $network, $functions); + } + + // }, $interval); + }); diff --git a/bin/usage b/bin/usage new file mode 100755 index 0000000000..2709200ae4 --- /dev/null +++ b/bin/usage @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php usage $@ \ No newline at end of file From 0f69216d66d247a33bc82f639f8fac979de0de9e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 11 Aug 2021 10:59:27 +0545 Subject: [PATCH 002/120] collection for usage stats --- app/config/collections2.php | 78 +++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/app/config/collections2.php b/app/config/collections2.php index ae9a4feeb1..ce9f4374ec 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -1403,6 +1403,84 @@ $collections = [ ], ], ], + 'stats' => [ + '$collection' => Database::COLLECTIONS, + '$id' => 'stats', + 'name' => 'Stats', + 'attributes' => [ + [ + '$id' => 'metric', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'value', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'time', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'period', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 4, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'type', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 1, + 'signed' => false, + 'required' => true, + 'default' => 0, // 0 -> count, 1 -> sum + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_time', + 'type' => Database::INDEX_KEY, + 'attributes' => ['time'], + 'lengths' => [100], + 'orders' => [Database::ORDER_DESC], + ], + [ + '$id' => '_key_time_period', + 'type' => Database::INDEX_KEY, + 'attributes' => ['time', 'period'], + 'lengths' => [100, 4], + 'orders' => [Database::ORDER_DESC], + ], + ], + ] ]; return $collections; From d77c36dcca35703c039f1fd27cbac5eb9b077ea3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 11 Aug 2021 11:29:08 +0545 Subject: [PATCH 003/120] usage container definition --- Dockerfile | 1 + docker-compose.yml | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/Dockerfile b/Dockerfile index 68b53747be..dd0dde11d2 100755 --- a/Dockerfile +++ b/Dockerfile @@ -225,6 +225,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/maintenance && \ + chmod +x /usr/local/bin/usage && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/schedule && \ diff --git a/docker-compose.yml b/docker-compose.yml index 42c909ced5..a1880dd434 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -344,6 +344,30 @@ services: - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + appwrite-usage: + entrypoint: usage + container_name: appwrite-usage + build: + context: . + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - influxdb + - mariadb + environment: + - _APP_ENV + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_SYNC_INTERVAL + appwrite-schedule: entrypoint: schedule container_name: appwrite-schedule From a29847e7fa8e89de68b1a13360af7ffcb7f14fa5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 11 Aug 2021 14:12:37 +0545 Subject: [PATCH 004/120] debugger support for appwrite-usage container --- docker-compose.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a1880dd434..707057d324 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -345,15 +345,22 @@ services: - _APP_MAINTENANCE_RETENTION_AUDIT appwrite-usage: - entrypoint: usage + entrypoint: + - php + - -e + - /usr/src/code/app/cli.php + - usage container_name: appwrite-usage build: context: . + args: + - DEBUG=false networks: - appwrite volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - ./dev:/usr/local/dev depends_on: - influxdb - mariadb From 949abf876b8df51078a1d8ced29ed46b9032e8e6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 11 Aug 2021 16:00:00 +0545 Subject: [PATCH 005/120] basic setup for syncing usage stats to appwrite db --- app/config/collections2.php | 4 +- app/tasks/usage.php | 206 ++++++++++++++++++++++++++---------- app/workers.php | 13 +++ 3 files changed, 166 insertions(+), 57 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index ce9f4374ec..2b2419204e 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -1469,14 +1469,14 @@ $collections = [ '$id' => '_key_time', 'type' => Database::INDEX_KEY, 'attributes' => ['time'], - 'lengths' => [100], + 'lengths' => [], 'orders' => [Database::ORDER_DESC], ], [ '$id' => '_key_time_period', 'type' => Database::INDEX_KEY, 'attributes' => ['time', 'period'], - 'lengths' => [100, 4], + 'lengths' => [], 'orders' => [Database::ORDER_DESC], ], ], diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 769d241042..857c71de40 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -1,74 +1,170 @@ task('usage') ->desc('Schedules syncing data from influxdb to Appwrite console db') - ->action(function () use ($register, $dbForConsole) { + ->action(function () use ($register) { Console::title('Usage Sync V1'); Console::success(APP_NAME . ' usage sync process v1 has started'); - $interval = (int) App::getEnv('_APP_USAGE_SYNC_INTERVAL', '10'); //10 seconds + $interval = (int) App::getEnv('_APP_USAGE_SYNC_INTERVAL', '30'); //30 seconds - // Console::loop(function () use ($interval, $register, $dbForConsole) { - $time = date('d-m-Y H:i:s', time()); - Console::info("[{$time}] Syncing usage data from influxdb to Appwrite Console DB every {$interval} seconds"); + $cacheAdapter = new Cache(new Redis($register->get('cache'))); + $dbForConsole = new Database(new MariaDB($register->get('db')), $cacheAdapter); + $dbForConsole->setNamespace('project_console_internal'); - $client = $register->get('influxdb'); + Authorization::disable(); + $projects = $dbForConsole->find('projects'); + $projectIds = []; + foreach ($projects as $project) { + $projectIds[] = $project['$id']; + } + $projects = null; - $client = $register->get('influxdb'); - - $requests = []; - $network = []; - $functions = []; - - if ($client) { - // $start30 = DateTime::createFromFormat('U', \strtotime('-30 minutes'))->format(DateTime::RFC3339); - $start = DateTime::createFromFormat('U', \strtotime('-30 days'))->format(DateTime::RFC3339); - $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); - - // Requests - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . '1d' . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $requests[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Network - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . '1d' . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $network[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Functions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . '1d' . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $functions[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - var_dump($requests, $network, $functions); + $latestData = []; + $dbForProject = new Database(new MariaDB($register->get('db')), $cacheAdapter); + foreach ($projectIds as $id) { + $dbForProject->setNamespace("project_{$id}_internal"); + $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"])], 1, 0, ['time'], [Database::ORDER_DESC]); + $doc = reset($doc); + $latestData[$id]["1d"] = $doc ? $doc->getAttribute('time') : null; + $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 1, 0, ['time'], [Database::ORDER_DESC]); + $doc = reset($doc); + $latestData[$id]["30m"] = $doc ? $doc->getAttribute('time') : null; } - // }, $interval); + $firstRun = true; + Console::loop(function () use ($interval, $register, $projectIds, $latestData, $dbForProject, &$firstRun) { + $time = date('d-m-Y H:i:s', time()); + Console::info("[{$time}] Syncing usage data from influxdb to Appwrite Console DB every {$interval} seconds"); + + $client = $register->get('influxdb'); + + $requests = []; + $network = []; + $functions = []; + + if ($client) { + foreach ($projectIds as $id) { + if ($firstRun) { + $start = DateTime::createFromFormat('U', \strtotime('-1 days'))->format(DateTime::RFC3339); + if (!empty($latestData[$id]["30m"])) { + $start = DateTime::createFromFormat('U', $latestData[$id]['30m'])->format(DateTime::RFC3339); + } + } else { + $start = DateTime::createFromFormat('U', \strtotime("-{$interval} seconds"))->format(DateTime::RFC3339); + } + $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); + $database = $client->selectDB('telegraf'); + + // Requests + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\' GROUP BY time(' . '30m' . ') FILL(null)'); + $points = $result->getPoints(); + + $dbForProject->setNamespace("project_{$id}_internal"); + foreach ($points as $point) { + $requests[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + 'dateStr' => $point['time'], + 'point' => $point, + ]; + $time = \strtotime($point['time']); + $id = \md5($time . '_' . '30m' . '_requests'); + $value = (!empty($point['value'])) ? $point['value'] : 0; + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '30m', + 'time' => $time, + 'metric' => 'requests', + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $firstRun ? $value : $document->getAttribute('value') + $value)); + } + + } + + // Network + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\'GROUP BY time(' . '30m' . ') FILL(null)'); + $points = $result->getPoints(); + + foreach ($points as $point) { + $network[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + 'dateStr' => $point['time'], + 'point' => $point, + ]; + $time = \strtotime($point['time']); + $id = \md5($time . '_' . '30m' . '_network'); + $value = (!empty($point['value'])) ? $point['value'] : 0; + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '30m', + 'time' => $time, + 'metric' => 'network', + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $firstRun ? $value : $document->getAttribute('value') + $value)); + } + } + + // Functions + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\'GROUP BY time(' . '30m' . ') FILL(null)'); + $points = $result->getPoints(); + + foreach ($points as $point) { + $functions[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + 'dateStr' => $point['time'], + 'point' => $point, + ]; + $time = \strtotime($point['time']); + $id = \md5($time . '_' . '30m' . '_functions'); + $value = (!empty($point['value'])) ? $point['value'] : 0; + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '30m', + 'time' => $time, + 'metric' => 'functions', + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $firstRun ? $value : $document->getAttribute('value') + $value)); + } + } + + } + } + $firstRun = false; + }, $interval); }); diff --git a/app/workers.php b/app/workers.php index 7dd48eaff8..56dc862fc6 100644 --- a/app/workers.php +++ b/app/workers.php @@ -32,3 +32,16 @@ $register->set('cache', function () { // Register cache connection return $redis; }); +$register->set('influxdb', function () { // Register DB connection + $host = App::getEnv('_APP_INFLUXDB_HOST', ''); + $port = App::getEnv('_APP_INFLUXDB_PORT', ''); + + if (empty($host) || empty($port)) { + return; + } + $driver = new InfluxDB\Driver\Curl("http://{$host}:{$port}"); + $client = new InfluxDB\Client($host, $port, '', '', false, false, 5); + $client->setDriver($driver); + + return $client; +}); \ No newline at end of file From dfde2b970f5a6defa0a60af8abe0b22a6e51c6aa Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 12 Aug 2021 11:38:02 +0545 Subject: [PATCH 006/120] refactor to sync usage for 1d and 30m periods --- app/tasks/usage.php | 247 ++++++++++++++++++++++++-------------------- 1 file changed, 134 insertions(+), 113 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 857c71de40..6154f5c969 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -5,7 +5,7 @@ global $cli, $register; require_once __DIR__ . '/../workers.php'; use Utopia\App; -use Utopia\Cache\Adapter\Redis; +use Utopia\Cache\Adapter\None; use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; @@ -23,7 +23,7 @@ $cli $interval = (int) App::getEnv('_APP_USAGE_SYNC_INTERVAL', '30'); //30 seconds - $cacheAdapter = new Cache(new Redis($register->get('cache'))); + $cacheAdapter = new Cache(new None()); $dbForConsole = new Database(new MariaDB($register->get('db')), $cacheAdapter); $dbForConsole->setNamespace('project_console_internal'); @@ -31,13 +31,13 @@ $cli $projects = $dbForConsole->find('projects'); $projectIds = []; foreach ($projects as $project) { - $projectIds[] = $project['$id']; + $projectIds[$project['$id']] = true; } $projects = null; $latestData = []; $dbForProject = new Database(new MariaDB($register->get('db')), $cacheAdapter); - foreach ($projectIds as $id) { + foreach ($projectIds as $id => $value) { $dbForProject->setNamespace("project_{$id}_internal"); $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"])], 1, 0, ['time'], [Database::ORDER_DESC]); $doc = reset($doc); @@ -48,123 +48,144 @@ $cli } $firstRun = true; - Console::loop(function () use ($interval, $register, $projectIds, $latestData, $dbForProject, &$firstRun) { + Console::loop(function () use ($interval, $register, &$projectIds, &$latestData, $dbForProject, $dbForConsole, &$firstRun) { $time = date('d-m-Y H:i:s', time()); Console::info("[{$time}] Syncing usage data from influxdb to Appwrite Console DB every {$interval} seconds"); $client = $register->get('influxdb'); - - $requests = []; - $network = []; - $functions = []; + //fetch delta projects + if (!$firstRun) { + $projects = $dbForConsole->find('projects'); + foreach ($projects as $project) { + $id = $project['$id']; + if (!$projectIds[$id]) { + $projectIds[$id] = true; + if ($latestData[$id] == null) { + $dbForProject->setNamespace("project_{$id}_internal"); + $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"])], 1, 0, ['time'], [Database::ORDER_DESC]); + $doc = reset($doc); + $latestData[$id]["1d"] = $doc ? $doc->getAttribute('time') : null; + $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 1, 0, ['time'], [Database::ORDER_DESC]); + $doc = reset($doc); + $latestData[$id]["30m"] = $doc ? $doc->getAttribute('time') : null; + } + } + } + } if ($client) { - foreach ($projectIds as $id) { - if ($firstRun) { - $start = DateTime::createFromFormat('U', \strtotime('-1 days'))->format(DateTime::RFC3339); - if (!empty($latestData[$id]["30m"])) { - $start = DateTime::createFromFormat('U', $latestData[$id]['30m'])->format(DateTime::RFC3339); - } - } else { - $start = DateTime::createFromFormat('U', \strtotime("-{$interval} seconds"))->format(DateTime::RFC3339); - } - $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); - - // Requests - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\' GROUP BY time(' . '30m' . ') FILL(null)'); - $points = $result->getPoints(); - - $dbForProject->setNamespace("project_{$id}_internal"); - foreach ($points as $point) { - $requests[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - 'dateStr' => $point['time'], - 'point' => $point, - ]; - $time = \strtotime($point['time']); - $id = \md5($time . '_' . '30m' . '_requests'); - $value = (!empty($point['value'])) ? $point['value'] : 0; - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => '30m', - 'time' => $time, - 'metric' => 'requests', - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $firstRun ? $value : $document->getAttribute('value') + $value)); - } - - } - - // Network - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\'GROUP BY time(' . '30m' . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $network[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - 'dateStr' => $point['time'], - 'point' => $point, - ]; - $time = \strtotime($point['time']); - $id = \md5($time . '_' . '30m' . '_network'); - $value = (!empty($point['value'])) ? $point['value'] : 0; - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => '30m', - 'time' => $time, - 'metric' => 'network', - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $firstRun ? $value : $document->getAttribute('value') + $value)); - } - } - - // Functions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\'GROUP BY time(' . '30m' . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $functions[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - 'dateStr' => $point['time'], - 'point' => $point, - ]; - $time = \strtotime($point['time']); - $id = \md5($time . '_' . '30m' . '_functions'); - $value = (!empty($point['value'])) ? $point['value'] : 0; - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => '30m', - 'time' => $time, - 'metric' => 'functions', - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $firstRun ? $value : $document->getAttribute('value') + $value)); - } - } - + foreach ($projectIds as $id => $value) { + syncData($client, $id, '30m', $latestData, $dbForProject); + syncData($client, $id, '1d', $latestData, $dbForProject); } } $firstRun = false; }, $interval); }); + +function syncData($client, $projectId, $period, &$latestData, $dbForProject) +{ + $requests = []; + $network = []; + $functions = []; + $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-90 days' : '-1 days'))->format(DateTime::RFC3339); + if (!empty($latestData[$projectId][$period])) { + $start = DateTime::createFromFormat('U', $latestData[$projectId][$period])->format(DateTime::RFC3339); + } + $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); + $database = $client->selectDB('telegraf'); + + // Requests + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\' GROUP BY time(' . $period . ') FILL(null)'); + $points = $result->getPoints(); + + $dbForProject->setNamespace("project_{$projectId}_internal"); + foreach ($points as $point) { + $requests[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + 'dateStr' => $point['time'], + 'point' => $point, + ]; + $time = \strtotime($point['time']); + $id = \md5($time . '_' . $period . '_requests'); + $value = (!empty($point['value'])) ? $point['value'] : 0; + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => 'requests', + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $value)); + } + $latestData[$id]["30m"] = $time; + } + + // Network + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . ') FILL(null)'); + $points = $result->getPoints(); + + foreach ($points as $point) { + $network[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + 'dateStr' => $point['time'], + 'point' => $point, + ]; + $time = \strtotime($point['time']); + $id = \md5($time . '_' . $period . '_network'); + $value = (!empty($point['value'])) ? $point['value'] : 0; + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => 'network', + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $value)); + } + $latestData[$id]["30m"] = $time; + } + + // Functions + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\'GROUP BY time(' . $period . ') FILL(null)'); + $points = $result->getPoints(); + + foreach ($points as $point) { + $functions[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + 'dateStr' => $point['time'], + 'point' => $point, + ]; + $time = \strtotime($point['time']); + $id = \md5($time . '_' . $period . '_functions'); + $value = (!empty($point['value'])) ? $point['value'] : 0; + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => 'functions', + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $value)); + } + $latestData[$id]["30m"] = $time; + } +} From 9ce9c32335402ad56fae01d47f33ff9ed5487d00 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 12 Aug 2021 16:56:33 +0545 Subject: [PATCH 007/120] refactor --- app/tasks/usage.php | 64 +++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 6154f5c969..4d85535097 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -26,51 +26,32 @@ $cli $cacheAdapter = new Cache(new None()); $dbForConsole = new Database(new MariaDB($register->get('db')), $cacheAdapter); $dbForConsole->setNamespace('project_console_internal'); + $dbForProject = new Database(new MariaDB($register->get('db')), $cacheAdapter); Authorization::disable(); - $projects = $dbForConsole->find('projects'); $projectIds = []; - foreach ($projects as $project) { - $projectIds[$project['$id']] = true; - } + $latestProject = null; + $latestData = []; + do { + $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + if (!empty($projects)) { + $latestProject = $projects[array_key_last($projects)]; + $latestData = getLatestData($projects, $latestData, $dbForProject); + } + } while (!empty($projects)); + $projects = null; - $latestData = []; - $dbForProject = new Database(new MariaDB($register->get('db')), $cacheAdapter); - foreach ($projectIds as $id => $value) { - $dbForProject->setNamespace("project_{$id}_internal"); - $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"])], 1, 0, ['time'], [Database::ORDER_DESC]); - $doc = reset($doc); - $latestData[$id]["1d"] = $doc ? $doc->getAttribute('time') : null; - $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 1, 0, ['time'], [Database::ORDER_DESC]); - $doc = reset($doc); - $latestData[$id]["30m"] = $doc ? $doc->getAttribute('time') : null; - } - $firstRun = true; - Console::loop(function () use ($interval, $register, &$projectIds, &$latestData, $dbForProject, $dbForConsole, &$firstRun) { + Console::loop(function () use ($interval, $register, &$projectIds, &$latestData, $dbForProject, $dbForConsole, &$firstRun, &$latestProject) { $time = date('d-m-Y H:i:s', time()); Console::info("[{$time}] Syncing usage data from influxdb to Appwrite Console DB every {$interval} seconds"); $client = $register->get('influxdb'); - //fetch delta projects if (!$firstRun) { - $projects = $dbForConsole->find('projects'); - foreach ($projects as $project) { - $id = $project['$id']; - if (!$projectIds[$id]) { - $projectIds[$id] = true; - if ($latestData[$id] == null) { - $dbForProject->setNamespace("project_{$id}_internal"); - $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"])], 1, 0, ['time'], [Database::ORDER_DESC]); - $doc = reset($doc); - $latestData[$id]["1d"] = $doc ? $doc->getAttribute('time') : null; - $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 1, 0, ['time'], [Database::ORDER_DESC]); - $doc = reset($doc); - $latestData[$id]["30m"] = $doc ? $doc->getAttribute('time') : null; - } - } - } + $projects = $dbForConsole->find('projects', limit:100, orderAfter:$latestProject); + $latestProject = $projects[array_key_last($projects)]; + $latestData = getLatestData($projects, $latestData, $dbForProject); } if ($client) { @@ -83,6 +64,21 @@ $cli }, $interval); }); +function getLatestData(&$projects, &$latestData, $dbForProject) +{ + foreach ($projects as $project) { + $id = $project->getId(); + $projectIds[$id] = true; + $dbForProject->setNamespace("project_{$id}_internal"); + $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"])], 0, ['time'], [Database::ORDER_DESC]); + $latestData[$id]["1d"] = $doc ? $doc->getAttribute('time') : null; + $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 0, ['time'], [Database::ORDER_DESC]); + $latestData[$id]["30m"] = $doc ? $doc->getAttribute('time') : null; + } + $projects = null; + return $latestData; +} + function syncData($client, $projectId, $period, &$latestData, $dbForProject) { $requests = []; From 56e3a2bd6cb95fc8d984d4a6e803f07d6d7b5eeb Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 13 Aug 2021 14:24:46 +0545 Subject: [PATCH 008/120] depricate worker.php, use init.php instead --- app/cli.php | 2 +- app/init.php | 25 ++++++++++++++++++++++++- app/tasks/usage.php | 14 ++++++++------ app/workers/audits.php | 2 +- app/workers/certificates.php | 2 +- app/workers/database.php | 2 +- app/workers/deletes.php | 2 +- app/workers/functions.php | 2 +- app/workers/mails.php | 2 +- app/workers/webhooks.php | 2 +- 10 files changed, 40 insertions(+), 15 deletions(-) diff --git a/app/cli.php b/app/cli.php index b48c853a24..99a77d6382 100644 --- a/app/cli.php +++ b/app/cli.php @@ -1,6 +1,6 @@ set('statsd', function () { // Register DB connection return $statsd; }); - $register->set('smtp', function () { $mail = new PHPMailer(true); @@ -324,6 +324,29 @@ $register->set('smtp', function () { $register->set('geodb', function () { return new Reader(__DIR__.'/db/DBIP/dbip-country-lite-2021-06.mmdb'); }); +$register->set('db', function () { // This is usually for our workers or CLI commands scope + $dbHost = App::getEnv('_APP_DB_HOST', ''); + $dbUser = App::getEnv('_APP_DB_USER', ''); + $dbPass = App::getEnv('_APP_DB_PASS', ''); + $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); + + $pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( + PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4', + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + )); + + return $pdo; +}); +$register->set('cache', function () { // This is usually for our workers or CLI commands scope + $redis = new Redis(); + $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + + return $redis; +}); /* * Localization diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 4d85535097..ae7de85e82 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -2,10 +2,10 @@ global $cli, $register; -require_once __DIR__ . '/../workers.php'; +require_once __DIR__ . '/../init.php'; use Utopia\App; -use Utopia\Cache\Adapter\None; +use Utopia\Cache\Adapter\Redis; use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; @@ -23,7 +23,7 @@ $cli $interval = (int) App::getEnv('_APP_USAGE_SYNC_INTERVAL', '30'); //30 seconds - $cacheAdapter = new Cache(new None()); + $cacheAdapter = new Cache(new Redis($register->get('cache'))); $dbForConsole = new Database(new MariaDB($register->get('db')), $cacheAdapter); $dbForConsole->setNamespace('project_console_internal'); $dbForProject = new Database(new MariaDB($register->get('db')), $cacheAdapter); @@ -47,13 +47,15 @@ $cli $time = date('d-m-Y H:i:s', time()); Console::info("[{$time}] Syncing usage data from influxdb to Appwrite Console DB every {$interval} seconds"); - $client = $register->get('influxdb'); if (!$firstRun) { $projects = $dbForConsole->find('projects', limit:100, orderAfter:$latestProject); - $latestProject = $projects[array_key_last($projects)]; - $latestData = getLatestData($projects, $latestData, $dbForProject); + if (!empty($projects)) { + $latestProject = $projects[array_key_last($projects)]; + $latestData = getLatestData($projects, $latestData, $dbForProject); + } } + $client = $register->get('influxdb'); if ($client) { foreach ($projectIds as $id => $value) { syncData($client, $id, '30m', $latestData, $dbForProject); diff --git a/app/workers/audits.php b/app/workers/audits.php index 663de3ce4f..b3e6759627 100644 --- a/app/workers/audits.php +++ b/app/workers/audits.php @@ -4,7 +4,7 @@ use Appwrite\Resque\Worker; use Utopia\Audit\Audit; use Utopia\CLI\Console; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Audits V1 Worker'); Console::success(APP_NAME.' audits worker v1 has started'); diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 96148b9e71..0b64c087f4 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -9,7 +9,7 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Domains\Domain; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Certificates V1 Worker'); Console::success(APP_NAME.' certificates worker v1 has started'); diff --git a/app/workers/database.php b/app/workers/database.php index 60c060ee7a..3f551db27c 100644 --- a/app/workers/database.php +++ b/app/workers/database.php @@ -5,7 +5,7 @@ use Utopia\CLI\Console; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Database V1 Worker'); Console::success(APP_NAME.' database worker v1 has started'."\n"); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index e98723a5ea..f0e66d2b46 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -11,7 +11,7 @@ use Utopia\Abuse\Adapters\TimeLimit; use Utopia\CLI\Console; use Utopia\Audit\Audit; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Deletes V1 Worker'); Console::success(APP_NAME.' deletes worker v1 has started'."\n"); diff --git a/app/workers/functions.php b/app/workers/functions.php index bed0bda138..49b42b9492 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -13,7 +13,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Runtime::enableCoroutine(0); diff --git a/app/workers/mails.php b/app/workers/mails.php index 25abe54aa5..9071c00901 100644 --- a/app/workers/mails.php +++ b/app/workers/mails.php @@ -6,7 +6,7 @@ use Utopia\App; use Utopia\CLI\Console; use Utopia\Locale\Locale; -require_once __DIR__ . '/../workers.php'; +require_once __DIR__ . '/../init.php'; Console::title('Mails V1 Worker'); Console::success(APP_NAME . ' mails worker v1 has started' . "\n"); diff --git a/app/workers/webhooks.php b/app/workers/webhooks.php index 00307bdda2..a73dbb6c79 100644 --- a/app/workers/webhooks.php +++ b/app/workers/webhooks.php @@ -4,7 +4,7 @@ use Appwrite\Resque\Worker; use Utopia\App; use Utopia\CLI\Console; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Webhooks V1 Worker'); Console::success(APP_NAME.' webhooks worker v1 has started'); From 19c97d750921a5ebe4857b0310bbaf12d22e60b2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 13 Aug 2021 14:25:01 +0545 Subject: [PATCH 009/120] fix model task related issue --- src/Appwrite/Utopia/Response.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 3cd25b6032..de328bce75 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -125,6 +125,7 @@ class Response extends SwooleResponse // Deprecated const MODEL_PERMISSIONS = 'permissions'; const MODEL_RULE = 'rule'; + const MODEL_TASK = 'task'; // Tests (keep last) const MODEL_MOCK = 'mock'; From 9499211fdb835d24bf08258b775c8588ef59deb1 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 13 Aug 2021 14:25:10 +0545 Subject: [PATCH 010/120] delete worker.php --- app/workers.php | 47 ----------------------------------------------- 1 file changed, 47 deletions(-) delete mode 100644 app/workers.php diff --git a/app/workers.php b/app/workers.php deleted file mode 100644 index 56dc862fc6..0000000000 --- a/app/workers.php +++ /dev/null @@ -1,47 +0,0 @@ -set('db', function () { - $dbHost = App::getEnv('_APP_DB_HOST', ''); - $dbUser = App::getEnv('_APP_DB_USER', ''); - $dbPass = App::getEnv('_APP_DB_PASS', ''); - $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); - - $pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( - PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4', - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, - )); - - return $pdo; -}); - -$register->set('cache', function () { // Register cache connection - $redis = new Redis(); - $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); - - return $redis; -}); - -$register->set('influxdb', function () { // Register DB connection - $host = App::getEnv('_APP_INFLUXDB_HOST', ''); - $port = App::getEnv('_APP_INFLUXDB_PORT', ''); - - if (empty($host) || empty($port)) { - return; - } - $driver = new InfluxDB\Driver\Curl("http://{$host}:{$port}"); - $client = new InfluxDB\Client($host, $port, '', '', false, false, 5); - $client->setDriver($driver); - - return $client; -}); \ No newline at end of file From 00e0e0ae0e69631371f5599b656e294460757182 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 13 Aug 2021 14:27:56 +0545 Subject: [PATCH 011/120] usage daemon to production compose stack --- app/views/install/compose.phtml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index f2a214a8f9..e0a4515de2 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -288,6 +288,26 @@ services: - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + appwrite-usage: + image: /: + entrypoint: usage + container_name: appwrite-usage + restart: unless-stopped + networks: + - appwrite + depends_on: + - influxdb + - mariadb + environment: + - _APP_ENV + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_SYNC_INTERVAL appwrite-schedule: image: /: From 87de85fd1e65ce4b4b7ba2afdceb184abd142333 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 13 Aug 2021 15:30:46 +0545 Subject: [PATCH 012/120] refactor duplicat codes --- app/tasks/usage.php | 117 +++++++++++++------------------------------- 1 file changed, 35 insertions(+), 82 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index ae7de85e82..c6f49ea5cc 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -5,6 +5,7 @@ global $cli, $register; require_once __DIR__ . '/../init.php'; use Utopia\App; +use Utopia\Cache\Adapter\None; use Utopia\Cache\Adapter\Redis; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -22,11 +23,28 @@ $cli Console::success(APP_NAME . ' usage sync process v1 has started'); $interval = (int) App::getEnv('_APP_USAGE_SYNC_INTERVAL', '30'); //30 seconds + $attempts = 0; + $max = 10; + $sleep = 1; + do { + try { + $attempts++; + $db = $register->get('db'); + $redis = $register->get('cache'); + break; // leave the do-while if successful + } catch (\Exception$e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('Failed to connect to database: ' . $e->getMessage()); + } + sleep($sleep); + } + } while ($attempts < $max); - $cacheAdapter = new Cache(new Redis($register->get('cache'))); - $dbForConsole = new Database(new MariaDB($register->get('db')), $cacheAdapter); + $cacheAdapter = new Cache(new None()); + $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); $dbForConsole->setNamespace('project_console_internal'); - $dbForProject = new Database(new MariaDB($register->get('db')), $cacheAdapter); + $dbForProject = new Database(new MariaDB($db), $cacheAdapter); Authorization::disable(); $projectIds = []; @@ -36,7 +54,7 @@ $cli $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); if (!empty($projects)) { $latestProject = $projects[array_key_last($projects)]; - $latestData = getLatestData($projects, $latestData, $dbForProject); + $latestData = getLatestData($projects, $latestData, $dbForProject, $projectIds); } } while (!empty($projects)); @@ -51,7 +69,7 @@ $cli $projects = $dbForConsole->find('projects', limit:100, orderAfter:$latestProject); if (!empty($projects)) { $latestProject = $projects[array_key_last($projects)]; - $latestData = getLatestData($projects, $latestData, $dbForProject); + $latestData = getLatestData($projects, $latestData, $dbForProject, $projectIds); } } @@ -66,7 +84,7 @@ $cli }, $interval); }); -function getLatestData(&$projects, &$latestData, $dbForProject) +function getLatestData(&$projects, &$latestData, $dbForProject, &$projectIds) { foreach ($projects as $project) { $id = $project->getId(); @@ -83,61 +101,27 @@ function getLatestData(&$projects, &$latestData, $dbForProject) function syncData($client, $projectId, $period, &$latestData, $dbForProject) { - $requests = []; - $network = []; - $functions = []; $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-90 days' : '-1 days'))->format(DateTime::RFC3339); if (!empty($latestData[$projectId][$period])) { $start = DateTime::createFromFormat('U', $latestData[$projectId][$period])->format(DateTime::RFC3339); } $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); $database = $client->selectDB('telegraf'); - - // Requests - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\' GROUP BY time(' . $period . ') FILL(null)'); - $points = $result->getPoints(); - $dbForProject->setNamespace("project_{$projectId}_internal"); - foreach ($points as $point) { - $requests[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - 'dateStr' => $point['time'], - 'point' => $point, - ]; - $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period . '_requests'); - $value = (!empty($point['value'])) ? $point['value'] : 0; - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => 'requests', - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $value)); - } - $latestData[$id]["30m"] = $time; - } + + syncMetric($database, $projectId, $period, 'requests', $start, $end, $dbForProject); + syncMetric($database, $projectId, $period, 'network', $start, $end, $dbForProject); + syncMetric($database, $projectId, $period, 'executions', $start, $end, $dbForProject); +} - // Network - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . ') FILL(null)'); +function syncMetric($database, $projectId, $period, $metric, $start, $end, $dbForProject) +{ + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_' . $metric . '_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . ') FILL(null)'); $points = $result->getPoints(); foreach ($points as $point) { - $network[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - 'dateStr' => $point['time'], - 'point' => $point, - ]; $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period . '_network'); + $id = \md5($time . '_' . $period . '_' . $metric); $value = (!empty($point['value'])) ? $point['value'] : 0; $document = $dbForProject->getDocument('stats', $id); if ($document->isEmpty()) { @@ -145,7 +129,7 @@ function syncData($client, $projectId, $period, &$latestData, $dbForProject) '$id' => $id, 'period' => $period, 'time' => $time, - 'metric' => 'network', + 'metric' => $metric, 'value' => $value, 'type' => 0, ])); @@ -153,37 +137,6 @@ function syncData($client, $projectId, $period, &$latestData, $dbForProject) $dbForProject->updateDocument('stats', $document->getId(), $document->setAttribute('value', $value)); } - $latestData[$id]["30m"] = $time; - } - - // Functions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\'GROUP BY time(' . $period . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $functions[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - 'dateStr' => $point['time'], - 'point' => $point, - ]; - $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period . '_functions'); - $value = (!empty($point['value'])) ? $point['value'] : 0; - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => 'functions', - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $value)); - } - $latestData[$id]["30m"] = $time; + $latestData[$id][$period] = $time; } } From 3e3dbb69578a40876050ff4c4d78580aca557d1e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 13 Aug 2021 15:35:18 +0545 Subject: [PATCH 013/120] refactor --- app/tasks/usage.php | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index c6f49ea5cc..688f58a901 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -76,8 +76,7 @@ $cli $client = $register->get('influxdb'); if ($client) { foreach ($projectIds as $id => $value) { - syncData($client, $id, '30m', $latestData, $dbForProject); - syncData($client, $id, '1d', $latestData, $dbForProject); + syncData($client, $id, $latestData, $dbForProject); } } $firstRun = false; @@ -99,19 +98,22 @@ function getLatestData(&$projects, &$latestData, $dbForProject, &$projectIds) return $latestData; } -function syncData($client, $projectId, $period, &$latestData, $dbForProject) +function syncData($client, $projectId, &$latestData, $dbForProject) { - $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-90 days' : '-1 days'))->format(DateTime::RFC3339); - if (!empty($latestData[$projectId][$period])) { - $start = DateTime::createFromFormat('U', $latestData[$projectId][$period])->format(DateTime::RFC3339); + foreach (['30m', '1d'] as $period) { + $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-90 days' : '-1 days'))->format(DateTime::RFC3339); + if (!empty($latestData[$projectId][$period])) { + $start = DateTime::createFromFormat('U', $latestData[$projectId][$period])->format(DateTime::RFC3339); + } + $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); + $database = $client->selectDB('telegraf'); + $dbForProject->setNamespace("project_{$projectId}_internal"); + + syncMetric($database, $projectId, $period, 'requests', $start, $end, $dbForProject); + syncMetric($database, $projectId, $period, 'network', $start, $end, $dbForProject); + syncMetric($database, $projectId, $period, 'executions', $start, $end, $dbForProject); } - $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); - $dbForProject->setNamespace("project_{$projectId}_internal"); - - syncMetric($database, $projectId, $period, 'requests', $start, $end, $dbForProject); - syncMetric($database, $projectId, $period, 'network', $start, $end, $dbForProject); - syncMetric($database, $projectId, $period, 'executions', $start, $end, $dbForProject); + } function syncMetric($database, $projectId, $period, $metric, $start, $end, $dbForProject) From d94b20387095eec601f819ed9de44c0e19da30a4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 13 Aug 2021 15:45:25 +0545 Subject: [PATCH 014/120] add http path to usage request stats --- app/controllers/shared/api.php | 1 + src/Appwrite/Stats/Stats.php | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 674a5004b4..1c5e0f1064 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -101,6 +101,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $register, $e ->setParam('httpRequest', 1) ->setParam('httpUrl', $request->getHostname().$request->getURI()) ->setParam('httpMethod', $request->getMethod()) + ->setParam('httpPath', $route->getURL()) ->setParam('networkRequestSize', 0) ->setParam('networkResponseSize', 0) ->setParam('storage', 0) diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index b07c4c85c5..46c263ac38 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -87,6 +87,7 @@ class Stats $networkResponseSize = $this->params['networkResponseSize'] ?? 0; $httpMethod = $this->params['httpMethod'] ?? ''; + $httpPath = $this->params['httpPath'] ?? ''; $httpRequest = $this->params['httpRequest'] ?? 0; $functionId = $this->params['functionId'] ?? ''; @@ -100,7 +101,7 @@ class Stats $this->statsd->setNamespace($this->namespace); if ($httpRequest >= 1) { - $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod)); + $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod).',path='.$httpPath); } if ($functionExecution >= 1) { From ca64195eb1dc9e9e4864e6e0f78e996882a66559 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 14:15:51 +0545 Subject: [PATCH 015/120] fix special chars in path in statsd --- src/Appwrite/Stats/Stats.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index 46c263ac38..9625d1c201 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -101,7 +101,7 @@ class Stats $this->statsd->setNamespace($this->namespace); if ($httpRequest >= 1) { - $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod).',path='.$httpPath); + $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod) . ',path=' . str_replace(':', '*', $httpPath)); } if ($functionExecution >= 1) { From 4686e15a74dbe00655b614910c2529a18e86dc8b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 14:17:01 +0545 Subject: [PATCH 016/120] updated to get database stats --- app/tasks/usage.php | 129 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 116 insertions(+), 13 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 688f58a901..bc838b8b3b 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -89,10 +89,12 @@ function getLatestData(&$projects, &$latestData, $dbForProject, &$projectIds) $id = $project->getId(); $projectIds[$id] = true; $dbForProject->setNamespace("project_{$id}_internal"); - $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"])], 0, ['time'], [Database::ORDER_DESC]); - $latestData[$id]["1d"] = $doc ? $doc->getAttribute('time') : null; - $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 0, ['time'], [Database::ORDER_DESC]); - $latestData[$id]["30m"] = $doc ? $doc->getAttribute('time') : null; + foreach (['requests', 'network', 'executions', 'database.reads', 'database.creates', 'database.updates', 'database.deletes'] as $metric) { + $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"]), new Query("metric", Query::TYPE_EQUAL, [$metric])], 0, ['time'], [Database::ORDER_DESC]); + $latestData[$id][$metric]["1d"] = $doc ? $doc->getAttribute('time') : null; + $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 0, ['time'], [Database::ORDER_DESC]); + $latestData[$id][$metric]["30m"] = $doc ? $doc->getAttribute('time') : null; + } } $projects = null; return $latestData; @@ -101,21 +103,122 @@ function getLatestData(&$projects, &$latestData, $dbForProject, &$projectIds) function syncData($client, $projectId, &$latestData, $dbForProject) { foreach (['30m', '1d'] as $period) { - $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-90 days' : '-1 days'))->format(DateTime::RFC3339); - if (!empty($latestData[$projectId][$period])) { - $start = DateTime::createFromFormat('U', $latestData[$projectId][$period])->format(DateTime::RFC3339); - } + $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-90 days' : '-24 hours'))->format(DateTime::RFC3339); $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); $database = $client->selectDB('telegraf'); $dbForProject->setNamespace("project_{$projectId}_internal"); - - syncMetric($database, $projectId, $period, 'requests', $start, $end, $dbForProject); - syncMetric($database, $projectId, $period, 'network', $start, $end, $dbForProject); - syncMetric($database, $projectId, $period, 'executions', $start, $end, $dbForProject); + foreach (['requests', 'network', 'executions'] as $metric) { + if (!empty($latestData[$projectId][$metric][$period])) { + $start = DateTime::createFromFormat('U', $latestData[$projectId][$metric][$period])->format(DateTime::RFC3339); + } + syncMetric($database, $projectId, $period, 'requests', $start, $end, $dbForProject); + } + // syncMetric($database, $projectId, $period, 'network', $start, $end, $dbForProject); + // syncMetric($database, $projectId, $period, 'executions', $start, $end, $dbForProject); + syncMetricPaths($database, $projectId, $period, $start, $end, $dbForProject); } } +function syncMetricPaths($database, $projectId, $period, $start, $end, $dbForProject) +{ + $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-7 days' : '-24 hours'))->format(DateTime::RFC3339); + if (!empty($latestData[$projectId]['database'][$period])) { + $start = DateTime::createFromFormat('U', $latestData[$projectId]['database'][$period])->format(DateTime::RFC3339); + } + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . '), "path", "method" FILL(null)'); + $points = $result->getPoints(); + + $databaseMetrics = [ + 'creates' => [ + 'method' => 'post', + 'paths' => [ + '/v1/database/collections' => true, + '/v1/database/collections/*collectionId/attributes/string' => true, + '/v1/database/collections/*collectionId/attributes/email' => true, + '/v1/database/collections/*collectionId/attributes/ip' => true, + '/v1/database/collections/*collectionId/attributes/url' => true, + '/v1/database/collections/*collectionId/attributes/integer' => true, + '/v1/database/collections/*collectionId/attributes/float' => true, + '/v1/database/collections/*collectionId/attributes/boolean' => true, + '/v1/database/collections/*collectionId/indexes' => true, + '/v1/database/collections/*collectionId/documents' => true, + ], + ], + 'reads' => [ + 'method' => 'get', + 'paths' => [ + '/v1/database/collections' => true, + '/v1/database/collections/*collectionId' => true, + '/v1/database/collections/*collectionId/attributes' => true, + '/v1/database/collections/*collectionId/attributes/*attributeId' => true, + '/v1/database/collections/*collectionId/indexes' => true, + '/v1/database/collections/*collectionId/indexes/*indexId' => true, + '/v1/database/collections/*collectionId/documents' => true, + '/v1/database/collections/*collectionId/documents/*documentId' => true, + ], + ], + 'updates' => [ + 'method' => 'put', + 'paths' => [ + '/v1/database/collections/*collectionId' => true, + '/v1/database/collections/*collectionId/documents/*documentId' => true, + ], + ], + 'deletes' => [ + 'method' => 'delete', + 'paths' => [ + '/v1/database/collections/*collectionId' => true, + '/v1/database/collections/*collectionId/attributes/*attributeId' => true, + '/v1/database/collections/*collectionId/indexes/*indexId' => true, + '/v1/database/collections/*collectionId/documents/*documentId' => true, + ], + ], + ]; + + $dbStats = []; + foreach ($points as $point) { + $time = \strtotime($point['time']); + $value = (!empty($point['value'])) ? $point['value'] : 0; + $path = $point['path'] ?? ''; + $method = $point['method'] ?? ''; + + foreach (['creates', 'reads', 'updates', 'deletes'] as $operation) { + if ($method == $databaseMetrics[$operation]['method'] + && array_key_exists($path, $databaseMetrics[$operation]['paths'])) { + if (empty($dbStats["database.{$operation}"][$period][$time])) { + $dbStats["database.{$operation}"][$period][$time] = 0; + } + $dbStats["database.{$operation}"][$period][$time] += $value; + } + } + } + + $time = \strtotime($start); + foreach ($dbStats as $metric => $stats) { + foreach ($stats as $period => $times) { + foreach ($times as $time => $value) { + $id = \md5($time . '_' . $period . '_' . $metric); + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $metric, + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $value)); + } + } + } + } + $latestData[$projectId]['database'][$period] = $time; +} + function syncMetric($database, $projectId, $period, $metric, $start, $end, $dbForProject) { $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_' . $metric . '_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . ') FILL(null)'); @@ -139,6 +242,6 @@ function syncMetric($database, $projectId, $period, $metric, $start, $end, $dbFo $dbForProject->updateDocument('stats', $document->getId(), $document->setAttribute('value', $value)); } - $latestData[$id][$period] = $time; + $latestData[$projectId][$metric][$period] = $time; } } From 21774decdd0f1f4b698ea5464d0a0e420cee85a5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 14:23:31 +0545 Subject: [PATCH 017/120] some refactor --- app/tasks/usage.php | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index bc838b8b3b..f227519080 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -5,7 +5,6 @@ global $cli, $register; require_once __DIR__ . '/../init.php'; use Utopia\App; -use Utopia\Cache\Adapter\None; use Utopia\Cache\Adapter\Redis; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -41,7 +40,7 @@ $cli } } while ($attempts < $max); - $cacheAdapter = new Cache(new None()); + $cacheAdapter = new Cache(new Redis($redis)); $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); $dbForConsole->setNamespace('project_console_internal'); $dbForProject = new Database(new MariaDB($db), $cacheAdapter); @@ -89,7 +88,7 @@ function getLatestData(&$projects, &$latestData, $dbForProject, &$projectIds) $id = $project->getId(); $projectIds[$id] = true; $dbForProject->setNamespace("project_{$id}_internal"); - foreach (['requests', 'network', 'executions', 'database.reads', 'database.creates', 'database.updates', 'database.deletes'] as $metric) { + foreach (['requests', 'network', 'executions', 'database.reads'] as $metric) { $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"]), new Query("metric", Query::TYPE_EQUAL, [$metric])], 0, ['time'], [Database::ORDER_DESC]); $latestData[$id][$metric]["1d"] = $doc ? $doc->getAttribute('time') : null; $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 0, ['time'], [Database::ORDER_DESC]); @@ -113,8 +112,6 @@ function syncData($client, $projectId, &$latestData, $dbForProject) } syncMetric($database, $projectId, $period, 'requests', $start, $end, $dbForProject); } - // syncMetric($database, $projectId, $period, 'network', $start, $end, $dbForProject); - // syncMetric($database, $projectId, $period, 'executions', $start, $end, $dbForProject); syncMetricPaths($database, $projectId, $period, $start, $end, $dbForProject); } @@ -124,7 +121,7 @@ function syncMetricPaths($database, $projectId, $period, $start, $end, $dbForPro { $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-7 days' : '-24 hours'))->format(DateTime::RFC3339); if (!empty($latestData[$projectId]['database'][$period])) { - $start = DateTime::createFromFormat('U', $latestData[$projectId]['database'][$period])->format(DateTime::RFC3339); + $start = DateTime::createFromFormat('U', $latestData[$projectId]['database.reads'][$period])->format(DateTime::RFC3339); } $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . '), "path", "method" FILL(null)'); $points = $result->getPoints(); @@ -216,7 +213,7 @@ function syncMetricPaths($database, $projectId, $period, $start, $end, $dbForPro } } } - $latestData[$projectId]['database'][$period] = $time; + $latestData[$projectId]['database.reads'][$period] = $time; } function syncMetric($database, $projectId, $period, $metric, $start, $end, $dbForProject) From 0c2cfe2efc57d7a4e19ebdb7205926f4e01b6fb3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 14:25:55 +0545 Subject: [PATCH 018/120] Update app/views/install/compose.phtml Co-authored-by: Eldad A. Fux --- app/views/install/compose.phtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index e0a4515de2..c016c8ec81 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -307,7 +307,7 @@ services: - _APP_DB_PASS - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT - - _APP_USAGE_SYNC_INTERVAL + - _APP_USAGE_AGGREGATION_INTERVAL appwrite-schedule: image: /: From d9980cbf89add3dc1c7c864659e539b949fc52b9 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 14:26:19 +0545 Subject: [PATCH 019/120] fix env var --- app/tasks/usage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index f227519080..d1a2905e02 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -21,7 +21,7 @@ $cli Console::title('Usage Sync V1'); Console::success(APP_NAME . ' usage sync process v1 has started'); - $interval = (int) App::getEnv('_APP_USAGE_SYNC_INTERVAL', '30'); //30 seconds + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); //30 seconds $attempts = 0; $max = 10; $sleep = 1; From 94b77953325910310c265c2e1a9a9c2620730950 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 14:30:14 +0545 Subject: [PATCH 020/120] usage aggregation environment variable description --- app/config/variables.php | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/config/variables.php b/app/config/variables.php index df7346cd59..132e279c2c 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -149,6 +149,15 @@ return [ 'required' => false, 'question' => '', 'filter' => '' + ], + [ + 'name' => '_APP_USAGE_AGGREGATION_INTERVAL', + 'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats and syncing it to mariadb from InfluxDB. The default value is 30 seconds.', + 'introduction' => '0.10.0', + 'default' => '30', + 'required' => false, + 'question' => '', + 'filter' => '' ] ], ], From 251f14cd697dae47dacf964942a9c537ee4d5360 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 16:53:32 +0545 Subject: [PATCH 021/120] usage params in database endpoints --- app/controllers/api/database.php | 165 ++++++++++++++++++++++++------- 1 file changed, 132 insertions(+), 33 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index c923fd85b0..6c2e2525bd 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -25,7 +25,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; -$attributesCallback = function ($attribute, $response, $dbForExternal, $database, $audits) { +$attributesCallback = function ($attribute, $response, $dbForExternal, $database, $audits, $usage) { /** @var Utopia\Database\Document $document*/ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ @@ -110,6 +110,8 @@ $attributesCallback = function ($attribute, $response, $dbForExternal, $database ->setParam('document', $attribute) ; + $usage->setParam('database.collections.update', 1); + $audits ->setParam('event', 'database.attributes.create') ->setParam('resource', 'database/attributes/'.$attribute->getId()) @@ -139,10 +141,12 @@ App::post('/v1/database/collections') ->inject('response') ->inject('dbForExternal') ->inject('audits') - ->action(function ($collectionId, $name, $read, $write, $response, $dbForExternal, $audits) { + ->inject('usage') + ->action(function ($collectionId, $name, $read, $write, $response, $dbForExternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collectionId = $collectionId == 'unique()' ? $dbForExternal->getId() : $collectionId; @@ -164,6 +168,8 @@ App::post('/v1/database/collections') ->setParam('data', $collection->getArrayCopy()) ; + $usage->setParam('database.collections.create', 1); + $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($collection, Response::MODEL_COLLECTION); }); @@ -186,9 +192,11 @@ App::get('/v1/database/collections') ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForExternal') - ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, [$search])] : []; @@ -200,6 +208,8 @@ App::get('/v1/database/collections') } } + $usage->setParam('database.collections.read', 1); + $response->dynamic(new Document([ 'collections' => $dbForExternal->find(Database::COLLECTIONS, $queries, $limit, $offset, [], [$orderType], $afterCollection ?? null), 'sum' => $dbForExternal->count(Database::COLLECTIONS, $queries, APP_LIMIT_COUNT), @@ -220,9 +230,11 @@ App::get('/v1/database/collections/:collectionId') ->param('collectionId', '', new UID(), 'Collection unique ID.') ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -230,6 +242,8 @@ App::get('/v1/database/collections/:collectionId') throw new Exception('Collection not found', 404); } + $usage->setParam('database.collections.read', 1); + $response->dynamic($collection, Response::MODEL_COLLECTION); }); @@ -252,10 +266,12 @@ App::put('/v1/database/collections/:collectionId') ->inject('response') ->inject('dbForExternal') ->inject('audits') - ->action(function ($collectionId, $name, $read, $write, $response, $dbForExternal, $audits) { + ->inject('usage') + ->action(function ($collectionId, $name, $read, $write, $response, $dbForExternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -278,6 +294,8 @@ App::put('/v1/database/collections/:collectionId') throw new Exception('Bad structure. '.$exception->getMessage(), 400); } + $usage->setParam('database.collections.update', 1); + $audits ->setParam('event', 'database.collections.update') ->setParam('resource', 'database/collections/'.$collection->getId()) @@ -304,11 +322,13 @@ App::delete('/v1/database/collections/:collectionId') ->inject('events') ->inject('audits') ->inject('deletes') - ->action(function ($collectionId, $response, $dbForExternal, $events, $audits, $deletes) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForExternal, $events, $audits, $deletes, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $audits */ $collection = $dbForExternal->getCollection($collectionId); @@ -318,6 +338,8 @@ App::delete('/v1/database/collections/:collectionId') $dbForExternal->deleteCollection($collectionId); + $usage->setParam('database.collections.delete', 1); + $events ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)) ; @@ -353,11 +375,13 @@ App::post('/v1/database/collections/:collectionId/attributes/string') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForExternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForExternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback(new Document([ '$collection' => $collectionId, @@ -367,7 +391,7 @@ App::post('/v1/database/collections/:collectionId/attributes/string') 'required' => $required, 'default' => $default, 'array' => $array, - ]), $response, $dbForExternal, $database, $audits); + ]), $response, $dbForExternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/email') @@ -391,11 +415,13 @@ App::post('/v1/database/collections/:collectionId/attributes/email') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForExternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForExternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback(new Document([ '$collection' => $collectionId, @@ -406,7 +432,7 @@ App::post('/v1/database/collections/:collectionId/attributes/email') 'default' => $default, 'array' => $array, 'format' => \json_encode(['name'=>'email']), - ]), $response, $dbForExternal, $database, $audits); + ]), $response, $dbForExternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/ip') @@ -430,11 +456,13 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForExternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForExternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback(new Document([ '$collection' => $collectionId, @@ -445,7 +473,7 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') 'default' => $default, 'array' => $array, 'format' => \json_encode(['name'=>'ip']), - ]), $response, $dbForExternal, $database, $audits); + ]), $response, $dbForExternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/url') @@ -470,11 +498,13 @@ App::post('/v1/database/collections/:collectionId/attributes/url') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForExternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForExternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback(new Document([ '$collection' => $collectionId, @@ -485,7 +515,7 @@ App::post('/v1/database/collections/:collectionId/attributes/url') 'default' => $default, 'array' => $array, 'format' => \json_encode(['name'=>'url']), - ]), $response, $dbForExternal, $database, $audits); + ]), $response, $dbForExternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/integer') @@ -511,12 +541,15 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForExternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForExternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ + return $attributesCallback(new Document([ '$collection' => $collectionId, '$id' => $attributeId, @@ -530,7 +563,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') 'min' => $min, 'max' => $max, ]), - ]), $response, $dbForExternal, $database, $audits); + ]), $response, $dbForExternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/float') @@ -556,11 +589,13 @@ App::post('/v1/database/collections/:collectionId/attributes/float') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForExternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForExternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback(new Document([ '$collection' => $collectionId, @@ -575,7 +610,7 @@ App::post('/v1/database/collections/:collectionId/attributes/float') 'min' => $min, 'max' => $max, ]), - ]), $response, $dbForExternal, $database, $audits); + ]), $response, $dbForExternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/boolean') @@ -599,11 +634,13 @@ App::post('/v1/database/collections/:collectionId/attributes/boolean') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForExternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForExternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback(new Document([ '$collection' => $collectionId, @@ -613,7 +650,7 @@ App::post('/v1/database/collections/:collectionId/attributes/boolean') 'required' => $required, 'default' => $default, 'array' => $array, - ]), $response, $dbForExternal, $database, $audits); + ]), $response, $dbForExternal, $database, $audits, $usage); }); App::get('/v1/database/collections/:collectionId/attributes') @@ -630,9 +667,11 @@ App::get('/v1/database/collections/:collectionId/attributes') ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -648,6 +687,8 @@ App::get('/v1/database/collections/:collectionId/attributes') ])]); }, $attributes); + $usage->setParam('database.collections.read', 1); + $response->dynamic(new Document([ 'sum' => \count($attributes), 'attributes' => $attributes @@ -669,9 +710,11 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId') ->param('attributeId', '', new Key(), 'Attribute ID.') ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $attributeId, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -691,6 +734,8 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId') $attribute = new Document([\array_merge($attributes[$attributeIndex], [ 'collectionId' => $collectionId, ])]); + + $usage->setParam('database.collections.read', 1); $response->dynamic($attribute, Response::MODEL_ATTRIBUTE); }); @@ -713,12 +758,14 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') ->inject('database') ->inject('events') ->inject('audits') - ->action(function ($collectionId, $attributeId, $response, $dbForExternal, $database, $events, $audits) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $response, $dbForExternal, $database, $events, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -747,6 +794,8 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') ->setParam('document', $attribute) ; + $usage->setParam('database.collections.update', 1); + $events ->setParam('payload', $response->output($attribute, Response::MODEL_ATTRIBUTE)) ; @@ -781,11 +830,13 @@ App::post('/v1/database/collections/:collectionId/indexes') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $indexId, $type, $attributes, $orders, $response, $dbForExternal, $database, $audits) { + ->inject('usage') + ->action(function ($collectionId, $indexId, $type, $attributes, $orders, $response, $dbForExternal, $database, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $audits */ $collection = $dbForExternal->getCollection($collectionId); @@ -837,6 +888,8 @@ App::post('/v1/database/collections/:collectionId/indexes') ->setParam('document', $index) ; + $usage->setParam('database.collections.update', 1); + $audits ->setParam('event', 'database.indexes.create') ->setParam('resource', 'database/indexes/'.$index->getId()) @@ -862,9 +915,11 @@ App::get('/v1/database/collections/:collectionId/indexes') ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -880,6 +935,8 @@ App::get('/v1/database/collections/:collectionId/indexes') ])]); }, $indexes); + $usage->setParam('database.collections.read', 1); + $response->dynamic(new Document([ 'sum' => \count($indexes), 'attributes' => $indexes, @@ -901,9 +958,11 @@ App::get('/v1/database/collections/:collectionId/indexes/:indexId') ->param('indexId', null, new Key(), 'Index ID.') ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $indexId, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $indexId, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -923,6 +982,8 @@ App::get('/v1/database/collections/:collectionId/indexes/:indexId') $index = new Document([\array_merge($indexes[$indexIndex], [ 'collectionId' => $collectionId, ])]); + + $usage->setParam('database.collections.read', 1); $response->dynamic($index, Response::MODEL_INDEX); }); @@ -945,12 +1006,14 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') ->inject('database') ->inject('events') ->inject('audits') - ->action(function ($collectionId, $indexId, $response, $dbForExternal, $database, $events, $audits) { + ->inject('usage') + ->action(function ($collectionId, $indexId, $response, $dbForExternal, $database, $events, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -979,6 +1042,8 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') ->setParam('document', $index) ; + $usage->setParam('database.collections.update', 1); + $events ->setParam('payload', $response->output($index, Response::MODEL_INDEX)) ; @@ -1013,11 +1078,13 @@ App::post('/v1/database/collections/:collectionId/documents') ->inject('dbForExternal') ->inject('user') ->inject('audits') - ->action(function ($documentId, $collectionId, $data, $read, $write, $response, $dbForExternal, $user, $audits) { + ->inject('usage') + ->action(function ($documentId, $collectionId, $data, $read, $write, $response, $dbForExternal, $user, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -1046,6 +1113,11 @@ App::post('/v1/database/collections/:collectionId/documents') throw new Exception($exception->getMessage(), 400); } + $usage + ->setParam('database.documents.create', 1) + ->setParam('database.collections.' . $collectionId . '.documents.create', 1) + ; + $audits ->setParam('event', 'database.documents.create') ->setParam('resource', 'database/document/'.$document->getId()) @@ -1076,9 +1148,11 @@ App::get('/v1/database/collections/:collectionId/documents') ->param('orderTypes', [], new ArrayList(new WhiteList(['DESC', 'ASC'], true)), 'Array of order directions for sorting attribtues. Possible values are DESC for descending order, or ASC for ascending order.', true) ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $queries, $limit, $offset, $after, $orderAttributes, $orderTypes, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $queries, $limit, $offset, $after, $orderAttributes, $orderTypes, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -1112,6 +1186,11 @@ App::get('/v1/database/collections/:collectionId/documents') $documents = $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null); + $usage + ->setParam('database.documents.read', 1) + ->setParam('database.collections.' . $collectionId . '.documents.read', 1) + ; + $response->dynamic(new Document([ 'sum' => \count($documents), 'documents' => $documents, @@ -1133,7 +1212,8 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') ->param('documentId', null, new UID(), 'Document unique ID.') ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $documentId, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $documentId, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ @@ -1149,6 +1229,11 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') throw new Exception('No document found', 404); } + $usage + ->setParam('database.documents.read', 1) + ->setParam('database.collections.' . $collectionId . '.documents.read', 1) + ; + $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -1172,10 +1257,12 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') ->inject('response') ->inject('dbForExternal') ->inject('audits') - ->action(function ($collectionId, $documentId, $data, $read, $write, $response, $dbForExternal, $audits) { + ->inject('usage') + ->action(function ($collectionId, $documentId, $data, $read, $write, $response, $dbForExternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -1212,7 +1299,12 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') throw new Exception('Unauthorized permissions', 401); } catch (StructureException $exception) { throw new Exception('Bad structure. '.$exception->getMessage(), 400); - } + } + + $usage + ->setParam('database.documents.update', 1) + ->setParam('database.collections.' . $collectionId . '.documents.update', 1) + ; $audits ->setParam('event', 'database.documents.update') @@ -1240,11 +1332,13 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') ->inject('dbForExternal') ->inject('events') ->inject('audits') - ->action(function ($collectionId, $documentId, $response, $dbForExternal, $events, $audits) { + ->inject('usage') + ->action(function ($collectionId, $documentId, $response, $dbForExternal, $events, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -1260,6 +1354,11 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') $success = $dbForExternal->deleteDocument($collectionId, $documentId); + $usage + ->setParam('database.documents.delete', 1) + ->setParam('database.collections.' . $collectionId . '.documents.delete', 1) + ; + $events ->setParam('eventData', $response->output($document, Response::MODEL_DOCUMENT)) ; From cc49cb6a0450a4e2a63f3ec19f043d90c1e23e1e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 17:02:46 +0545 Subject: [PATCH 022/120] set collection Id as param --- app/controllers/api/database.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 6c2e2525bd..c3ed0edfa5 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1115,7 +1115,7 @@ App::post('/v1/database/collections/:collectionId/documents') $usage ->setParam('database.documents.create', 1) - ->setParam('database.collections.' . $collectionId . '.documents.create', 1) + ->setParam('collectionId', $collectionId) ; $audits @@ -1188,7 +1188,7 @@ App::get('/v1/database/collections/:collectionId/documents') $usage ->setParam('database.documents.read', 1) - ->setParam('database.collections.' . $collectionId . '.documents.read', 1) + ->setParam('collectionId', $collectionId) ; $response->dynamic(new Document([ @@ -1231,7 +1231,7 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.read', 1) - ->setParam('database.collections.' . $collectionId . '.documents.read', 1) + ->setParam('collectionId', $collectionId) ; $response->dynamic($document, Response::MODEL_DOCUMENT); @@ -1303,7 +1303,7 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.update', 1) - ->setParam('database.collections.' . $collectionId . '.documents.update', 1) + ->setParam('collectionId', $collectionId) ; $audits @@ -1356,7 +1356,7 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.delete', 1) - ->setParam('database.collections.' . $collectionId . '.documents.delete', 1) + ->setParam('collectionId', $collectionId) ; $events From 4bab152b3d5a5b61378388932df2c2918f48c78e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 17:24:49 +0545 Subject: [PATCH 023/120] initial db metrics collection --- src/Appwrite/Stats/Stats.php | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index 9625d1c201..2f85be805c 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -87,7 +87,6 @@ class Stats $networkResponseSize = $this->params['networkResponseSize'] ?? 0; $httpMethod = $this->params['httpMethod'] ?? ''; - $httpPath = $this->params['httpPath'] ?? ''; $httpRequest = $this->params['httpRequest'] ?? 0; $functionId = $this->params['functionId'] ?? ''; @@ -101,7 +100,7 @@ class Stats $this->statsd->setNamespace($this->namespace); if ($httpRequest >= 1) { - $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod) . ',path=' . str_replace(':', '*', $httpPath)); + $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod)); } if ($functionExecution >= 1) { @@ -113,6 +112,25 @@ class Stats $this->statsd->count('network.outbound' . $tags, $networkResponseSize); $this->statsd->count('network.all' . $tags, $networkRequestSize + $networkResponseSize); + $dbMetrics = [ + 'database.collections.create', + 'database.collections.read', + 'database.collections.update', + 'database.collections.delete', + 'database.documents.create', + 'database.documents.read', + 'database.documents.update', + 'database.documents.delete', + ]; + + foreach ($dbMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $dbTags = ",project={$projectId},collectionId=" . ($this->params['collectionId'] ?? ''); + $this->statsd->increment($metric . $dbTags); + } + } + if ($storage >= 1) { $this->statsd->count('storage.all' . $tags, $storage); } From 8a207f02a0261391ed91d0034d8af6164e1c6614 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 12:43:34 +0545 Subject: [PATCH 024/120] project stats and database stats --- app/controllers/api/projects.php | 6 +- app/tasks/usage.php | 345 ++++++++++++++----------------- src/Appwrite/Stats/Stats.php | 4 +- 3 files changed, 165 insertions(+), 190 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index e34c948837..c302e882b3 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -279,7 +279,7 @@ App::get('/v1/projects/:projectId/usage') $database = $client->selectDB('telegraf'); // Requests - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); $points = $result->getPoints(); foreach ($points as $point) { @@ -290,7 +290,7 @@ App::get('/v1/projects/:projectId/usage') } // Network - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); $points = $result->getPoints(); foreach ($points as $point) { @@ -301,7 +301,7 @@ App::get('/v1/projects/:projectId/usage') } // Functions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); $points = $result->getPoints(); foreach ($points as $point) { diff --git a/app/tasks/usage.php b/app/tasks/usage.php index d1a2905e02..fa8de4c86f 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -14,6 +14,34 @@ use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; +/** + * 1. Load all the projects + * 2. Load latest data entered entered for each project, for each period + * 3. Start the loop + * 4. Fore each project, for each metric, for each period - sync data + */ + +/** + * Only succefull operations + * + * database.collections.CRUD (project=x) + * database.documents.CRUD (project=x,collection=y) + * + * users.CRUD + * users.sessions.create (project=x,provider=y) + * users.sessions.delete (project=x,provider=y) + * + * storage.buckets.CRUD (project=x) + * storage.files.CRUD (project=x,files=y) + * + * refactor later + * - functions + * - realtime + * - teams + * - webhooks + * - keys - really later! + */ + $cli ->task('usage') ->desc('Schedules syncing data from influxdb to Appwrite console db') @@ -22,10 +50,90 @@ $cli Console::success(APP_NAME . ' usage sync process v1 has started'); $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); //30 seconds + $periods = [ + [ + 'key' => '30m', + 'startTime' => '-24 hours', + ], + [ + 'key' => '1d', + 'startTime' => '-90 days', + ], + ]; + + //use projectId from influxdb instead of iterating over projects from DB + + $globalMetrics = [ + 'requests' => [ + 'method' => 'getGlobalMetrics', + 'table' => 'appwrite_usage_requests_all', + ], + 'network' => [ + 'method' => 'getGlobalMetrics', + 'table' => 'appwrite_usage_network_all', + ], + 'executions' => [ + 'method' => 'getGlobalMetrics', + 'table' => 'appwrite_usage_executions_all', + ], + 'database.collections.create' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_collections_create', + ], + 'database.collections.read' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_collections_read', + ], + 'database.collections.update' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_collections_update', + ], + 'database.collections.delete' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_collections_delete', + ], + 'database.documents.create' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_create', + ], + 'database.documents.read' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_read', + ], + 'database.documents.update' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_update', + ], + 'database.documents.delete' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_delete', + ], + 'database.documents.collectionId.create' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_create', + 'groupBy' => 'collectionId', + ], + 'database.documents.collectionId.read' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_read', + 'groupBy' => 'collectionId', + ], + 'database.documents.collectionId.update' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_update', + 'groupBy' => 'collectionId', + ], + 'database.documents.collectionId.delete' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_delete', + 'groupBy' => 'collectionId', + ], + ]; + $attempts = 0; $max = 10; $sleep = 1; - do { + do { // connect to db try { $attempts++; $db = $register->get('db'); @@ -46,199 +154,66 @@ $cli $dbForProject = new Database(new MariaDB($db), $cacheAdapter); Authorization::disable(); - $projectIds = []; - $latestProject = null; $latestData = []; - do { - $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); - if (!empty($projects)) { - $latestProject = $projects[array_key_last($projects)]; - $latestData = getLatestData($projects, $latestData, $dbForProject, $projectIds); - } - } while (!empty($projects)); - - $projects = null; $firstRun = true; - Console::loop(function () use ($interval, $register, &$projectIds, &$latestData, $dbForProject, $dbForConsole, &$firstRun, &$latestProject) { + Console::loop(function () use ($interval, $register, &$latestData, $dbForProject, $dbForConsole, &$firstRun, $globalMetrics, $periods) { $time = date('d-m-Y H:i:s', time()); - Console::info("[{$time}] Syncing usage data from influxdb to Appwrite Console DB every {$interval} seconds"); - - if (!$firstRun) { - $projects = $dbForConsole->find('projects', limit:100, orderAfter:$latestProject); - if (!empty($projects)) { - $latestProject = $projects[array_key_last($projects)]; - $latestData = getLatestData($projects, $latestData, $dbForProject, $projectIds); - } - } + Console::info("[{$time}] Aggregating usage data every {$interval} seconds"); $client = $register->get('influxdb'); if ($client) { - foreach ($projectIds as $id => $value) { - syncData($client, $id, $latestData, $dbForProject); + $database = $client->selectDB('telegraf'); + // sync data + foreach ($globalMetrics as $metric => $options) { + foreach ($periods as $period) { + $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339); + $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); + + $table = $options['table']; + $groupBy = $options['groupBy'] ?? ''; + + $query = 'SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . $period['key'] . '), "projectId"' . (empty($groupBy) ? '' : ', "' . $groupBy . '"') . ' FILL(null)'; + $result = $database->query($query); + $points = $result->getPoints(); + foreach ($points as $point) { + $projectId = $point['projectId']; + if (!empty($projectId) && $projectId != 'console') { + $dbForProject->setNamespace('project_' . $projectId . '_internal'); + if (!empty($groupBy)) { + $groupedBy = $point[$groupBy]; + if (empty($groupedBy)) { + continue; + } + $metric = str_replace($groupBy, $groupedBy, $metric); + } + $time = \strtotime($point['time']); + $id = \md5($time . '_' . $period['key'] . '_' . $metric); + $value = (!empty($point['value'])) ? $point['value'] : 0; + try { + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period['key'], + 'time' => $time, + 'metric' => $metric, + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $value)); + } + $latestData[$projectId][$metric][$period['key']] = $time; + } catch (\Exception$e) { + Console::warning("Failed to save data for project {$projectId} and metric {$metric}"); + } + } + } + } } } $firstRun = false; }, $interval); }); - -function getLatestData(&$projects, &$latestData, $dbForProject, &$projectIds) -{ - foreach ($projects as $project) { - $id = $project->getId(); - $projectIds[$id] = true; - $dbForProject->setNamespace("project_{$id}_internal"); - foreach (['requests', 'network', 'executions', 'database.reads'] as $metric) { - $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"]), new Query("metric", Query::TYPE_EQUAL, [$metric])], 0, ['time'], [Database::ORDER_DESC]); - $latestData[$id][$metric]["1d"] = $doc ? $doc->getAttribute('time') : null; - $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 0, ['time'], [Database::ORDER_DESC]); - $latestData[$id][$metric]["30m"] = $doc ? $doc->getAttribute('time') : null; - } - } - $projects = null; - return $latestData; -} - -function syncData($client, $projectId, &$latestData, $dbForProject) -{ - foreach (['30m', '1d'] as $period) { - $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-90 days' : '-24 hours'))->format(DateTime::RFC3339); - $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); - $dbForProject->setNamespace("project_{$projectId}_internal"); - foreach (['requests', 'network', 'executions'] as $metric) { - if (!empty($latestData[$projectId][$metric][$period])) { - $start = DateTime::createFromFormat('U', $latestData[$projectId][$metric][$period])->format(DateTime::RFC3339); - } - syncMetric($database, $projectId, $period, 'requests', $start, $end, $dbForProject); - } - syncMetricPaths($database, $projectId, $period, $start, $end, $dbForProject); - } - -} - -function syncMetricPaths($database, $projectId, $period, $start, $end, $dbForProject) -{ - $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-7 days' : '-24 hours'))->format(DateTime::RFC3339); - if (!empty($latestData[$projectId]['database'][$period])) { - $start = DateTime::createFromFormat('U', $latestData[$projectId]['database.reads'][$period])->format(DateTime::RFC3339); - } - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . '), "path", "method" FILL(null)'); - $points = $result->getPoints(); - - $databaseMetrics = [ - 'creates' => [ - 'method' => 'post', - 'paths' => [ - '/v1/database/collections' => true, - '/v1/database/collections/*collectionId/attributes/string' => true, - '/v1/database/collections/*collectionId/attributes/email' => true, - '/v1/database/collections/*collectionId/attributes/ip' => true, - '/v1/database/collections/*collectionId/attributes/url' => true, - '/v1/database/collections/*collectionId/attributes/integer' => true, - '/v1/database/collections/*collectionId/attributes/float' => true, - '/v1/database/collections/*collectionId/attributes/boolean' => true, - '/v1/database/collections/*collectionId/indexes' => true, - '/v1/database/collections/*collectionId/documents' => true, - ], - ], - 'reads' => [ - 'method' => 'get', - 'paths' => [ - '/v1/database/collections' => true, - '/v1/database/collections/*collectionId' => true, - '/v1/database/collections/*collectionId/attributes' => true, - '/v1/database/collections/*collectionId/attributes/*attributeId' => true, - '/v1/database/collections/*collectionId/indexes' => true, - '/v1/database/collections/*collectionId/indexes/*indexId' => true, - '/v1/database/collections/*collectionId/documents' => true, - '/v1/database/collections/*collectionId/documents/*documentId' => true, - ], - ], - 'updates' => [ - 'method' => 'put', - 'paths' => [ - '/v1/database/collections/*collectionId' => true, - '/v1/database/collections/*collectionId/documents/*documentId' => true, - ], - ], - 'deletes' => [ - 'method' => 'delete', - 'paths' => [ - '/v1/database/collections/*collectionId' => true, - '/v1/database/collections/*collectionId/attributes/*attributeId' => true, - '/v1/database/collections/*collectionId/indexes/*indexId' => true, - '/v1/database/collections/*collectionId/documents/*documentId' => true, - ], - ], - ]; - - $dbStats = []; - foreach ($points as $point) { - $time = \strtotime($point['time']); - $value = (!empty($point['value'])) ? $point['value'] : 0; - $path = $point['path'] ?? ''; - $method = $point['method'] ?? ''; - - foreach (['creates', 'reads', 'updates', 'deletes'] as $operation) { - if ($method == $databaseMetrics[$operation]['method'] - && array_key_exists($path, $databaseMetrics[$operation]['paths'])) { - if (empty($dbStats["database.{$operation}"][$period][$time])) { - $dbStats["database.{$operation}"][$period][$time] = 0; - } - $dbStats["database.{$operation}"][$period][$time] += $value; - } - } - } - - $time = \strtotime($start); - foreach ($dbStats as $metric => $stats) { - foreach ($stats as $period => $times) { - foreach ($times as $time => $value) { - $id = \md5($time . '_' . $period . '_' . $metric); - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $metric, - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $value)); - } - } - } - } - $latestData[$projectId]['database.reads'][$period] = $time; -} - -function syncMetric($database, $projectId, $period, $metric, $start, $end, $dbForProject) -{ - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_' . $metric . '_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period . '_' . $metric); - $value = (!empty($point['value'])) ? $point['value'] : 0; - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $metric, - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $value)); - } - $latestData[$projectId][$metric][$period] = $time; - } -} diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index 2f85be805c..60609f1a4e 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -94,7 +94,7 @@ class Stats $functionExecutionTime = $this->params['functionExecutionTime'] ?? 0; $functionStatus = $this->params['functionStatus'] ?? ''; - $tags = ",project={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); + $tags = ",projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); // the global namespace is prepended to every key (optional) $this->statsd->setNamespace($this->namespace); @@ -126,7 +126,7 @@ class Stats foreach ($dbMetrics as $metric) { $value = $this->params[$metric] ?? 0; if ($value >= 1) { - $dbTags = ",project={$projectId},collectionId=" . ($this->params['collectionId'] ?? ''); + $dbTags = ",projectId={$projectId},collectionId=" . ($this->params['collectionId'] ?? ''); $this->statsd->increment($metric . $dbTags); } } From 1e9f3e38b200aec910774686b8e80bb20ed8757d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 13:10:20 +0545 Subject: [PATCH 025/120] storage stats --- app/controllers/api/storage.php | 64 ++++++++++++++++++++++++++++----- app/tasks/usage.php | 22 +++++++++++- src/Appwrite/Stats/Stats.php | 15 ++++++++ 3 files changed, 91 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 5a5c43de58..651985057a 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -54,7 +54,7 @@ App::post('/v1/storage/files') /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Stats\Stats $usage */ $file = $request->getFiles('file'); @@ -150,6 +150,8 @@ App::post('/v1/storage/files') $usage ->setParam('storage', $sizeActual) + ->setParam('storage.files.create', 1) + ->setParam('bucketId', 'default') ; $response->setStatusCode(Response::STATUS_CODE_CREATED); @@ -175,9 +177,11 @@ App::get('/v1/storage/files') ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, $search)] : []; @@ -189,6 +193,11 @@ App::get('/v1/storage/files') } } + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + $response->dynamic(new Document([ 'files' => $dbForInternal->find('files', $queries, $limit, $offset, [], [$orderType], $afterFile ?? null), 'sum' => $dbForInternal->count('files', $queries, APP_LIMIT_COUNT), @@ -209,16 +218,21 @@ App::get('/v1/storage/files/:fileId') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($fileId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); if (empty($file->getId())) { throw new Exception('File not found', 404); } - + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; $response->dynamic($file, Response::MODEL_FILE); }); @@ -249,11 +263,13 @@ App::get('/v1/storage/files/:fileId/preview') ->inject('response') ->inject('project') ->inject('dbForInternal') - ->action(function ($fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForInternal, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $stats */ $storage = 'files'; @@ -366,6 +382,11 @@ App::get('/v1/storage/files/:fileId/preview') $cache->save($key, $data); + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + $response ->setContentType($outputs[$output]) ->addHeader('Expires', $date) @@ -390,9 +411,11 @@ App::get('/v1/storage/files/:fileId/download') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($fileId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); @@ -424,6 +447,11 @@ App::get('/v1/storage/files/:fileId/download') $source = $compressor->decompress($source); + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + // Response $response ->setContentType($file->getAttribute('mimeType')) @@ -448,9 +476,11 @@ App::get('/v1/storage/files/:fileId/view') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($fileId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); $mimes = Config::getParam('storage-mimes'); @@ -490,6 +520,11 @@ App::get('/v1/storage/files/:fileId/view') $output = $compressor->decompress($source); $fileName = $file->getAttribute('name', ''); + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + // Response $response ->setContentType($contentType) @@ -520,7 +555,8 @@ App::put('/v1/storage/files/:fileId') ->inject('response') ->inject('dbForInternal') ->inject('audits') - ->action(function ($fileId, $read, $write, $response, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($fileId, $read, $write, $response, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ @@ -542,6 +578,11 @@ App::put('/v1/storage/files/:fileId') ->setParam('resource', 'storage/files/'.$file->getId()) ; + $usage + ->setParam('storage.files.update', 1) + ->setParam('bucketId', 'default') + ; + $response->dynamic($file, Response::MODEL_FILE); }); @@ -567,7 +608,7 @@ App::delete('/v1/storage/files/:fileId') /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); @@ -596,5 +637,10 @@ App::delete('/v1/storage/files/:fileId') ->setParam('eventData', $response->output($file, Response::MODEL_FILE)) ; + $usage + ->setParam('storage.files.delete', 1) + ->setParam('bucketId', 'default') + ; + $response->noContent(); }); \ No newline at end of file diff --git a/app/tasks/usage.php b/app/tasks/usage.php index fa8de4c86f..5230963098 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -32,7 +32,7 @@ use Utopia\Database\Validator\Authorization; * users.sessions.delete (project=x,provider=y) * * storage.buckets.CRUD (project=x) - * storage.files.CRUD (project=x,files=y) + * storage.files.CRUD (project=x,bucket=y) * * refactor later * - functions @@ -128,6 +128,26 @@ $cli 'table' => 'appwrite_usage_database_documents_delete', 'groupBy' => 'collectionId', ], + 'storage.buckets.bucketId.files.create' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_storage_files_create', + 'groupBy' => 'bucketId', + ], + 'storage.buckets.bucketId.files.read' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_storage_files_read', + 'groupBy' => 'bucketId', + ], + 'storage.buckets.bucketId.files.update' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_storage_files_update', + 'groupBy' => 'bucketId', + ], + 'storage.buckets.bucketId.files.delete' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_storage_files_delete', + 'groupBy' => 'bucketId', + ], ]; $attempts = 0; diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index 60609f1a4e..5c24e1ae94 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -131,6 +131,21 @@ class Stats } } + $storageMertics = [ + 'storage.files.create', + 'storage.files.read', + 'storage.files.update', + 'storage.files.delete', + ]; + + foreach ($storageMertics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $storageTags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? ''); + $this->statsd->increment($metric . $storageTags); + } + } + if ($storage >= 1) { $this->statsd->count('storage.all' . $tags, $storage); } From b4c794c7ba8bdf507f466286bc763b9acb76f988 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 13:41:18 +0545 Subject: [PATCH 026/120] users usage params --- app/controllers/api/users.php | 94 ++++++++++++++++++++++++++++------- 1 file changed, 77 insertions(+), 17 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 11beca048b..eb3abd12cf 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -36,9 +36,11 @@ App::post('/v1/users') ->param('name', '', new Text(128), 'User name. Max length: 128 chars.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $email, $password, $name, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $email, $password, $name, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $email = \strtolower($email); @@ -65,6 +67,10 @@ App::post('/v1/users') throw new Exception('Account already exists', 409); } + $usage + ->setParam('users.create', 1) + ; + $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($user, Response::MODEL_USER); }); @@ -87,9 +93,11 @@ App::get('/v1/users') ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ if (!empty($after)) { $afterUser = $dbForInternal->getDocument('users', $after); @@ -101,6 +109,10 @@ App::get('/v1/users') $results = $dbForInternal->find('users', [], $limit, $offset, [], [$orderType], $afterUser ?? null); $sum = $dbForInternal->count('users', [], APP_LIMIT_COUNT); + + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document([ 'users' => $results, @@ -122,9 +134,11 @@ App::get('/v1/users/:userId') ->param('userId', '', new UID(), 'User unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -132,6 +146,9 @@ App::get('/v1/users/:userId') throw new Exception('User not found', 404); } + $usage + ->setParam('users.read', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -149,9 +166,11 @@ App::get('/v1/users/:userId/prefs') ->param('userId', '', new UID(), 'User unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -161,6 +180,9 @@ App::get('/v1/users/:userId/prefs') $prefs = $user->getAttribute('prefs', new \stdClass()); + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES); }); @@ -179,10 +201,12 @@ App::get('/v1/users/:userId/sessions') ->inject('response') ->inject('dbForInternal') ->inject('locale') - ->action(function ($userId, $response, $dbForInternal, $locale) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $locale, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Locale\Locale $locale */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -202,6 +226,9 @@ App::get('/v1/users/:userId/sessions') $sessions[$key] = $session; } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document([ 'sessions' => $sessions, 'sum' => count($sessions), @@ -224,12 +251,14 @@ App::get('/v1/users/:userId/logs') ->inject('dbForInternal') ->inject('locale') ->inject('geodb') - ->action(function ($userId, $response, $dbForInternal, $locale, $geodb) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $locale, $geodb, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Locale\Locale $locale */ /** @var MaxMind\Db\Reader $geodb */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -311,6 +340,9 @@ App::get('/v1/users/:userId/logs') } } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document(['logs' => $output]), Response::MODEL_LOG_LIST); }); @@ -330,9 +362,11 @@ App::patch('/v1/users/:userId/status') ->param('status', null, new Boolean(true), 'User Status. To activate the user pass `true` and to block the user pass `false`') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $status, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $status, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -342,6 +376,9 @@ App::patch('/v1/users/:userId/status') $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('status', (bool) $status)); + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -361,9 +398,11 @@ App::patch('/v1/users/:userId/verification') ->param('emailVerification', false, new Boolean(), 'User Email Verification Status.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $emailVerification, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $emailVerification, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -373,6 +412,9 @@ App::patch('/v1/users/:userId/verification') $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('emailVerification', $emailVerification)); + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -392,9 +434,11 @@ App::patch('/v1/users/:userId/prefs') ->param('prefs', '', new Assoc(), 'Prefs key-value JSON object.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $prefs, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $prefs, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -404,6 +448,9 @@ App::patch('/v1/users/:userId/prefs') $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('prefs', $prefs)); + $usage + ->setParam('users.update', 1) + ; $response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES); }); @@ -423,10 +470,12 @@ App::delete('/v1/users/:userId/sessions/:sessionId') ->inject('response') ->inject('dbForInternal') ->inject('events') - ->action(function ($userId, $sessionId, $response, $dbForInternal, $events) { + ->inject('usage') + ->action(function ($userId, $sessionId, $response, $dbForInternal, $events, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -453,6 +502,11 @@ App::delete('/v1/users/:userId/sessions/:sessionId') } } + $usage + ->setParam('users.update', 1) + ->setParam('users.sessions.delete', 1) + ; + $response->noContent(); }); @@ -471,10 +525,12 @@ App::delete('/v1/users/:userId/sessions') ->inject('response') ->inject('dbForInternal') ->inject('events') - ->action(function ($userId, $response, $dbForInternal, $events) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $events, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -494,6 +550,10 @@ App::delete('/v1/users/:userId/sessions') ->setParam('eventData', $response->output($user, Response::MODEL_USER)) ; + $usage + ->setParam('users.update', 1) + ->setParam('users.sessions.delete', 1) + ; $response->noContent(); }); @@ -513,11 +573,13 @@ App::delete('/v1/users/:userId') ->inject('dbForInternal') ->inject('events') ->inject('deletes') - ->action(function ($userId, $response, $dbForInternal, $events, $deletes) { + ->inject('users') + ->action(function ($userId, $response, $dbForInternal, $events, $deletes, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $deletes */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -528,11 +590,6 @@ App::delete('/v1/users/:userId') if (!$dbForInternal->deleteDocument('users', $userId)) { throw new Exception('Failed to remove user from DB', 500); } - - // $dbForInternal->createDocument('users', new Document([ - // '$id' => $userId, - // '$read' => ['role:all'], - // ])); $deletes ->setParam('type', DELETE_TYPE_DOCUMENT) @@ -543,5 +600,8 @@ App::delete('/v1/users/:userId') ->setParam('eventData', $response->output($user, Response::MODEL_USER)) ; + $usage + ->setParam('users.delete', 1) + ; $response->noContent(); }); From 00a1612b2431967cae6388a195e5b5b3ff167e7b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 14:38:34 +0545 Subject: [PATCH 027/120] users and sessions stats --- app/controllers/api/account.php | 150 +++++++++++++++++++++++++++----- app/controllers/api/users.php | 2 +- app/tasks/usage.php | 83 ++++++------------ src/Appwrite/Stats/Stats.php | 36 +++++++- 4 files changed, 189 insertions(+), 82 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index ef08805117..b2baa4fcbe 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -52,12 +52,14 @@ App::post('/v1/account') ->inject('project') ->inject('dbForInternal') ->inject('audits') - ->action(function ($userId, $email, $password, $name, $request, $response, $project, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($userId, $email, $password, $name, $request, $response, $project, $dbForInternal, $audits, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Usage\Usage $usage */ $email = \strtolower($email); if ('console' === $project->getId()) { @@ -120,6 +122,9 @@ App::post('/v1/account') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.create', 1) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($user, Response::MODEL_USER); }); @@ -147,13 +152,15 @@ App::post('/v1/account/sessions') ->inject('locale') ->inject('geodb') ->inject('audits') - ->action(function ($email, $password, $request, $response, $dbForInternal, $locale, $geodb, $audits) { + ->inject('usage') + ->action(function ($email, $password, $request, $response, $dbForInternal, $locale, $geodb, $audits, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Locale\Locale $locale */ /** @var MaxMind\Db\Reader $geodb */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Usage\Usage $usage */ $email = \strtolower($email); $protocol = $request->getProtocol(); @@ -227,6 +234,11 @@ App::post('/v1/account/sessions') ->setAttribute('countryName', $countryName) ; + $usage + ->setParam('users.update', 1) + ->setParam('users.sessions.create', 1) + ->setParam('provider', 'email') + ; $response->dynamic($session, Response::MODEL_SESSION); }); @@ -357,7 +369,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') ->inject('geodb') ->inject('audits') ->inject('events') - ->action(function ($provider, $code, $state, $request, $response, $project, $user, $dbForInternal, $geodb, $audits, $events) use ($oauthDefaultSuccess) { + ->inject('usage') + ->action(function ($provider, $code, $state, $request, $response, $project, $user, $dbForInternal, $geodb, $audits, $events, $usage) use ($oauthDefaultSuccess) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ @@ -365,6 +378,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') /** @var Utopia\Database\Database $dbForInternal */ /** @var MaxMind\Db\Reader $geodb */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $callback = $protocol . '://' . $request->getHostname() . '/v1/account/sessions/oauth2/callback/' . $provider . '/' . $project->getId(); @@ -545,6 +559,11 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $events->setParam('eventData', $response->output($session, Response::MODEL_SESSION)); + $usage + ->setParam('users.sessions.create', 1) + ->setParam('projectId', $project->getId()) + ->setParam('provider', $provider) + ; if (!Config::getParam('domainVerification')) { $response ->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])) @@ -595,7 +614,8 @@ App::post('/v1/account/sessions/anonymous') ->inject('dbForInternal') ->inject('geodb') ->inject('audits') - ->action(function ($request, $response, $locale, $user, $project, $dbForInternal, $geodb, $audits) { + ->inject('usage') + ->action(function ($request, $response, $locale, $user, $project, $dbForInternal, $geodb, $audits, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Locale\Locale $locale */ @@ -604,6 +624,7 @@ App::post('/v1/account/sessions/anonymous') /** @var Utopia\Database\Database $dbForInternal */ /** @var MaxMind\Db\Reader $geodb */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); @@ -686,6 +707,11 @@ App::post('/v1/account/sessions/anonymous') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.sessions.create', 1) + ->setParam('provider', 'anonymous') + ; + if (!Config::getParam('domainVerification')) { $response ->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])) @@ -771,10 +797,15 @@ App::get('/v1/account') ->label('sdk.response.model', Response::MODEL_USER) ->inject('response') ->inject('user') - ->action(function ($response, $user) { + ->inject('usage') + ->action(function ($response, $user, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ + /** @var Appwrite\Stats\Stats $usage */ + $usage + ->setParam('users.read', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -791,12 +822,17 @@ App::get('/v1/account/prefs') ->label('sdk.response.model', Response::MODEL_PREFERENCES) ->inject('response') ->inject('user') - ->action(function ($response, $user) { + ->inject('usage') + ->action(function ($response, $user, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ + /** @var Appwrite\Stats\Stats $usage */ $prefs = $user->getAttribute('prefs', new \stdClass()); + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES); }); @@ -814,10 +850,12 @@ App::get('/v1/account/sessions') ->inject('response') ->inject('user') ->inject('locale') - ->action(function ($response, $user, $locale) { + ->inject('usage') + ->action(function ($response, $user, $locale, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Locale\Locale $locale */ + /** @var Appwrite\Stats\Stats $usage */ $sessions = $user->getAttribute('sessions', []); $current = Auth::sessionVerify($sessions, Auth::$secret); @@ -831,6 +869,9 @@ App::get('/v1/account/sessions') $sessions[$key] = $session; } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document([ 'sessions' => $sessions, 'sum' => count($sessions), @@ -853,13 +894,15 @@ App::get('/v1/account/logs') ->inject('locale') ->inject('geodb') ->inject('dbForInternal') - ->action(function ($response, $user, $locale, $geodb, $dbForInternal) { + ->inject('usage') + ->action(function ($response, $user, $locale, $geodb, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Locale\Locale $locale */ /** @var MaxMind\Db\Reader $geodb */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $audit = new Audit($dbForInternal); @@ -906,6 +949,9 @@ App::get('/v1/account/logs') } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document(['logs' => $output]), Response::MODEL_LOG_LIST); }); @@ -925,11 +971,13 @@ App::get('/v1/account/sessions/:sessionId') ->inject('user') ->inject('locale') ->inject('dbForInternal') - ->action(function ($sessionId, $response, $user, $locale, $dbForInternal) { + ->inject('usage') + ->action(function ($sessionId, $response, $user, $locale, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Locale\Locale $locale */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $sessions = $user->getAttribute('sessions', []); $sessionId = ($sessionId === 'current') @@ -948,6 +996,10 @@ App::get('/v1/account/sessions/:sessionId') ->setAttribute('countryName', $countryName) ; + $usage + ->setParam('users.read', 1) + ; + return $response->dynamic($session, Response::MODEL_SESSION); } } @@ -972,11 +1024,13 @@ App::patch('/v1/account/name') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($name, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($name, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('name', $name)); @@ -986,6 +1040,10 @@ App::patch('/v1/account/name') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; + $response->dynamic($user, Response::MODEL_USER); }); @@ -1007,11 +1065,13 @@ App::patch('/v1/account/password') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($password, $oldPassword, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($password, $oldPassword, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ // Check old password only if its an existing user. if ($user->getAttribute('passwordUpdate') !== 0 && !Auth::passwordVerify($oldPassword, $user->getAttribute('password'))) { // Double check user password @@ -1029,6 +1089,9 @@ App::patch('/v1/account/password') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -1050,11 +1113,13 @@ App::patch('/v1/account/email') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($email, $password, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($email, $password, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $isAnonymousUser = is_null($user->getAttribute('email')) && is_null($user->getAttribute('password')); // Check if request is from an anonymous account for converting @@ -1084,6 +1149,9 @@ App::patch('/v1/account/email') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -1104,11 +1172,13 @@ App::patch('/v1/account/prefs') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($prefs, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($prefs, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('prefs', $prefs)); @@ -1117,6 +1187,9 @@ App::patch('/v1/account/prefs') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -1137,13 +1210,15 @@ App::delete('/v1/account') ->inject('dbForInternal') ->inject('audits') ->inject('events') - ->action(function ($request, $response, $user, $dbForInternal, $audits, $events) { + ->inject('usage') + ->action(function ($request, $response, $user, $dbForInternal, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('status', false)); @@ -1173,6 +1248,9 @@ App::delete('/v1/account') ; } + $usage + ->setParam('users.delete', 1) + ; $response ->addCookie(Auth::$cookieName . '_legacy', '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) ->addCookie(Auth::$cookieName, '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) @@ -1200,7 +1278,8 @@ App::delete('/v1/account/sessions/:sessionId') ->inject('locale') ->inject('audits') ->inject('events') - ->action(function ($sessionId, $request, $response, $user, $dbForInternal, $locale, $audits, $events) { + ->inject('usage') + ->action(function ($sessionId, $request, $response, $user, $dbForInternal, $locale, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ @@ -1208,6 +1287,7 @@ App::delete('/v1/account/sessions/:sessionId') /** @var Utopia\Locale\Locale $locale */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $sessionId = ($sessionId === 'current') @@ -1254,6 +1334,10 @@ App::delete('/v1/account/sessions/:sessionId') ->setParam('eventData', $response->output($session, Response::MODEL_SESSION)) ; + $usage + ->setParam('users.sessions.delete', 1) + ->setParam('users.update', 1) + ; return $response->noContent(); } } @@ -1280,7 +1364,8 @@ App::delete('/v1/account/sessions') ->inject('locale') ->inject('audits') ->inject('events') - ->action(function ($request, $response, $user, $dbForInternal, $locale, $audits, $events) { + ->inject('usage') + ->action(function ($request, $response, $user, $dbForInternal, $locale, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ @@ -1288,6 +1373,7 @@ App::delete('/v1/account/sessions') /** @var Utopia\Locale\Locale $locale */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $sessions = $user->getAttribute('sessions', []); @@ -1330,6 +1416,10 @@ App::delete('/v1/account/sessions') ]), Response::MODEL_SESSION_LIST)) ; + $usage + ->setParam('users.sessions.delete', 1) + ->setParam('users.update', 1) + ; $response->noContent(); }); @@ -1357,7 +1447,8 @@ App::post('/v1/account/recovery') ->inject('mails') ->inject('audits') ->inject('events') - ->action(function ($email, $url, $request, $response, $dbForInternal, $project, $locale, $mails, $audits, $events) { + ->inject('usage') + ->action(function ($email, $url, $request, $response, $dbForInternal, $project, $locale, $mails, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -1366,6 +1457,7 @@ App::post('/v1/account/recovery') /** @var Appwrite\Event\Event $mails */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::$roles); $isAppUser = Auth::isAppUser(Authorization::$roles); @@ -1433,6 +1525,9 @@ App::post('/v1/account/recovery') ->setParam('resource', 'users/' . $profile->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($recovery, Response::MODEL_TOKEN); }); @@ -1458,10 +1553,12 @@ App::put('/v1/account/recovery') ->inject('response') ->inject('dbForInternal') ->inject('audits') - ->action(function ($userId, $secret, $password, $passwordAgain, $response, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($userId, $secret, $password, $passwordAgain, $response, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ if ($password !== $passwordAgain) { throw new Exception('Passwords must match', 400); @@ -1508,6 +1605,9 @@ App::put('/v1/account/recovery') ->setParam('resource', 'users/' . $profile->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($recovery, Response::MODEL_TOKEN); }); @@ -1535,7 +1635,8 @@ App::post('/v1/account/verification') ->inject('audits') ->inject('events') ->inject('mails') - ->action(function ($url, $request, $response, $project, $user, $dbForInternal, $locale, $audits, $events, $mails) { + ->inject('usage') + ->action(function ($url, $request, $response, $project, $user, $dbForInternal, $locale, $audits, $events, $mails, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ @@ -1545,6 +1646,7 @@ App::post('/v1/account/verification') /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $mails */ + /** @var Appwrite\Stats\Stats $usage */ $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::$roles); $isAppUser = Auth::isAppUser(Authorization::$roles); @@ -1602,6 +1704,9 @@ App::post('/v1/account/verification') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($verification, Response::MODEL_TOKEN); }); @@ -1626,11 +1731,13 @@ App::put('/v1/account/verification') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($userId, $secret, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($userId, $secret, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $profile = $dbForInternal->getDocument('users', $userId); @@ -1668,5 +1775,8 @@ App::put('/v1/account/verification') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($verification, Response::MODEL_TOKEN); }); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index eb3abd12cf..ad600ce480 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -573,7 +573,7 @@ App::delete('/v1/users/:userId') ->inject('dbForInternal') ->inject('events') ->inject('deletes') - ->inject('users') + ->inject('usage') ->action(function ($userId, $response, $dbForInternal, $events, $deletes, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 5230963098..fc8d0478c1 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -11,37 +11,8 @@ use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; -/** - * 1. Load all the projects - * 2. Load latest data entered entered for each project, for each period - * 3. Start the loop - * 4. Fore each project, for each metric, for each period - sync data - */ - -/** - * Only succefull operations - * - * database.collections.CRUD (project=x) - * database.documents.CRUD (project=x,collection=y) - * - * users.CRUD - * users.sessions.create (project=x,provider=y) - * users.sessions.delete (project=x,provider=y) - * - * storage.buckets.CRUD (project=x) - * storage.files.CRUD (project=x,bucket=y) - * - * refactor later - * - functions - * - realtime - * - teams - * - webhooks - * - keys - really later! - */ - $cli ->task('usage') ->desc('Schedules syncing data from influxdb to Appwrite console db') @@ -61,93 +32,91 @@ $cli ], ]; - //use projectId from influxdb instead of iterating over projects from DB - $globalMetrics = [ 'requests' => [ - 'method' => 'getGlobalMetrics', 'table' => 'appwrite_usage_requests_all', ], 'network' => [ - 'method' => 'getGlobalMetrics', 'table' => 'appwrite_usage_network_all', ], 'executions' => [ - 'method' => 'getGlobalMetrics', 'table' => 'appwrite_usage_executions_all', ], 'database.collections.create' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_collections_create', ], 'database.collections.read' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_collections_read', ], 'database.collections.update' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_collections_update', ], 'database.collections.delete' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_collections_delete', ], 'database.documents.create' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_create', ], 'database.documents.read' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_read', ], 'database.documents.update' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_update', ], 'database.documents.delete' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_delete', ], 'database.documents.collectionId.create' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_create', 'groupBy' => 'collectionId', ], 'database.documents.collectionId.read' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_read', 'groupBy' => 'collectionId', ], 'database.documents.collectionId.update' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_update', 'groupBy' => 'collectionId', ], 'database.documents.collectionId.delete' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_delete', 'groupBy' => 'collectionId', ], 'storage.buckets.bucketId.files.create' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_storage_files_create', 'groupBy' => 'bucketId', ], 'storage.buckets.bucketId.files.read' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_storage_files_read', 'groupBy' => 'bucketId', ], 'storage.buckets.bucketId.files.update' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_storage_files_update', 'groupBy' => 'bucketId', ], 'storage.buckets.bucketId.files.delete' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_storage_files_delete', 'groupBy' => 'bucketId', ], + 'users.create' => [ + 'table' => 'appwrite_usage_users_create', + ], + 'users.read' => [ + 'table' => 'appwrite_usage_users_read', + ], + 'users.update' => [ + 'table' => 'appwrite_usage_users_update', + ], + 'users.delete' => [ + 'table' => 'appwrite_usage_users_delete', + ], + 'users.sessions.create' => [ + 'table' => 'appwrite_usage_users_sessions_create', + 'groupBy' => 'provider', + ], + 'users.sessions.delete' => [ + 'table' => 'appwrite_usage_users_sessions_delete', + ], ]; $attempts = 0; @@ -169,18 +138,16 @@ $cli } while ($attempts < $max); $cacheAdapter = new Cache(new Redis($redis)); - $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); - $dbForConsole->setNamespace('project_console_internal'); $dbForProject = new Database(new MariaDB($db), $cacheAdapter); Authorization::disable(); - $latestData = []; - $firstRun = true; - Console::loop(function () use ($interval, $register, &$latestData, $dbForProject, $dbForConsole, &$firstRun, $globalMetrics, $periods) { + Console::loop(function () use ($interval, $register, $dbForProject, $globalMetrics, $periods) { $time = date('d-m-Y H:i:s', time()); Console::info("[{$time}] Aggregating usage data every {$interval} seconds"); + $loopStart = microtime(true); + $client = $register->get('influxdb'); if ($client) { $database = $client->selectDB('telegraf'); @@ -225,7 +192,6 @@ $cli $dbForProject->updateDocument('stats', $document->getId(), $document->setAttribute('value', $value)); } - $latestData[$projectId][$metric][$period['key']] = $time; } catch (\Exception$e) { Console::warning("Failed to save data for project {$projectId} and metric {$metric}"); } @@ -234,6 +200,9 @@ $cli } } } - $firstRun = false; + + $loopTook = microtime(true) - $loopStart; + $time = date('d-m-Y H:i:s', time()); + Console::info("[{$time}] Aggregation took {$loopTook} seconds"); }, $interval); }); diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index 5c24e1ae94..f56f6a4e7e 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -126,8 +126,8 @@ class Stats foreach ($dbMetrics as $metric) { $value = $this->params[$metric] ?? 0; if ($value >= 1) { - $dbTags = ",projectId={$projectId},collectionId=" . ($this->params['collectionId'] ?? ''); - $this->statsd->increment($metric . $dbTags); + $tags = ",projectId={$projectId},collectionId=" . ($this->params['collectionId'] ?? ''); + $this->statsd->increment($metric . $tags); } } @@ -141,8 +141,36 @@ class Stats foreach ($storageMertics as $metric) { $value = $this->params[$metric] ?? 0; if ($value >= 1) { - $storageTags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? ''); - $this->statsd->increment($metric . $storageTags); + $tags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? ''); + $this->statsd->increment($metric . $tags); + } + } + + $usersMetrics = [ + 'users.create', + 'users.read', + 'users.update', + 'users.delete', + ]; + + foreach ($usersMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $tags = ",projectId={$projectId}"; + $this->statsd->increment($metric . $tags); + } + } + + $sessionsMetrics = [ + 'users.sessions.create', + 'users.sessions.delete', + ]; + + foreach ($sessionsMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $tags = ",projectId={$projectId},provider=". ($this->params['provider'] ?? ''); + $this->statsd->increment($metric . $tags); } } From 22f0aaf161f270adff600009a07c66bc347ec674 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 14:47:35 +0545 Subject: [PATCH 028/120] improvement with latest data --- app/tasks/usage.php | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index fc8d0478c1..e5b8ac509c 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -140,11 +140,13 @@ $cli $cacheAdapter = new Cache(new Redis($redis)); $dbForProject = new Database(new MariaDB($db), $cacheAdapter); + $latestTime = []; + Authorization::disable(); - Console::loop(function () use ($interval, $register, $dbForProject, $globalMetrics, $periods) { - $time = date('d-m-Y H:i:s', time()); - Console::info("[{$time}] Aggregating usage data every {$interval} seconds"); + Console::loop(function () use ($interval, $register, $dbForProject, $globalMetrics, $periods, &$latestTime) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating usage data every {$interval} seconds"); $loopStart = microtime(true); @@ -155,6 +157,9 @@ $cli foreach ($globalMetrics as $metric => $options) { foreach ($periods as $period) { $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339); + if(!empty($latestTime[$metric][$period['key']])) { + $start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339); + } $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); $table = $options['table']; @@ -192,6 +197,7 @@ $cli $dbForProject->updateDocument('stats', $document->getId(), $document->setAttribute('value', $value)); } + $latestTime[$metric][$period['key']] = $time; } catch (\Exception$e) { Console::warning("Failed to save data for project {$projectId} and metric {$metric}"); } @@ -202,7 +208,7 @@ $cli } $loopTook = microtime(true) - $loopStart; - $time = date('d-m-Y H:i:s', time()); - Console::info("[{$time}] Aggregation took {$loopTook} seconds"); + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); }); From 13a91907b436f43d3532cbccb041c7f1bce8ceb5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 18:07:54 +0545 Subject: [PATCH 029/120] fix db metric name --- app/tasks/usage.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index e5b8ac509c..f8cf70070d 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -66,19 +66,19 @@ $cli 'database.documents.delete' => [ 'table' => 'appwrite_usage_database_documents_delete', ], - 'database.documents.collectionId.create' => [ + 'database.collections.collectionId.documents.create' => [ 'table' => 'appwrite_usage_database_documents_create', 'groupBy' => 'collectionId', ], - 'database.documents.collectionId.read' => [ + 'database.collections.collectionId.documents.read' => [ 'table' => 'appwrite_usage_database_documents_read', 'groupBy' => 'collectionId', ], - 'database.documents.collectionId.update' => [ + 'database.collections.collectionId.documents.update' => [ 'table' => 'appwrite_usage_database_documents_update', 'groupBy' => 'collectionId', ], - 'database.documents.collectionId.delete' => [ + 'database.collections.collectionId.documents.delete' => [ 'table' => 'appwrite_usage_database_documents_delete', 'groupBy' => 'collectionId', ], From 3e0cc30bddc622e8eb980f4e327f504c6b9278b0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 18:16:49 +0545 Subject: [PATCH 030/120] some comments and small fixes --- app/controllers/api/account.php | 2 +- app/tasks/usage.php | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index b2baa4fcbe..0f357e045c 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -562,7 +562,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $usage ->setParam('users.sessions.create', 1) ->setParam('projectId', $project->getId()) - ->setParam('provider', $provider) + ->setParam('provider', 'oauth2-'.$provider) ; if (!Config::getParam('domainVerification')) { $response diff --git a/app/tasks/usage.php b/app/tasks/usage.php index f8cf70070d..7c821b2306 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -17,8 +17,8 @@ $cli ->task('usage') ->desc('Schedules syncing data from influxdb to Appwrite console db') ->action(function () use ($register) { - Console::title('Usage Sync V1'); - Console::success(APP_NAME . ' usage sync process v1 has started'); + Console::title('Usage Aggregation V1'); + Console::success(APP_NAME . ' usage aggregation process v1 has started'); $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); //30 seconds $periods = [ @@ -32,6 +32,7 @@ $cli ], ]; + // all the metrics that we are collecting at the moment $globalMetrics = [ 'requests' => [ 'table' => 'appwrite_usage_requests_all', @@ -154,19 +155,18 @@ $cli if ($client) { $database = $client->selectDB('telegraf'); // sync data - foreach ($globalMetrics as $metric => $options) { - foreach ($periods as $period) { + foreach ($globalMetrics as $metric => $options) { //for each metrics + foreach ($periods as $period) { // aggregate data for each period $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339); if(!empty($latestTime[$metric][$period['key']])) { $start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339); } $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); - $table = $options['table']; - $groupBy = $options['groupBy'] ?? ''; + $table = $options['table']; //which influxdb table to query for this metric + $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //some sub level metrics may be grouped by other tags like collectionId, bucketId, etc - $query = 'SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . $period['key'] . '), "projectId"' . (empty($groupBy) ? '' : ', "' . $groupBy . '"') . ' FILL(null)'; - $result = $database->query($query); + $result = $database->query('SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . $period['key'] . '), "projectId"' . $groupBy . ' FILL(null)'); $points = $result->getPoints(); foreach ($points as $point) { $projectId = $point['projectId']; @@ -180,7 +180,7 @@ $cli $metric = str_replace($groupBy, $groupedBy, $metric); } $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period['key'] . '_' . $metric); + $id = \md5($time . '_' . $period['key'] . '_' . $metric); //construct unique id for each metric using time, period and metric $value = (!empty($point['value'])) ? $point['value'] : 0; try { $document = $dbForProject->getDocument('stats', $id); @@ -198,7 +198,8 @@ $cli $document->setAttribute('value', $value)); } $latestTime[$metric][$period['key']] = $time; - } catch (\Exception$e) { + } catch (\Exception $e) { + // if projects are deleted this might fail Console::warning("Failed to save data for project {$projectId} and metric {$metric}"); } } From 62e422b76ef1a58f59bc26b707f99827208a5736 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 19:12:15 +0545 Subject: [PATCH 031/120] project usage from appwrite db (wip) --- app/controllers/api/projects.php | 301 ++++++++++++++++++------------- 1 file changed, 178 insertions(+), 123 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index c302e882b3..950a4df0c5 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -5,13 +5,15 @@ use Appwrite\Database\Validator\CustomId; use Appwrite\Network\Validator\CNAME; use Appwrite\Network\Validator\Domain as DomainValidator; use Appwrite\Network\Validator\URL; -use Appwrite\Task\Validator\Cron; use Appwrite\Utopia\Response; -use Cron\CronExpression; +use Utopia\Abuse\Adapters\TimeLimit; use Utopia\App; +use Utopia\Audit\Audit; use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Domains\Domain; use Utopia\Exception; @@ -21,13 +23,11 @@ use Utopia\Validator\Integer; use Utopia\Validator\Range; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; -use Utopia\Audit\Audit; -use Utopia\Abuse\Adapters\TimeLimit; App::init(function ($project) { /** @var Utopia\Database\Document $project */ - if($project->getId() !== 'console') { + if ($project->getId() !== 'console') { throw new Exception('Access to this API is forbidden.', 401); } }, ['project'], 'projects'); @@ -71,7 +71,7 @@ App::post('/v1/projects') if ($team->isEmpty()) { throw new Exception('Team not found', 404); } - + $auth = Config::getParam('auth', []); $auths = ['limit' => 0]; foreach ($auth as $index => $method) { @@ -228,12 +228,12 @@ App::get('/v1/projects/:projectId/usage') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForConsole') - ->inject('projectDB') + ->inject('dbForInternal') ->inject('register') - ->action(function ($projectId, $range, $response, $dbForConsole, $projectDB, $register) { + ->action(function ($projectId, $range, $response, $dbForConsole, $dbForInternal, $register) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ - /** @var Appwrite\Database\Database $projectDB */ + /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Registry\Registry $register */ $project = $dbForConsole->getDocument('projects', $projectId); @@ -246,71 +246,126 @@ App::get('/v1/projects/:projectId/usage') $period = [ '24h' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-24 hours')), - 'end' => DateTime::createFromFormat('U', \strtotime('+1 hour')), - 'group' => '30m', + // 'start' => DateTime::createFromFormat('U', \strtotime('-24 hours')), + // 'end' => DateTime::createFromFormat('U', \strtotime('+1 hour')), + 'period' => '30m', + 'limit' => 48, ], '7d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-7 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + // 'start' => DateTime::createFromFormat('U', \strtotime('-7 days')), + // 'end' => DateTime::createFromFormat('U', \strtotime('now')), + 'period' => '1d', + 'limit' => 7, ], '30d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-30 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + // 'start' => DateTime::createFromFormat('U', \strtotime('-30 days')), + // 'end' => DateTime::createFromFormat('U', \strtotime('now')), + 'period' => '1d', + 'limit' => 30, ], '90d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-90 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + // 'start' => DateTime::createFromFormat('U', \strtotime('-90 days')), + // 'end' => DateTime::createFromFormat('U', \strtotime('now')), + 'period' => '1d', + 'limit' => 90, ], ]; - $client = $register->get('influxdb'); + $dbForInternal->setNamespace('project_' . $projectId . '_internal'); + + $requestDocs = Authorization::skip(function () use ($dbForInternal, $period, $range) { + return $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, ['requests']), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + }); $requests = []; - $network = []; - $functions = []; - - if ($client) { - $start = $period[$range]['start']->format(DateTime::RFC3339); - $end = $period[$range]['end']->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); - - // Requests - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $requests[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Network - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $network[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Functions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $functions[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } + foreach ($requestDocs as $requestDoc) { + $requests[] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; } + + $requests = array_reverse($requests); + + $networkDocs = Authorization::skip(function () use ($dbForInternal, $period, $range) { + return $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, ['network']), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + }); + + $network = []; + foreach ($networkDocs as $networkDoc) { + $network[] = [ + 'value' => $networkDoc->getAttribute('value'), + 'date' => $networkDoc->getAttribute('time'), + ]; + } + + $network = array_reverse($network); + + $functionsDocs = Authorization::skip(function () use ($dbForInternal, $period, $range) { + return $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, ['executions']), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + }); + + $functions = []; + foreach ($functionsDocs as $functionDoc) { + $functions[] = [ + 'value' => $functionDoc->getAttribute('value'), + 'date' => $functionDoc->getAttribute('time'), + ]; + } + $functions = array_reverse($functions); + + // $requests = []; + // $network = []; + // $functions = []; + + // $client = $register->get('influxdb'); + // if ($client) { + // $start = $period[$range]['start']->format(DateTime::RFC3339); + // $end = $period[$range]['end']->format(DateTime::RFC3339); + // $database = $client->selectDB('telegraf'); + + // // Requests + // $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); + // $points = $result->getPoints(); + + // foreach ($points as $point) { + // $requests[] = [ + // 'value' => (!empty($point['value'])) ? $point['value'] : 0, + // 'date' => \strtotime($point['time']), + // ]; + // } + + // // Network + // $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); + // $points = $result->getPoints(); + + // foreach ($points as $point) { + // $network[] = [ + // 'value' => (!empty($point['value'])) ? $point['value'] : 0, + // 'date' => \strtotime($point['time']), + // ]; + // } + + // // Functions + // $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); + // $points = $result->getPoints(); + + // foreach ($points as $point) { + // $functions[] = [ + // 'value' => (!empty($point['value'])) ? $point['value'] : 0, + // 'date' => \strtotime($point['time']), + // ]; + // } + // } } else { $requests = []; $network = []; @@ -319,41 +374,41 @@ App::get('/v1/projects/:projectId/usage') // Users - $projectDB->getCollection([ - 'limit' => 0, - 'offset' => 0, - 'filters' => [ - '$collection=users', - ], - ]); + // $projectDB->getCollection([ + // 'limit' => 0, + // 'offset' => 0, + // 'filters' => [ + // '$collection=users', + // ], + // ]); - $usersTotal = $projectDB->getSum(); + // $usersTotal = $projectDB->getSum(); - // Documents + // // Documents - $collections = $projectDB->getCollection([ - 'limit' => 100, - 'offset' => 0, - 'filters' => [ - '$collection=collections', - ], - ]); + // $collections = $projectDB->getCollection([ + // 'limit' => 100, + // 'offset' => 0, + // 'filters' => [ + // '$collection=collections', + // ], + // ]); - $collectionsTotal = $projectDB->getSum(); + // $collectionsTotal = $projectDB->getSum(); - $documents = []; + // $documents = []; - foreach ($collections as $collection) { - $result = $projectDB->getCollection([ - 'limit' => 0, - 'offset' => 0, - 'filters' => [ - '$collection=' . $collection['$id'], - ], - ]); + // foreach ($collections as $collection) { + // $result = $projectDB->getCollection([ + // 'limit' => 0, + // 'offset' => 0, + // 'filters' => [ + // '$collection=' . $collection['$id'], + // ], + // ]); - $documents[] = ['name' => $collection['name'], 'total' => $projectDB->getSum()]; - } + // $documents[] = ['name' => $collection['name'], 'total' => $projectDB->getSum()]; + // } $response->json([ 'range' => $range, @@ -375,38 +430,38 @@ App::get('/v1/projects/:projectId/usage') return $item['value']; }, $functions)), ], - 'collections' => [ - 'data' => $collections, - 'total' => $collectionsTotal, - ], - 'documents' => [ - 'data' => $documents, - 'total' => \array_sum(\array_map(function ($item) { - return $item['total']; - }, $documents)), - ], - 'users' => [ - 'data' => [], - 'total' => $usersTotal, - ], - 'storage' => [ - 'total' => $projectDB->getCount( - [ - 'attribute' => 'sizeOriginal', - 'filters' => [ - '$collection=files', - ], - ] - ) + - $projectDB->getCount( - [ - 'attribute' => 'size', - 'filters' => [ - '$collection=tags', - ], - ] - ), - ], + // 'collections' => [ + // 'data' => $collections, + // 'total' => $collectionsTotal, + // ], + // 'documents' => [ + // 'data' => $documents, + // 'total' => \array_sum(\array_map(function ($item) { + // return $item['total']; + // }, $documents)), + // ], + // 'users' => [ + // 'data' => [], + // 'total' => $usersTotal, + // ], + // 'storage' => [ + // 'total' => $projectDB->getCount( + // [ + // 'attribute' => 'sizeOriginal', + // 'filters' => [ + // '$collection=files', + // ], + // ] + // ) + + // $projectDB->getCount( + // [ + // 'attribute' => 'size', + // 'filters' => [ + // '$collection=tags', + // ], + // ] + // ), + // ], ]); }); @@ -470,7 +525,7 @@ App::patch('/v1/projects/:projectId/service') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_PROJECT) ->param('projectId', '', new UID(), 'Project unique ID.') - ->param('service', '', new WhiteList(array_keys(array_filter(Config::getParam('services'), function($element) {return $element['optional'];})), true), 'Service name.') + ->param('service', '', new WhiteList(array_keys(array_filter(Config::getParam('services'), function ($element) {return $element['optional'];})), true), 'Service name.') ->param('status', null, new Boolean(), 'Service status.') ->inject('response') ->inject('dbForConsole') From 354633afa8275392ed7e57963b484b1388a21e09 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Aug 2021 10:47:31 +0545 Subject: [PATCH 032/120] fix usage stats --- src/Appwrite/Stats/Stats.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index f56f6a4e7e..2d5a5cb16d 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -175,6 +175,7 @@ class Stats } if ($storage >= 1) { + $tags = ",projectId={$projectId},bucketId={($this->params['bucketId'] ?? '')}"; $this->statsd->count('storage.all' . $tags, $storage); } From 0d73ac953dc15500a57968e14f52a2a0a4851d9d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Aug 2021 11:08:43 +0545 Subject: [PATCH 033/120] count total number of sessions deleted --- app/controllers/api/account.php | 6 ++++-- src/Appwrite/Stats/Stats.php | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 0f357e045c..9674d668e4 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1409,15 +1409,17 @@ App::delete('/v1/account/sessions') $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('sessions', [])); + $numOfSessions = count($sessions); + $events ->setParam('eventData', $response->output(new Document([ 'sessions' => $sessions, - 'sum' => count($sessions), + 'sum' => $numOfSessions, ]), Response::MODEL_SESSION_LIST)) ; $usage - ->setParam('users.sessions.delete', 1) + ->setParam('users.sessions.delete', $numOfSessions) ->setParam('users.update', 1) ; $response->noContent(); diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index 2d5a5cb16d..e7720088f5 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -170,7 +170,7 @@ class Stats $value = $this->params[$metric] ?? 0; if ($value >= 1) { $tags = ",projectId={$projectId},provider=". ($this->params['provider'] ?? ''); - $this->statsd->increment($metric . $tags); + $this->statsd->count($metric . $tags, $value); } } From 583c447b4e345c3b76b04ee5a0eb5006648bd85c Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Aug 2021 11:30:07 +0545 Subject: [PATCH 034/120] database object count aggregation --- app/tasks/usage.php | 47 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 7c821b2306..34e7dcb2d6 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -11,6 +11,7 @@ use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; $cli @@ -158,7 +159,7 @@ $cli foreach ($globalMetrics as $metric => $options) { //for each metrics foreach ($periods as $period) { // aggregate data for each period $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339); - if(!empty($latestTime[$metric][$period['key']])) { + if (!empty($latestTime[$metric][$period['key']])) { $start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339); } $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); @@ -198,7 +199,7 @@ $cli $document->setAttribute('value', $value)); } $latestTime[$metric][$period['key']] = $time; - } catch (\Exception $e) { + } catch (\Exception$e) { // if projects are deleted this might fail Console::warning("Failed to save data for project {$projectId} and metric {$metric}"); } @@ -212,4 +213,46 @@ $cli $now = date('d-m-Y H:i:s', time()); Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); + + //aggregate number of objects in database + $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); + $dbForConsole->setNamespace('project_console_internal'); + + $dbCountInterval = 15*60; //aggregate data every 15 minutes + Console::loop(function () use ($dbForConsole, $dbForProject, $dbCountInterval) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating Database object count every {$dbCountInterval} seconds"); + $loopStart = microtime(true); + + $latestProject = null; + do { + $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + if (!empty($projects)) { + $latestProject = $projects[array_key_last($projects)]; + + foreach ($projects as $project) { + $id = $project->getId(); + $dbForProject->setNamespace("project_{$id}_internal"); + $collections = ['users', 'collections', 'files']; + foreach ($collections as $collection) { + $count = $dbForProject->count($collection); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => "{$collection}.count", + 'value' => $count, + 'type' => 1, + ])); + } + } + } + + } while (!empty($projects)); + + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] DB objects count aggregation took {$loopTook} seconds"); + + }, $dbCountInterval); }); From eeee0f712bcd4080de016cacead2318c0029ea82 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Aug 2021 11:48:27 +0545 Subject: [PATCH 035/120] refactor db object counts --- app/tasks/usage.php | 85 ++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 34e7dcb2d6..afe5958406 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -141,12 +141,15 @@ $cli $cacheAdapter = new Cache(new Redis($redis)); $dbForProject = new Database(new MariaDB($db), $cacheAdapter); + $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); + $dbForConsole->setNamespace('project_console_internal'); $latestTime = []; Authorization::disable(); - Console::loop(function () use ($interval, $register, $dbForProject, $globalMetrics, $periods, &$latestTime) { + $iterations = 0; + Console::loop(function () use ($interval, $register, $dbForProject, $dbForConsole, $globalMetrics, $periods, &$latestTime, &$iterations) { $now = date('d-m-Y H:i:s', time()); Console::info("[{$now}] Aggregating usage data every {$interval} seconds"); @@ -174,7 +177,7 @@ $cli if (!empty($projectId) && $projectId != 'console') { $dbForProject->setNamespace('project_' . $projectId . '_internal'); if (!empty($groupBy)) { - $groupedBy = $point[$groupBy]; + $groupedBy = $point[$groupBy] ?? ''; if (empty($groupedBy)) { continue; } @@ -209,50 +212,44 @@ $cli } } + if ($iterations % 30 == 0) { + //aggregate number of objects in database + $latestProject = null; + do { + $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + if (!empty($projects)) { + $latestProject = $projects[array_key_last($projects)]; + + foreach ($projects as $project) { + $id = $project->getId(); + $collections = ['users' => [ + 'namespace' => 'internal', + ], 'collections' => [ + 'namespace' => 'external', + ], 'files' => [ + 'namespace' => 'internal', + ]]; + foreach ($collections as $collection => $options) { + $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $count = $dbForProject->count($collection); + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => "{$collection}.count", + 'value' => $count, + 'type' => 1, + ])); + } + } + } + + } while (!empty($projects)); + } + $iterations++; $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); - - //aggregate number of objects in database - $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); - $dbForConsole->setNamespace('project_console_internal'); - - $dbCountInterval = 15*60; //aggregate data every 15 minutes - Console::loop(function () use ($dbForConsole, $dbForProject, $dbCountInterval) { - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregating Database object count every {$dbCountInterval} seconds"); - $loopStart = microtime(true); - - $latestProject = null; - do { - $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); - if (!empty($projects)) { - $latestProject = $projects[array_key_last($projects)]; - - foreach ($projects as $project) { - $id = $project->getId(); - $dbForProject->setNamespace("project_{$id}_internal"); - $collections = ['users', 'collections', 'files']; - foreach ($collections as $collection) { - $count = $dbForProject->count($collection); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => "{$collection}.count", - 'value' => $count, - 'type' => 1, - ])); - } - } - } - - } while (!empty($projects)); - - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] DB objects count aggregation took {$loopTook} seconds"); - - }, $dbCountInterval); }); From ee048caf93a17cba2f88bce560e0e9e0867de653 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Aug 2021 12:18:32 +0545 Subject: [PATCH 036/120] fix usage small issues --- app/controllers/api/storage.php | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 651985057a..7daeee1546 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -631,16 +631,13 @@ App::delete('/v1/storage/files/:fileId') $usage ->setParam('storage', $file->getAttribute('size', 0) * -1) + ->setParam('storage.files.delete', 1) + ->setParam('bucketId', 'default') ; $events ->setParam('eventData', $response->output($file, Response::MODEL_FILE)) ; - $usage - ->setParam('storage.files.delete', 1) - ->setParam('bucketId', 'default') - ; - $response->noContent(); }); \ No newline at end of file From 4bb911867c1fe28542b4194ac500e926828d096b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Aug 2021 12:18:51 +0545 Subject: [PATCH 037/120] more stats from db --- app/controllers/api/projects.php | 100 ++++++++----------------------- 1 file changed, 26 insertions(+), 74 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 950a4df0c5..cf337a83e2 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -322,79 +322,22 @@ App::get('/v1/projects/:projectId/usage') ]; } $functions = array_reverse($functions); - - // $requests = []; - // $network = []; - // $functions = []; - - // $client = $register->get('influxdb'); - // if ($client) { - // $start = $period[$range]['start']->format(DateTime::RFC3339); - // $end = $period[$range]['end']->format(DateTime::RFC3339); - // $database = $client->selectDB('telegraf'); - - // // Requests - // $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - // $points = $result->getPoints(); - - // foreach ($points as $point) { - // $requests[] = [ - // 'value' => (!empty($point['value'])) ? $point['value'] : 0, - // 'date' => \strtotime($point['time']), - // ]; - // } - - // // Network - // $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - // $points = $result->getPoints(); - - // foreach ($points as $point) { - // $network[] = [ - // 'value' => (!empty($point['value'])) ? $point['value'] : 0, - // 'date' => \strtotime($point['time']), - // ]; - // } - - // // Functions - // $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - // $points = $result->getPoints(); - - // foreach ($points as $point) { - // $functions[] = [ - // 'value' => (!empty($point['value'])) ? $point['value'] : 0, - // 'date' => \strtotime($point['time']), - // ]; - // } - // } } else { $requests = []; $network = []; $functions = []; } - // Users + $usersCount = Authorization::skip(function () use ($dbForInternal) { + return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); + }); + $usersTotal = $usersCount ? $usersCount->getAttribute('value', 0) : 0; - // $projectDB->getCollection([ - // 'limit' => 0, - // 'offset' => 0, - // 'filters' => [ - // '$collection=users', - // ], - // ]); - - // $usersTotal = $projectDB->getSum(); - - // // Documents - - // $collections = $projectDB->getCollection([ - // 'limit' => 100, - // 'offset' => 0, - // 'filters' => [ - // '$collection=collections', - // ], - // ]); - - // $collectionsTotal = $projectDB->getSum(); + $collectionsCount = Authorization::skip(function () use ($dbForInternal, $period, $range) { + return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['collections.count'])], 0, ['time'], [Database::ORDER_DESC]); + }); + $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; + // $documents = []; @@ -410,6 +353,11 @@ App::get('/v1/projects/:projectId/usage') // $documents[] = ['name' => $collection['name'], 'total' => $projectDB->getSum()]; // } + $filesCount = Authorization::skip(function () use ($dbForInternal, $period, $range) { + return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['files.count'])], 0, ['time'], [Database::ORDER_DESC]); + }); + $filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0; + $response->json([ 'range' => $range, 'requests' => [ @@ -430,20 +378,24 @@ App::get('/v1/projects/:projectId/usage') return $item['value']; }, $functions)), ], - // 'collections' => [ - // 'data' => $collections, - // 'total' => $collectionsTotal, - // ], + 'collections' => [ + 'data' => [], + 'total' => $collectionsTotal, + ], + 'files' => [ + 'data' => [], + 'total' => $filesTotal, + ], // 'documents' => [ // 'data' => $documents, // 'total' => \array_sum(\array_map(function ($item) { // return $item['total']; // }, $documents)), // ], - // 'users' => [ - // 'data' => [], - // 'total' => $usersTotal, - // ], + 'users' => [ + 'data' => [], + 'total' => $usersTotal, + ], // 'storage' => [ // 'total' => $projectDB->getCount( // [ From e82cf42fe30727f80dafa1345f4e7f9fbc49a5a8 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 12:23:56 +0545 Subject: [PATCH 038/120] refactor usage endpoint --- app/controllers/api/projects.php | 89 +++++++++----------------------- 1 file changed, 24 insertions(+), 65 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index f8fe0e8a96..f660bf2f60 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -241,30 +241,22 @@ App::get('/v1/projects/:projectId/usage') throw new Exception('Project not found', 404); } + $stats = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $period = [ '24h' => [ - // 'start' => DateTime::createFromFormat('U', \strtotime('-24 hours')), - // 'end' => DateTime::createFromFormat('U', \strtotime('+1 hour')), 'period' => '30m', 'limit' => 48, ], '7d' => [ - // 'start' => DateTime::createFromFormat('U', \strtotime('-7 days')), - // 'end' => DateTime::createFromFormat('U', \strtotime('now')), 'period' => '1d', 'limit' => 7, ], '30d' => [ - // 'start' => DateTime::createFromFormat('U', \strtotime('-30 days')), - // 'end' => DateTime::createFromFormat('U', \strtotime('now')), 'period' => '1d', 'limit' => 30, ], '90d' => [ - // 'start' => DateTime::createFromFormat('U', \strtotime('-90 days')), - // 'end' => DateTime::createFromFormat('U', \strtotime('now')), 'period' => '1d', 'limit' => 90, ], @@ -272,59 +264,25 @@ App::get('/v1/projects/:projectId/usage') $dbForInternal->setNamespace('project_' . $projectId . '_internal'); - $requestDocs = Authorization::skip(function () use ($dbForInternal, $period, $range) { - return $dbForInternal->find('stats', [ + Authorization::disable(); + + $metrics = ['requests', 'network', 'executions']; + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, ['requests']), + new Query('metric', Query::TYPE_EQUAL, [$metric]), ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - }); - $requests = []; - foreach ($requestDocs as $requestDoc) { - $requests[] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); } - - $requests = array_reverse($requests); - - $networkDocs = Authorization::skip(function () use ($dbForInternal, $period, $range) { - return $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, ['network']), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - }); - - $network = []; - foreach ($networkDocs as $networkDoc) { - $network[] = [ - 'value' => $networkDoc->getAttribute('value'), - 'date' => $networkDoc->getAttribute('time'), - ]; - } - - $network = array_reverse($network); - - $functionsDocs = Authorization::skip(function () use ($dbForInternal, $period, $range) { - return $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, ['executions']), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - }); - - $functions = []; - foreach ($functionsDocs as $functionDoc) { - $functions[] = [ - 'value' => $functionDoc->getAttribute('value'), - 'date' => $functionDoc->getAttribute('time'), - ]; - } - $functions = array_reverse($functions); - } else { - $requests = []; - $network = []; - $functions = []; } $usersCount = Authorization::skip(function () use ($dbForInternal) { @@ -336,7 +294,6 @@ App::get('/v1/projects/:projectId/usage') return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['collections.count'])], 0, ['time'], [Database::ORDER_DESC]); }); $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; - // $documents = []; @@ -357,25 +314,27 @@ App::get('/v1/projects/:projectId/usage') }); $filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0; + Authorization::reset(); + $response->json([ 'range' => $range, 'requests' => [ - 'data' => $requests, + 'data' => $stats['requests'] ?? [], 'total' => \array_sum(\array_map(function ($item) { return $item['value']; - }, $requests)), + }, $stats['requests'] ?? [])), ], 'network' => [ - 'data' => \array_map(function ($value) {return ['value' => \round($value['value'] / 1000000, 2), 'date' => $value['date']];}, $network), // convert bytes to mb + 'data' => \array_map(function ($value) {return ['value' => \round($value['value'] / 1000000, 2), 'date' => $value['date']];}, $stats['network'] ?? []), // convert bytes to mb 'total' => \array_sum(\array_map(function ($item) { return $item['value']; - }, $network)), + }, $stats['network'] ?? [])), ], 'functions' => [ - 'data' => $functions, + 'data' => $stats['executions'] ?? [], 'total' => \array_sum(\array_map(function ($item) { return $item['value']; - }, $functions)), + }, $stats['executions'] ?? [])), ], 'collections' => [ 'data' => [], From e574bdc3b39a6a74d810814a2349c46c54064069 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 12:39:03 +0545 Subject: [PATCH 039/120] remove influxdb dependency on appwrite container --- docker-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 143f124f20..a9911b1d6d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,7 +73,6 @@ services: - mariadb - redis # - clamav - - influxdb entrypoint: - php - -e @@ -112,8 +111,6 @@ services: - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD - _APP_USAGE_STATS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - _APP_STORAGE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_CONTAINERS From 91dc62d733bf081b86881e7d01cc0beb5f51bb28 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 12:39:32 +0545 Subject: [PATCH 040/120] metrics list comment --- app/tasks/usage.php | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index afe5958406..216949f6d2 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -14,6 +14,43 @@ use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; +/** + * Metrics We collect + * + * requests + * network + * executions + * database.collections.create + * database.collections.read + * database.collections.update + * database.collections.delete + * database.documents.create + * database.documents.read + * database.documents.update + * database.documents.delete + * database.collections.{collectionId}.documents.create + * database.collections.{collectionId}.documents.read + * database.collections.{collectionId}.documents.update + * database.collections.{collectionId}.documents.delete + * storage.buckets.{bucketId}.files.create + * storage.buckets.{bucketId}.files.read + * storage.buckets.{bucketId}.files.update + * storage.buckets.{bucketId}.files.delete + * users.create + * users.read + * users.update + * users.delete + * users.sessions.create + * users.sessions.delete + * + * Counters + * + * users.count + * files.count + * collections.count + * + */ + $cli ->task('usage') ->desc('Schedules syncing data from influxdb to Appwrite console db') From 800a76db729a8b85587537860d7366da69d59791 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 13:46:22 +0545 Subject: [PATCH 041/120] counter stats update --- app/controllers/api/projects.php | 37 ++++--------- app/tasks/usage.php | 91 ++++++++++++++++++++++++++------ 2 files changed, 84 insertions(+), 44 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index f660bf2f60..bea5332f11 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -285,33 +285,16 @@ App::get('/v1/projects/:projectId/usage') } } - $usersCount = Authorization::skip(function () use ($dbForInternal) { - return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); - }); + $usersCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); $usersTotal = $usersCount ? $usersCount->getAttribute('value', 0) : 0; - $collectionsCount = Authorization::skip(function () use ($dbForInternal, $period, $range) { - return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['collections.count'])], 0, ['time'], [Database::ORDER_DESC]); - }); + $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.collections.count'])], 0, ['time'], [Database::ORDER_DESC]); $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; - // $documents = []; + $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); + $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; - // foreach ($collections as $collection) { - // $result = $projectDB->getCollection([ - // 'limit' => 0, - // 'offset' => 0, - // 'filters' => [ - // '$collection=' . $collection['$id'], - // ], - // ]); - - // $documents[] = ['name' => $collection['name'], 'total' => $projectDB->getSum()]; - // } - - $filesCount = Authorization::skip(function () use ($dbForInternal, $period, $range) { - return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['files.count'])], 0, ['time'], [Database::ORDER_DESC]); - }); + $filesCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.files.count'])], 0, ['time'], [Database::ORDER_DESC]); $filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0; Authorization::reset(); @@ -344,12 +327,10 @@ App::get('/v1/projects/:projectId/usage') 'data' => [], 'total' => $filesTotal, ], - // 'documents' => [ - // 'data' => $documents, - // 'total' => \array_sum(\array_map(function ($item) { - // return $item['total']; - // }, $documents)), - // ], + 'documents' => [ + 'data' => [], + 'total' => $documentsTotal, + ], 'users' => [ 'data' => [], 'total' => $usersTotal, diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 216949f6d2..ffa2df490b 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -16,7 +16,7 @@ use Utopia\Database\Validator\Authorization; /** * Metrics We collect - * + * * requests * network * executions @@ -42,13 +42,15 @@ use Utopia\Database\Validator\Authorization; * users.delete * users.sessions.create * users.sessions.delete - * + * * Counters - * + * * users.count - * files.count - * collections.count - * + * storage.files.count + * database.collections.count + * database.documents.count + * database.collections.{collectionId}.documents.count + * */ $cli @@ -249,8 +251,10 @@ $cli } } - if ($iterations % 30 == 0) { - //aggregate number of objects in database + if ($iterations % 30 == 0) { //every 15 minutes + // aggregate number of objects in database + // get count of all the documents per collection - + // buckets will have the same $latestProject = null; do { $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); @@ -259,25 +263,80 @@ $cli foreach ($projects as $project) { $id = $project->getId(); - $collections = ['users' => [ - 'namespace' => 'internal', - ], 'collections' => [ - 'namespace' => 'external', - ], 'files' => [ - 'namespace' => 'internal', - ]]; + $collections = [ + 'users' => [ + 'namespace' => 'internal', + ], + 'collections' => [ + 'metricPrefix' => 'database', + 'namespace' => 'external', //new change will make this internal + 'subCollections' => [ + 'documents' => [ + 'namespace' => 'external', + ], + ], + ], + 'files' => [ + 'metricPrefix' => 'storage', + 'namespace' => 'internal', + ], + ]; foreach ($collections as $collection => $options) { $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); $count = $dbForProject->count($collection); $dbForProject->setNamespace("project_{$id}_internal"); + $metricPrefix = $options['metricPrefix'] ?? ''; + $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; $dbForProject->createDocument('stats', new Document([ '$id' => $dbForProject->getId(), 'time' => time(), 'period' => '15m', - 'metric' => "{$collection}.count", + 'metric' => $metric, 'value' => $count, 'type' => 1, ])); + + $subCollections = $options['subCollections'] ?? []; + if (!empty($subCollections)) { + $latestParent = null; + $subCollectionCounts = []; //total project level count of sub collections + do { + $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); + if (!empty($parents)) { + $latestParent = $parents[array_key_last($parents)]; + foreach ($parents as $parent) { + foreach ($subCollections as $subCollection => $subOptions) { + $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); + $count = $dbForProject->count($parent->getId()); + $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; + + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", + 'value' => $count, + 'type' => 1, + ])); + } + } + } + } while (!empty($parents)); + + foreach ($subCollectionsCounts as $subCollection => $count) { + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", + 'value' => $count, + 'type' => 1, + ])); + } + } } } } From 423ecb10712425d131c04b5d2c60ed2835540ba7 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 13:47:28 +0545 Subject: [PATCH 042/120] update stats table indexes --- app/config/collections2.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index 2b2419204e..0ed14dc369 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -1473,9 +1473,16 @@ $collections = [ 'orders' => [Database::ORDER_DESC], ], [ - '$id' => '_key_time_period', + '$id' => '_key_metric', 'type' => Database::INDEX_KEY, - 'attributes' => ['time', 'period'], + 'attributes' => ['metric'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_metric_period', + 'type' => Database::INDEX_KEY, + 'attributes' => ['metric', 'period'], 'lengths' => [], 'orders' => [Database::ORDER_DESC], ], From 39d4924fe82e2d0a71eba466df77a223a2982dd0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 13:59:23 +0545 Subject: [PATCH 043/120] storage total stats --- app/controllers/api/projects.php | 24 ++++++------------------ app/tasks/usage.php | 20 +++++++++++++++++++- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index bea5332f11..b655f7a6f0 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -296,6 +296,9 @@ App::get('/v1/projects/:projectId/usage') $filesCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.files.count'])], 0, ['time'], [Database::ORDER_DESC]); $filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0; + + $storageTotal = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.total'])], 0, ['time'], [Database::ORDER_DESC]); + $storage = $storageTotal ? $storageTotal->getAttribute('value', 0) : 0; Authorization::reset(); @@ -335,24 +338,9 @@ App::get('/v1/projects/:projectId/usage') 'data' => [], 'total' => $usersTotal, ], - // 'storage' => [ - // 'total' => $projectDB->getCount( - // [ - // 'attribute' => 'sizeOriginal', - // 'filters' => [ - // '$collection=files', - // ], - // ] - // ) + - // $projectDB->getCount( - // [ - // 'attribute' => 'size', - // 'filters' => [ - // '$collection=tags', - // ], - // ] - // ), - // ], + 'storage' => [ + 'total' => $storage, + ], ]); }); diff --git a/app/tasks/usage.php b/app/tasks/usage.php index ffa2df490b..3f938e46bb 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -51,6 +51,10 @@ use Utopia\Database\Validator\Authorization; * database.documents.count * database.collections.{collectionId}.documents.count * + * Totals + * + * storage.total + * */ $cli @@ -263,6 +267,20 @@ $cli foreach ($projects as $project) { $id = $project->getId(); + + // get total storage + $dbForProject->setNamespace('project_' . $id . '_internal'); + $storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size'); + + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'period' => '15m', + 'time' => time(), + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + $collections = [ 'users' => [ 'namespace' => 'internal', @@ -310,7 +328,7 @@ $cli $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); $count = $dbForProject->count($parent->getId()); $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; - + $dbForProject->setNamespace("project_{$id}_internal"); $dbForProject->createDocument('stats', new Document([ '$id' => $dbForProject->getId(), From 87415c3d953e60fb23bda48f4ac51eecfafd81f8 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 14:43:10 +0545 Subject: [PATCH 044/120] use internals table for collections --- app/tasks/usage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 3f938e46bb..21b67a6217 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -287,7 +287,7 @@ $cli ], 'collections' => [ 'metricPrefix' => 'database', - 'namespace' => 'external', //new change will make this internal + 'namespace' => 'internal', 'subCollections' => [ 'documents' => [ 'namespace' => 'external', From 71eabfff0d2df4931a90b7f41ac12142d3826fda Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 14:47:38 +0545 Subject: [PATCH 045/120] handling errors --- app/tasks/usage.php | 104 +++++++++++++++++++++++--------------------- 1 file changed, 54 insertions(+), 50 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 21b67a6217..7e7f7d0a12 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -300,60 +300,64 @@ $cli ], ]; foreach ($collections as $collection => $options) { - $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); - $count = $dbForProject->count($collection); - $dbForProject->setNamespace("project_{$id}_internal"); - $metricPrefix = $options['metricPrefix'] ?? ''; - $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => $metric, - 'value' => $count, - 'type' => 1, - ])); - - $subCollections = $options['subCollections'] ?? []; - if (!empty($subCollections)) { - $latestParent = null; - $subCollectionCounts = []; //total project level count of sub collections - do { - $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); - $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); - if (!empty($parents)) { - $latestParent = $parents[array_key_last($parents)]; - foreach ($parents as $parent) { - foreach ($subCollections as $subCollection => $subOptions) { - $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); - $count = $dbForProject->count($parent->getId()); - $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; - - $dbForProject->setNamespace("project_{$id}_internal"); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", - 'value' => $count, - 'type' => 1, - ])); + try { + $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $count = $dbForProject->count($collection); + $dbForProject->setNamespace("project_{$id}_internal"); + $metricPrefix = $options['metricPrefix'] ?? ''; + $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + + $subCollections = $options['subCollections'] ?? []; + if (!empty($subCollections)) { + $latestParent = null; + $subCollectionCounts = []; //total project level count of sub collections + do { + $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); + if (!empty($parents)) { + $latestParent = $parents[array_key_last($parents)]; + foreach ($parents as $parent) { + foreach ($subCollections as $subCollection => $subOptions) { + $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); + $count = $dbForProject->count($parent->getId()); + $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; + + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", + 'value' => $count, + 'type' => 1, + ])); + } } } + } while (!empty($parents)); + + foreach ($subCollectionsCounts as $subCollection => $count) { + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", + 'value' => $count, + 'type' => 1, + ])); } - } while (!empty($parents)); - - foreach ($subCollectionsCounts as $subCollection => $count) { - $dbForProject->setNamespace("project_{$id}_internal"); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", - 'value' => $count, - 'type' => 1, - ])); } + } catch(\Exception $e) { + Console::warning("Failed to save database counters data for project {$collection}"); } } } From 08bdf089f62f84464fc188701683a04c872a7fda Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 14:48:53 +0545 Subject: [PATCH 046/120] remove unused stats --- app/controllers/api/projects.php | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index b655f7a6f0..7d8f4ed54c 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -288,14 +288,8 @@ App::get('/v1/projects/:projectId/usage') $usersCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); $usersTotal = $usersCount ? $usersCount->getAttribute('value', 0) : 0; - $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.collections.count'])], 0, ['time'], [Database::ORDER_DESC]); - $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; - $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; - - $filesCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.files.count'])], 0, ['time'], [Database::ORDER_DESC]); - $filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0; $storageTotal = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.total'])], 0, ['time'], [Database::ORDER_DESC]); $storage = $storageTotal ? $storageTotal->getAttribute('value', 0) : 0; @@ -322,14 +316,6 @@ App::get('/v1/projects/:projectId/usage') return $item['value']; }, $stats['executions'] ?? [])), ], - 'collections' => [ - 'data' => [], - 'total' => $collectionsTotal, - ], - 'files' => [ - 'data' => [], - 'total' => $filesTotal, - ], 'documents' => [ 'data' => [], 'total' => $documentsTotal, From 2a1ccafb62a597ff3ab1d0e05387467a23d78f05 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 16:58:26 +0545 Subject: [PATCH 047/120] Update app/controllers/api/account.php Co-authored-by: Christy Jacob --- app/controllers/api/account.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 3d574054bb..273e7226fa 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -160,7 +160,7 @@ App::post('/v1/account/sessions') /** @var Utopia\Locale\Locale $locale */ /** @var MaxMind\Db\Reader $geodb */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Usage\Usage $usage */ + /** @var Appwrite\Stats\Stats $usage */ $email = \strtolower($email); $protocol = $request->getProtocol(); From 4b5db30a43c1be52c3f562c9ea68a23c53585960 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 16:58:32 +0545 Subject: [PATCH 048/120] Update app/controllers/api/account.php Co-authored-by: Christy Jacob --- app/controllers/api/account.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 273e7226fa..dafddb4fee 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -59,7 +59,7 @@ App::post('/v1/account') /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Usage\Usage $usage */ + /** @var Appwrite\Stats\Stats $usage */ $email = \strtolower($email); if ('console' === $project->getId()) { From 1d7daf951530839a705626b10303f1e8b71ec103 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 20 Aug 2021 11:40:23 +0545 Subject: [PATCH 049/120] new functions stats --- app/tasks/usage.php | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 7e7f7d0a12..6b9c56b718 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -43,6 +43,12 @@ use Utopia\Database\Validator\Authorization; * users.sessions.create * users.sessions.delete * + * Functions + * + * functions.{functionId}.executions + * functions.{functionId}.failures + * functions.{functionId}.compute + * * Counters * * users.count @@ -162,6 +168,21 @@ $cli 'users.sessions.delete' => [ 'table' => 'appwrite_usage_users_sessions_delete', ], + 'functions.functionId.executions' => [ + 'table' => 'appwrite_usage_executions_all', + 'groupBy' => 'functionId', + ], + 'functions.functionId.compute' => [ + 'table' => 'appwrite_usage_executions_time', + 'groupBy' => 'functionId', + ], + 'functions.functionId.failures' => [ + 'table' => 'appwrite_usage_executions_all', + 'groupBy' => 'functionId', + 'filters' => [ + 'functionStatus' => 'failed', + ], + ], ]; $attempts = 0; @@ -213,7 +234,15 @@ $cli $table = $options['table']; //which influxdb table to query for this metric $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //some sub level metrics may be grouped by other tags like collectionId, bucketId, etc - $result = $database->query('SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . $period['key'] . '), "projectId"' . $groupBy . ' FILL(null)'); + $filters = $options['filters'] ?? []; + if (!empty($filters)) { + $filters = ' AND ' . implode(' AND ', array_map(function ($filter, $value) { + return '"' . $filter . '"=\'' . $value . '\''; + }, array_keys($filters), array_values($filters))); + } + + $result = $database->query('SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\'' . (empty($filters) ? '' : $filters) . ' GROUP BY time(' . $period['key'] . '), "projectId"' . $groupBy . ' FILL(null)'); + $points = $result->getPoints(); foreach ($points as $point) { $projectId = $point['projectId']; @@ -314,7 +343,7 @@ $cli 'value' => $count, 'type' => 1, ])); - + $subCollections = $options['subCollections'] ?? []; if (!empty($subCollections)) { $latestParent = null; @@ -329,7 +358,7 @@ $cli $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); $count = $dbForProject->count($parent->getId()); $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; - + $dbForProject->setNamespace("project_{$id}_internal"); $dbForProject->createDocument('stats', new Document([ '$id' => $dbForProject->getId(), @@ -343,7 +372,7 @@ $cli } } } while (!empty($parents)); - + foreach ($subCollectionsCounts as $subCollection => $count) { $dbForProject->setNamespace("project_{$id}_internal"); $dbForProject->createDocument('stats', new Document([ @@ -356,7 +385,7 @@ $cli ])); } } - } catch(\Exception $e) { + } catch (\Exception$e) { Console::warning("Failed to save database counters data for project {$collection}"); } } From 7949d5b1385d9fc574cdd669459ea072111eaeee Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 20 Aug 2021 12:23:35 +0545 Subject: [PATCH 050/120] fixed issues with metrics --- app/tasks/usage.php | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 6b9c56b718..a6c9f3edac 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -248,15 +248,19 @@ $cli $projectId = $point['projectId']; if (!empty($projectId) && $projectId != 'console') { $dbForProject->setNamespace('project_' . $projectId . '_internal'); + if($metric == 'functions.functionId.executions') { + var_dump($points); + } + $metricUpdated = $metric; if (!empty($groupBy)) { - $groupedBy = $point[$groupBy] ?? ''; + $groupedBy = $point[$options['groupBy']] ?? ''; if (empty($groupedBy)) { continue; } - $metric = str_replace($groupBy, $groupedBy, $metric); + $metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric); } $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period['key'] . '_' . $metric); //construct unique id for each metric using time, period and metric + $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //construct unique id for each metric using time, period and metric $value = (!empty($point['value'])) ? $point['value'] : 0; try { $document = $dbForProject->getDocument('stats', $id); @@ -265,7 +269,7 @@ $cli '$id' => $id, 'period' => $period['key'], 'time' => $time, - 'metric' => $metric, + 'metric' => $metricUpdated, 'value' => $value, 'type' => 0, ])); @@ -276,7 +280,7 @@ $cli $latestTime[$metric][$period['key']] = $time; } catch (\Exception$e) { // if projects are deleted this might fail - Console::warning("Failed to save data for project {$projectId} and metric {$metric}"); + Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}"); } } } From 2c69fab6cd05994b74987dc6a844493a064a51f5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 20 Aug 2021 12:29:30 +0545 Subject: [PATCH 051/120] use metadata instead --- app/config/collections2.php | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index 0ed14dc369..4e71c06256 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -8,7 +8,7 @@ $auth = Config::getParam('auth', []); $collections = [ 'projects' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'projects', 'name' => 'Projects', 'attributes' => [ @@ -234,7 +234,7 @@ $collections = [ ], 'users' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'users', 'name' => 'Users', 'attributes' => [ @@ -383,7 +383,7 @@ $collections = [ ], 'sessions' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'sessions', 'name' => 'Sessions', 'attributes' => [ @@ -631,7 +631,7 @@ $collections = [ ], 'teams' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'teams', 'name' => 'Teams', 'attributes' => [ @@ -681,7 +681,7 @@ $collections = [ ], 'memberships' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'memberships', 'name' => 'Memberships', 'attributes' => [ @@ -789,7 +789,7 @@ $collections = [ ], 'files' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'files', 'name' => 'Files', 'attributes' => [ @@ -968,7 +968,7 @@ $collections = [ ], 'functions' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'functions', 'name' => 'Functions', 'attributes' => [ @@ -1129,7 +1129,7 @@ $collections = [ ], 'tags' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'tags', 'name' => 'Tags', 'attributes' => [ @@ -1202,7 +1202,7 @@ $collections = [ ], 'executions' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'executions', 'name' => 'Executions', 'attributes' => [ @@ -1319,7 +1319,7 @@ $collections = [ ], 'certificates' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'certificates', 'name' => 'Certificates', 'attributes' => [ @@ -1404,7 +1404,7 @@ $collections = [ ], ], 'stats' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'stats', 'name' => 'Stats', 'attributes' => [ From cfe49b68927fb6b2be41e90dff2362d1071f4a05 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 13:54:45 +0530 Subject: [PATCH 052/120] feat(usage): refactored functions usage endpoint --- app/controllers/api/database.php | 261 ++++++++++++++++++++---------- app/controllers/api/functions.php | 95 +++++++++++ app/controllers/api/projects.php | 2 +- 3 files changed, 273 insertions(+), 85 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index db8a97a9ec..86f4029cdb 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1,5 +1,6 @@ isValid($default)) { throw new Exception('Length of default attribute exceeds attribute size', 400); } - } + } if (!\is_null($format)) { $name = \json_decode($format, true)['name']; @@ -110,16 +111,14 @@ $attributesCallback = function ($attribute, $response, $dbForExternal, $database $database ->setParam('type', DATABASE_TYPE_CREATE_ATTRIBUTE) - ->setParam('document', $attribute) - ; + ->setParam('document', $attribute); $usage->setParam('database.collections.update', 1); $audits ->setParam('event', 'database.attributes.create') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $attribute) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $attribute); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($attribute, Response::MODEL_ATTRIBUTE); @@ -167,9 +166,8 @@ App::post('/v1/database/collections') $audits ->setParam('event', 'database.collections.create') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $collection->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $collection->getArrayCopy()); $usage->setParam('database.collections.create', 1); @@ -250,6 +248,112 @@ App::get('/v1/database/collections/:collectionId') $response->dynamic($collection, Response::MODEL_COLLECTION); }); +App::get('/v1/database/usage') + ->desc('Get Database Usage') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'database') + ->label('sdk.method', 'getUsage') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForConsole') + ->inject('dbForInternal') + ->inject('register') + ->action(function ($range, $response, $dbForConsole, $dbForInternal, $register) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForConsole */ + /** @var Utopia\Database\Database $dbForInternal */ + /** @var Utopia\Registry\Registry $register */ + + $stats = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + + $metrics = [ + 'database.collections.create', + 'database.collections.read', + 'database.collections.update', + 'database.collections.delete', + 'database.documents.create', + 'database.documents.read', + 'database.documents.update', + 'database.documents.delete' + ]; + + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + } + } + + $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); + $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; + + $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['n + '])], 0, ['time'], [Database::ORDER_DESC]); + $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; + + Authorization::reset(); + + $response->json([ + 'range' => $range, + 'stats' => $stats, + 'requests' => [ + 'data' => $stats['requests'] ?? [], + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $stats['requests'] ?? [])), + ], + 'documents' => [ + 'data' => [], + 'total' => $documentsTotal, + ], + 'collections' => [ + 'data' => [], + 'total' => $collectionsTotal, + ] + ]); + }); + + +// :collectionId/usage +// 'reads', +// 'writes', +// 'updates', +// 'delete' + App::get('/v1/database/collections/:collectionId/logs') ->desc('List Collection Logs') ->groups(['api', 'database']) @@ -283,7 +387,7 @@ App::get('/v1/database/collections/:collectionId/logs') $audit = new Audit($dbForInternal); - $logs = $audit->getLogsByResource('database/collection/'.$collection->getId()); + $logs = $audit->getLogsByResource('database/collection/' . $collection->getId()); $output = []; @@ -335,8 +439,8 @@ App::get('/v1/database/collections/:collectionId/logs') $record = $geodb->get($log['ip']); if ($record) { - $output[$i]['countryCode'] = $locale->getText('countries.'.strtolower($record['country']['iso_code']), false) ? \strtolower($record['country']['iso_code']) : '--'; - $output[$i]['countryName'] = $locale->getText('countries.'.strtolower($record['country']['iso_code']), $locale->getText('locale.country.unknown')); + $output[$i]['countryCode'] = $locale->getText('countries.' . strtolower($record['country']['iso_code']), false) ? \strtolower($record['country']['iso_code']) : '--'; + $output[$i]['countryName'] = $locale->getText('countries.' . strtolower($record['country']['iso_code']), $locale->getText('locale.country.unknown')); } else { $output[$i]['countryCode'] = '--'; $output[$i]['countryName'] = $locale->getText('locale.country.unknown'); @@ -390,16 +494,15 @@ App::put('/v1/database/collections/:collectionId') } catch (AuthorizationException $exception) { throw new Exception('Unauthorized permissions', 401); } catch (StructureException $exception) { - throw new Exception('Bad structure. '.$exception->getMessage(), 400); - } + throw new Exception('Bad structure. ' . $exception->getMessage(), 400); + } $usage->setParam('database.collections.update', 1); $audits ->setParam('event', 'database.collections.update') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $collection->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $collection->getArrayCopy()); $response->dynamic($collection, Response::MODEL_COLLECTION); }); @@ -440,14 +543,12 @@ App::delete('/v1/database/collections/:collectionId') $usage->setParam('database.collections.delete', 1); $events - ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)) - ; + ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)); $audits ->setParam('event', 'database.collections.delete') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $collection->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $collection->getArrayCopy()); $response->noContent(); }); @@ -530,7 +631,7 @@ App::post('/v1/database/collections/:collectionId/attributes/email') 'required' => $required, 'default' => $default, 'array' => $array, - 'format' => \json_encode(['name'=>'email']), + 'format' => \json_encode(['name' => 'email']), ]), $response, $dbForExternal, $database, $audits, $usage); }); @@ -571,7 +672,7 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') 'required' => $required, 'default' => $default, 'array' => $array, - 'format' => \json_encode(['name'=>'ip']), + 'format' => \json_encode(['name' => 'ip']), ]), $response, $dbForExternal, $database, $audits, $usage); }); @@ -613,7 +714,7 @@ App::post('/v1/database/collections/:collectionId/attributes/url') 'required' => $required, 'default' => $default, 'array' => $array, - 'format' => \json_encode(['name'=>'url']), + 'format' => \json_encode(['name' => 'url']), ]), $response, $dbForExternal, $database, $audits, $usage); }); @@ -648,7 +749,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Stats\Stats $usage */ - + return $attributesCallback(new Document([ '$collection' => $collectionId, '$id' => $attributeId, @@ -658,7 +759,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') 'default' => $default, 'array' => $array, 'format' => \json_encode([ - 'name'=>'int-range', + 'name' => 'int-range', 'min' => $min, 'max' => $max, ]), @@ -705,7 +806,7 @@ App::post('/v1/database/collections/:collectionId/attributes/float') 'default' => $default, 'array' => $array, 'format' => \json_encode([ - 'name'=>'float-range', + 'name' => 'float-range', 'min' => $min, 'max' => $max, ]), @@ -835,7 +936,7 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId') ])]); $usage->setParam('database.collections.read', 1); - + $response->dynamic($attribute, Response::MODEL_ATTRIBUTE); }); @@ -890,20 +991,17 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') $database ->setParam('type', DATABASE_TYPE_DELETE_ATTRIBUTE) - ->setParam('document', $attribute) - ; + ->setParam('document', $attribute); $usage->setParam('database.collections.update', 1); $events - ->setParam('payload', $response->output($attribute, Response::MODEL_ATTRIBUTE)) - ; + ->setParam('payload', $response->output($attribute, Response::MODEL_ATTRIBUTE)); $audits ->setParam('event', 'database.attributes.delete') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $attribute->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $attribute->getArrayCopy()); $response->noContent(); }); @@ -984,20 +1082,17 @@ App::post('/v1/database/collections/:collectionId/indexes') $database ->setParam('type', DATABASE_TYPE_CREATE_INDEX) - ->setParam('document', $index) - ; + ->setParam('document', $index); $usage->setParam('database.collections.update', 1); $audits ->setParam('event', 'database.indexes.create') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $index->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $index->getArrayCopy()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($index, Response::MODEL_INDEX); - }); App::get('/v1/database/collections/:collectionId/indexes') @@ -1083,7 +1178,7 @@ App::get('/v1/database/collections/:collectionId/indexes/:indexId') ])]); $usage->setParam('database.collections.read', 1); - + $response->dynamic($index, Response::MODEL_INDEX); }); @@ -1124,7 +1219,7 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') $indexes = $collection->getAttribute('indexes'); // find attribute in collection - $index= null; + $index = null; foreach ($indexes as $i) { if ($i->getId() === $indexId) { $index = $i->setAttribute('$collection', $collectionId); // set the collectionId @@ -1138,20 +1233,17 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') $database ->setParam('type', DATABASE_TYPE_DELETE_INDEX) - ->setParam('document', $index) - ; + ->setParam('document', $index); $usage->setParam('database.collections.update', 1); $events - ->setParam('payload', $response->output($index, Response::MODEL_INDEX)) - ; + ->setParam('payload', $response->output($index, Response::MODEL_INDEX)); $audits ->setParam('event', 'database.indexes.delete') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $index->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $index->getArrayCopy()); $response->noContent(); }); @@ -1184,7 +1276,7 @@ App::post('/v1/database/collections/:collectionId/documents') /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Stats\Stats $usage */ - + $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array if (empty($data)) { @@ -1194,7 +1286,7 @@ App::post('/v1/database/collections/:collectionId/documents') if (isset($data['$id'])) { throw new Exception('$id is not allowed for creating new documents, try update instead', 400); } - + $collection = $dbForExternal->getCollection($collectionId); if ($collection->isEmpty()) { @@ -1203,26 +1295,23 @@ App::post('/v1/database/collections/:collectionId/documents') $data['$collection'] = $collection->getId(); // Adding this param to make API easier for developers $data['$id'] = $documentId == 'unique()' ? $dbForExternal->getId() : $documentId; - $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? []; // By default set read permissions for user - $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? []; // By default set write permissions for user + $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user + $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? []; // By default set write permissions for user try { $document = $dbForExternal->createDocument($collectionId, new Document($data)); - } - catch (StructureException $exception) { + } catch (StructureException $exception) { throw new Exception($exception->getMessage(), 400); } $usage ->setParam('database.documents.create', 1) - ->setParam('collectionId', $collectionId) - ; + ->setParam('collectionId', $collectionId); $audits ->setParam('event', 'database.documents.create') - ->setParam('resource', 'database/document/'.$document->getId()) - ->setParam('data', $document->getArrayCopy()) - ; + ->setParam('resource', 'database/document/' . $document->getId()) + ->setParam('data', $document->getArrayCopy()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($document, Response::MODEL_DOCUMENT); @@ -1286,9 +1375,8 @@ App::get('/v1/database/collections/:collectionId/documents') $usage ->setParam('database.documents.read', 1) - ->setParam('collectionId', $collectionId) - ; - + ->setParam('collectionId', $collectionId); + $response->dynamic(new Document([ 'sum' => $dbForExternal->count($collectionId, $queries, APP_LIMIT_COUNT), 'documents' => $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null), @@ -1329,8 +1417,7 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.read', 1) - ->setParam('collectionId', $collectionId) - ; + ->setParam('collectionId', $collectionId); $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -1379,7 +1466,7 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') if (empty($data)) { throw new Exception('Missing payload', 400); } - + if (!\is_array($data)) { throw new Exception('Data param should be a valid JSON object', 400); } @@ -1393,24 +1480,20 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') try { $document = $dbForExternal->updateDocument($collection->getId(), $document->getId(), new Document($data)); - } - catch (AuthorizationException $exception) { + } catch (AuthorizationException $exception) { throw new Exception('Unauthorized permissions', 401); + } catch (StructureException $exception) { + throw new Exception('Bad structure. ' . $exception->getMessage(), 400); } - catch (StructureException $exception) { - throw new Exception('Bad structure. '.$exception->getMessage(), 400); - } - + $usage ->setParam('database.documents.update', 1) - ->setParam('collectionId', $collectionId) - ; + ->setParam('collectionId', $collectionId); $audits ->setParam('event', 'database.documents.update') - ->setParam('resource', 'database/document/'.$document->getId()) - ->setParam('data', $document->getArrayCopy()) - ; + ->setParam('resource', 'database/document/' . $document->getId()) + ->setParam('data', $document->getArrayCopy()); $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -1456,18 +1539,28 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.delete', 1) - ->setParam('collectionId', $collectionId) - ; + ->setParam('collectionId', $collectionId); $events - ->setParam('eventData', $response->output($document, Response::MODEL_DOCUMENT)) - ; + ->setParam('eventData', $response->output($document, Response::MODEL_DOCUMENT)); $audits ->setParam('event', 'database.documents.delete') - ->setParam('resource', 'database/document/'.$document->getId()) + ->setParam('resource', 'database/document/' . $document->getId()) ->setParam('data', $document->getArrayCopy()) // Audit document in case of malicious or disastrous action ; $response->noContent(); }); + + +// Refactor usage endpoint in Functions API + + +// Create usage endpoint in the Database API + + +// Create usage endpoint in the users API + + +// Create usage endpoint in the Storage API \ No newline at end of file diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index fbd27c47f3..0f7a4f9a99 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -259,6 +259,101 @@ App::get('/v1/functions/:functionId/usage') } }); +App::get('/v1/functions/:functionId/usage2') + ->desc('Get Function Usage') + ->groups(['api', 'functions']) + ->label('scope', 'functions.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'functions') + ->label('sdk.method', 'getUsage') + ->param('functionId', '', new UID(), 'Function unique ID.') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true) + ->inject('response') + ->inject('project') + ->inject('dbForInternal') + ->inject('register') + ->action(function ($functionId, $range, $response, $project, $dbForInternal, $register) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Document $project */ + /** @var Utopia\Database\Database $dbForInternal */ + /** @var Utopia\Registry\Registry $register */ + + $function = $dbForInternal->getDocument('functions', $functionId); + + if ($function->isEmpty()) { + throw new Exception('Function not found', 404); + } + + if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + + $metrics = ["functions.$functionId.executions", "functions.$functionId.failures", "functions.$functionId.compute"]; + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + } + + $executions = $stats["functions.$functionId.executions"] ?? []; + $failures = $stats["functions.$functionId.failures"] ?? []; + $compute = $stats["functions.$functionId.compute"] ?? []; + + $response->json([ + 'range' => $range, + 'executions' => [ + 'data' => $executions, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $executions)), + ], + 'failures' => [ + 'data' => $failures, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $failures)), + ], + 'compute' => [ + 'data' => $compute, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $compute)), + ], + ]); + } else { + $response->json([]); + } + }); + App::put('/v1/functions/:functionId') ->groups(['api', 'functions']) ->desc('Update Function') diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 7d8f4ed54c..ba9d5627b5 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -217,7 +217,7 @@ App::get('/v1/projects/:projectId') }); App::get('/v1/projects/:projectId/usage') - ->desc('Get Project') + ->desc('Get Project Usage') ->groups(['api', 'projects']) ->label('scope', 'projects.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) From d654e4cd090f68b4f0320998f945f383f7454fa7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 14:28:42 +0530 Subject: [PATCH 053/120] feat(usage): added usage endpoint to storage api --- app/controllers/api/storage.php | 102 ++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index ed3c61dcdb..daecfb64e6 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -10,6 +10,7 @@ use Utopia\Validator\HexColor; use Utopia\Cache\Cache; use Utopia\Cache\Adapter\Filesystem; use Appwrite\ClamAV\Network; +use Appwrite\Database\Validator\Authorization; use Appwrite\Database\Validator\CustomId; use Utopia\Database\Document; use Utopia\Database\Validator\UID; @@ -22,6 +23,7 @@ use Utopia\Image\Image; use Appwrite\OpenSSL\OpenSSL; use Appwrite\Utopia\Response; use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\Database\Query; App::post('/v1/storage/files') @@ -640,4 +642,104 @@ App::delete('/v1/storage/files/:fileId') ; $response->noContent(); + }); + +App::get('/v1/storage/:bucketId/usage') + ->desc('Get Bucket Usage') + ->groups(['api', 'storage']) + ->label('scope', 'storage.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'storage') + ->label('sdk.method', 'getUsage') + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForInternal') + ->inject('register') + ->action(function ($bucketId, $range, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForInternal */ + + $stats = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + + $metrics = [ + "storage.buckets.$bucketId.files.create", + "storage.buckets.$bucketId.files.read", + "storage.buckets.$bucketId.files.update", + "storage.buckets.$bucketId.files.delete" + ]; + + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + } + } + + Authorization::reset(); + + $create = $stats["storage.buckets.$bucketId.files.create"] ?? []; + $read = $stats["storage.buckets.$bucketId.files.read"] ?? []; + $update = $stats["storage.buckets.$bucketId.files.update"] ?? []; + $delete = $stats["storage.buckets.$bucketId.files.delete"] ?? []; + + $response->json([ + 'range' => $range, + 'create' => [ + 'data' => $create, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $create)), + ], + 'read' => [ + 'data' => $read, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $read)), + ], + 'update' => [ + 'data' => $update, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $update)), + ], + 'delete' => [ + 'data' => $delete, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $delete)), + ], + ]); }); \ No newline at end of file From 070b7d827013a39a42bf8a09edbc6b40f1ea01c4 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 14:30:39 +0530 Subject: [PATCH 054/120] feat(usage): added usage endpoint to storage api --- app/controllers/api/storage.php | 72 +++++++++++++++++---------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index daecfb64e6..d167ee88b7 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -706,40 +706,42 @@ App::get('/v1/storage/:bucketId/usage') $stats[$metric] = array_reverse($stats[$metric]); } + + Authorization::reset(); + + $create = $stats["storage.buckets.$bucketId.files.create"] ?? []; + $read = $stats["storage.buckets.$bucketId.files.read"] ?? []; + $update = $stats["storage.buckets.$bucketId.files.update"] ?? []; + $delete = $stats["storage.buckets.$bucketId.files.delete"] ?? []; + + $response->json([ + 'range' => $range, + 'create' => [ + 'data' => $create, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $create)), + ], + 'read' => [ + 'data' => $read, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $read)), + ], + 'update' => [ + 'data' => $update, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $update)), + ], + 'delete' => [ + 'data' => $delete, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $delete)), + ], + ]); + } else { + $response->json([]); } - - Authorization::reset(); - - $create = $stats["storage.buckets.$bucketId.files.create"] ?? []; - $read = $stats["storage.buckets.$bucketId.files.read"] ?? []; - $update = $stats["storage.buckets.$bucketId.files.update"] ?? []; - $delete = $stats["storage.buckets.$bucketId.files.delete"] ?? []; - - $response->json([ - 'range' => $range, - 'create' => [ - 'data' => $create, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $create)), - ], - 'read' => [ - 'data' => $read, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $read)), - ], - 'update' => [ - 'data' => $update, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $update)), - ], - 'delete' => [ - 'data' => $delete, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $delete)), - ], - ]); }); \ No newline at end of file From aefbdb05974274eb8217e2ff14b09847af2454c4 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 15:27:46 +0530 Subject: [PATCH 055/120] feat(usage): added usage endpoint to users api --- app/controllers/api/users.php | 127 ++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 09d9d1bcdf..45d4eb03ed 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -2,6 +2,7 @@ use Appwrite\Auth\Auth; use Appwrite\Auth\Validator\Password; +use Appwrite\Database\Validator\Authorization; use Appwrite\Utopia\Response; use Utopia\App; use Utopia\Exception; @@ -17,6 +18,8 @@ use Utopia\Database\Exception\Duplicate; use Utopia\Database\Validator\UID; use DeviceDetector\DeviceDetector; use Appwrite\Database\Validator\CustomId; +use Utopia\Database\Database; +use Utopia\Database\Query; App::post('/v1/users') ->desc('Create User') @@ -605,3 +608,127 @@ App::delete('/v1/users/:userId') ; $response->noContent(); }); + +App::get('/v1/users/usage') + ->desc('Get Users Usage') + ->groups(['api', 'users']) + ->label('scope', 'users.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'users') + ->label('sdk.method', 'getUsage') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForInternal') + ->inject('register') + ->action(function ($range, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForInternal */ + + $stats = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + + $metrics = [ + "users.create", + "users.read", + "users.update", + "users.delete", + "users.sessions.create", + "users.sessions.delete" + ]; + + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + } + + $usersCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); + $usersTotal = $usersCount ? $usersCount->getAttribute('value', 0) : 0; + + Authorization::reset(); + + $create = $stats["users.create"] ?? []; + $read = $stats["users.read"] ?? []; + $update = $stats["users.update"] ?? []; + $delete = $stats["users.delete"] ?? []; + $sessionsCreate = $stats["users.sessions.create"] ?? []; + $sessionsDelete = $stats["users.sessions.delete"] ?? []; + + $response->json([ + 'range' => $range, + 'users' => [ + 'data' => [], + 'total' => $usersTotal, + ], + 'create' => [ + 'data' => $create, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $create)), + ], + 'read' => [ + 'data' => $read, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $read)), + ], + 'update' => [ + 'data' => $update, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $update)), + ], + 'delete' => [ + 'data' => $delete, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $delete)), + ], + 'sessionsCreate' => [ + 'data' => $sessionsCreate, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $sessionsCreate)), + ], + 'sessionsDelete' => [ + 'data' => $sessionsDelete, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $sessionsDelete)), + ] + ]); + } else { + $response->json([]); + } + }); \ No newline at end of file From f3074bc024b64b043d658146d0be9132b724e64a Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 15:34:57 +0530 Subject: [PATCH 056/120] feat(usage): some refactoring --- app/controllers/api/database.php | 58 ++++++++++++++++--------------- app/controllers/api/functions.php | 2 ++ 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 86f4029cdb..f2bf0329ca 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -316,35 +316,37 @@ App::get('/v1/database/usage') $stats[$metric] = array_reverse($stats[$metric]); } + + $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); + $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; + + $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['n + '])], 0, ['time'], [Database::ORDER_DESC]); + $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; + + Authorization::reset(); + + $response->json([ + 'range' => $range, + 'stats' => $stats, + 'requests' => [ + 'data' => $stats['requests'] ?? [], + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $stats['requests'] ?? [])), + ], + 'documents' => [ + 'data' => [], + 'total' => $documentsTotal, + ], + 'collections' => [ + 'data' => [], + 'total' => $collectionsTotal, + ] + ]); + } else { + $response->json([]); } - - $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); - $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; - - $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['n - '])], 0, ['time'], [Database::ORDER_DESC]); - $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; - - Authorization::reset(); - - $response->json([ - 'range' => $range, - 'stats' => $stats, - 'requests' => [ - 'data' => $stats['requests'] ?? [], - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $stats['requests'] ?? [])), - ], - 'documents' => [ - 'data' => [], - 'total' => $documentsTotal, - ], - 'collections' => [ - 'data' => [], - 'total' => $collectionsTotal, - ] - ]); }); diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 0f7a4f9a99..ee3088ce18 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -323,6 +323,8 @@ App::get('/v1/functions/:functionId/usage2') $stats[$metric] = array_reverse($stats[$metric]); } + + Authorization::reset(); $executions = $stats["functions.$functionId.executions"] ?? []; $failures = $stats["functions.$functionId.failures"] ?? []; From 8b393b11653e0f8cf8ff86050be2890f28cdc2eb Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 16:53:33 +0530 Subject: [PATCH 057/120] feat(usage): added usage endpoint for database --- app/controllers/api/database.php | 65 ++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index f2bf0329ca..b4238d284a 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -320,26 +320,75 @@ App::get('/v1/database/usage') $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; - $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['n - '])], 0, ['time'], [Database::ORDER_DESC]); + $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.collections.count'])], 0, ['time'], [Database::ORDER_DESC]); $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; Authorization::reset(); + $documentsCreate = $stats["database.documents.create"] ?? []; + $documentsRead = $stats["database.documents.read"] ?? []; + $documentsUpdate = $stats["database.documents.update"] ?? []; + $documentsDelete = $stats["database.documents.delete"] ?? []; + $collectionsCreate = $stats["database.collections.create"] ?? []; + $collectionsRead = $stats["database.collections.read"] ?? []; + $collectionsUpdate = $stats["database.collections.update"] ?? []; + $collectionsDelete = $stats["database.collections.delete"] ?? []; + $response->json([ 'range' => $range, - 'stats' => $stats, - 'requests' => [ - 'data' => $stats['requests'] ?? [], + 'documents.create' => [ + 'data' => $documentsCreate, 'total' => \array_sum(\array_map(function ($item) { return $item['value']; - }, $stats['requests'] ?? [])), + }, $documentsCreate)), ], - 'documents' => [ + 'documents.read' => [ + 'data' => $documentsRead, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $documentsRead)), + ], + 'documents.update' => [ + 'data' => $documentsUpdate, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $documentsUpdate)), + ], + 'documents.delete' => [ + 'data' => $documentsDelete, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $documentsDelete)), + ], + 'collections.create' => [ + 'data' => $collectionsCreate, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $collectionsCreate)), + ], + 'collections.read' => [ + 'data' => $collectionsRead, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $collectionsRead)), + ], + 'collections.update' => [ + 'data' => $collectionsUpdate, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $collectionsUpdate)), + ], + 'collections.delete' => [ + 'data' => $collectionsDelete, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $collectionsDelete)), + ], + 'documentCount' => [ 'data' => [], 'total' => $documentsTotal, ], - 'collections' => [ + 'collectionCount' => [ 'data' => [], 'total' => $collectionsTotal, ] From 2468e6e97d5a1c5a03f88487e227c9bfe0cc38e4 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 17:08:57 +0530 Subject: [PATCH 058/120] feat(usage): added usage endpoint for collections --- app/controllers/api/database.php | 124 +++++++++++++++++++++++++++++-- 1 file changed, 117 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index b4238d284a..5b41fbcf6f 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -257,10 +257,9 @@ App::get('/v1/database/usage') ->label('sdk.method', 'getUsage') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') - ->inject('dbForConsole') ->inject('dbForInternal') ->inject('register') - ->action(function ($range, $response, $dbForConsole, $dbForInternal, $register) { + ->action(function ($range, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ /** @var Utopia\Database\Database $dbForInternal */ @@ -399,11 +398,122 @@ App::get('/v1/database/usage') }); -// :collectionId/usage -// 'reads', -// 'writes', -// 'updates', -// 'delete' +App::get('/v1/database/:collectionId/usage') + ->desc('Get Database Usage') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'database') + ->label('sdk.method', 'getUsage') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->param('collectionId', '', new UID(), 'Collection unique ID.') + ->inject('response') + ->inject('dbForInternal') + ->inject('dbForExternal') + ->action(function ($range, $collectionId, $response, $dbForInternal, $dbForExternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForConsole */ + /** @var Utopia\Database\Database $dbForInternal */ + /** @var Utopia\Registry\Registry $register */ + + $collection = $dbForExternal->getCollection($collectionId); + + if ($collection->isEmpty()) { + throw new Exception('Collection not found', 404); + } + + $stats = []; + if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + + $metrics = [ + "database.collections.$collectionId.documents.create", + "database.collections.$collectionId.documents.read", + "database.collections.$collectionId.documents.update", + "database.collections.$collectionId.documents.delete", + ]; + + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + } + + $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ["database.collections.$collectionId.documents.count"])], 0, ['time'], [Database::ORDER_DESC]); + $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; + + Authorization::reset(); + + $create = $stats["database.collections.$collectionId.documents.create"] ?? []; + $read = $stats["database.collections.$collectionId.documents.read"] ?? []; + $update = $stats["database.collections.$collectionId.documents.update"] ?? []; + $delete = $stats["database.collections.$collectionId.documents.delete"] ?? []; + + $response->json([ + 'range' => $range, + 'create' => [ + 'data' => $create, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $create)), + ], + 'read' => [ + 'data' => $read, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $read)), + ], + 'update' => [ + 'data' => $update, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $update)), + ], + 'delete' => [ + 'data' => $delete, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $delete)), + ], + 'documentCount' => [ + 'data' => [], + 'total' => $documentsTotal, + ], + ]); + } else { + $response->json([]); + } + }); App::get('/v1/database/collections/:collectionId/logs') ->desc('List Collection Logs') From ae030ebd4f383f86983e01be5981c6f116e28446 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 17:10:35 +0530 Subject: [PATCH 059/120] feat(usage): doc fix --- app/controllers/api/database.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 5b41fbcf6f..e65d6929fd 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -399,7 +399,7 @@ App::get('/v1/database/usage') App::get('/v1/database/:collectionId/usage') - ->desc('Get Database Usage') + ->desc('Get Database Usage for a collection') ->groups(['api', 'database']) ->label('scope', 'collections.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) From c36b5b8b30716bfdae32fb6c1364a329db38981a Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 17:11:13 +0530 Subject: [PATCH 060/120] feat(usage): cleanup comments --- app/controllers/api/database.php | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index e65d6929fd..c167c2f6dd 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1712,16 +1712,4 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') ; $response->noContent(); - }); - - -// Refactor usage endpoint in Functions API - - -// Create usage endpoint in the Database API - - -// Create usage endpoint in the users API - - -// Create usage endpoint in the Storage API \ No newline at end of file + }); \ No newline at end of file From 24925d2eb259ea003f945c8cf7d6a7f5a0bf6bb8 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 17:40:52 +0530 Subject: [PATCH 061/120] feat(usage): cleanup comments --- app/controllers/api/database.php | 5 ++--- app/controllers/api/storage.php | 4 +++- app/controllers/api/users.php | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index c167c2f6dd..ddbbe7e39b 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -249,7 +249,7 @@ App::get('/v1/database/collections/:collectionId') }); App::get('/v1/database/usage') - ->desc('Get Database Usage') + ->desc('Get usage stats for the database') ->groups(['api', 'database']) ->label('scope', 'collections.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) @@ -258,7 +258,6 @@ App::get('/v1/database/usage') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') - ->inject('register') ->action(function ($range, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ @@ -399,7 +398,7 @@ App::get('/v1/database/usage') App::get('/v1/database/:collectionId/usage') - ->desc('Get Database Usage for a collection') + ->desc('Get usage stats for a collection') ->groups(['api', 'database']) ->label('scope', 'collections.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index d167ee88b7..0800f2ac65 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -645,7 +645,7 @@ App::delete('/v1/storage/files/:fileId') }); App::get('/v1/storage/:bucketId/usage') - ->desc('Get Bucket Usage') + ->desc('Get usage stats for a storage bucket') ->groups(['api', 'storage']) ->label('scope', 'storage.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) @@ -660,6 +660,8 @@ App::get('/v1/storage/:bucketId/usage') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + // TODO: Check is the storage bucket exists else throw 404 + $stats = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 45d4eb03ed..6d82c4f472 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -610,7 +610,7 @@ App::delete('/v1/users/:userId') }); App::get('/v1/users/usage') - ->desc('Get Users Usage') + ->desc('Get usage stats for the users API') ->groups(['api', 'users']) ->label('scope', 'users.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) From 9ebc5b891b29654f699cb518bfafa017ee4a142e Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 17:47:19 +0530 Subject: [PATCH 062/120] feat(usage): added usage endpoint for storage --- app/controllers/api/storage.php | 62 +++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 0800f2ac65..3b433988e0 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -644,6 +644,64 @@ App::delete('/v1/storage/files/:fileId') $response->noContent(); }); +App::get('/v1/storage/usage') + ->desc('Get usage stats for storage') + ->groups(['api', 'storage']) + ->label('scope', 'storage.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'storage') + ->label('sdk.method', 'getUsage') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForInternal') + ->action(function ($range, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForInternal */ + + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + + $storageTotal = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.total'])], 0, ['time'], [Database::ORDER_DESC]); + $storage = $storageTotal ? $storageTotal->getAttribute('value', 0) : 0; + + $filesCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.files.count'])], 0, ['time'], [Database::ORDER_DESC]); + $filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0; + + Authorization::reset(); + + $response->json([ + 'range' => $range, + 'storage' => [ + 'total' => $storage, + ], + 'files' => [ + 'total' => $filesTotal, + ], + ]); + } else { + $response->json([]); + } + }); + App::get('/v1/storage/:bucketId/usage') ->desc('Get usage stats for a storage bucket') ->groups(['api', 'storage']) @@ -655,13 +713,11 @@ App::get('/v1/storage/:bucketId/usage') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') - ->inject('register') ->action(function ($bucketId, $range, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - // TODO: Check is the storage bucket exists else throw 404 - + // TODO: Check if the storage bucket exists else throw 404 $stats = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ From e7eb0e7cdfdad9cbd1c9f0f5c02c2b8b82746141 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 17:47:56 +0530 Subject: [PATCH 063/120] feat(usage): added usage endpoint for storage --- app/controllers/api/projects.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index ba9d5627b5..45b7d7b010 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -217,7 +217,7 @@ App::get('/v1/projects/:projectId') }); App::get('/v1/projects/:projectId/usage') - ->desc('Get Project Usage') + ->desc('Get usage stats for a project') ->groups(['api', 'projects']) ->label('scope', 'projects.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) From a2a73d87e25a3efd14000d81984cb221613ea81d Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 17:52:04 +0530 Subject: [PATCH 064/120] feat(usage): added usage endpoint for database --- app/controllers/api/database.php | 149 ++++++++++++++++++------------- 1 file changed, 87 insertions(+), 62 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index ddbbe7e39b..59083546a1 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -61,7 +61,7 @@ $attributesCallback = function ($attribute, $response, $dbForExternal, $database if (!$validator->isValid($default)) { throw new Exception('Length of default attribute exceeds attribute size', 400); } - } + } if (!\is_null($format)) { $name = \json_decode($format, true)['name']; @@ -111,14 +111,16 @@ $attributesCallback = function ($attribute, $response, $dbForExternal, $database $database ->setParam('type', DATABASE_TYPE_CREATE_ATTRIBUTE) - ->setParam('document', $attribute); + ->setParam('document', $attribute) + ; $usage->setParam('database.collections.update', 1); $audits ->setParam('event', 'database.attributes.create') - ->setParam('resource', 'database/collection/' . $collection->getId()) - ->setParam('data', $attribute); + ->setParam('resource', 'database/collection/'.$collection->getId()) + ->setParam('data', $attribute) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($attribute, Response::MODEL_ATTRIBUTE); @@ -166,8 +168,9 @@ App::post('/v1/database/collections') $audits ->setParam('event', 'database.collections.create') - ->setParam('resource', 'database/collection/' . $collection->getId()) - ->setParam('data', $collection->getArrayCopy()); + ->setParam('resource', 'database/collection/'.$collection->getId()) + ->setParam('data', $collection->getArrayCopy()) + ; $usage->setParam('database.collections.create', 1); @@ -248,7 +251,7 @@ App::get('/v1/database/collections/:collectionId') $response->dynamic($collection, Response::MODEL_COLLECTION); }); -App::get('/v1/database/usage') + App::get('/v1/database/usage') ->desc('Get usage stats for the database') ->groups(['api', 'database']) ->label('scope', 'collections.read') @@ -396,7 +399,6 @@ App::get('/v1/database/usage') } }); - App::get('/v1/database/:collectionId/usage') ->desc('Get usage stats for a collection') ->groups(['api', 'database']) @@ -547,7 +549,7 @@ App::get('/v1/database/collections/:collectionId/logs') $audit = new Audit($dbForInternal); - $logs = $audit->getLogsByResource('database/collection/' . $collection->getId()); + $logs = $audit->getLogsByResource('database/collection/'.$collection->getId()); $output = []; @@ -599,8 +601,8 @@ App::get('/v1/database/collections/:collectionId/logs') $record = $geodb->get($log['ip']); if ($record) { - $output[$i]['countryCode'] = $locale->getText('countries.' . strtolower($record['country']['iso_code']), false) ? \strtolower($record['country']['iso_code']) : '--'; - $output[$i]['countryName'] = $locale->getText('countries.' . strtolower($record['country']['iso_code']), $locale->getText('locale.country.unknown')); + $output[$i]['countryCode'] = $locale->getText('countries.'.strtolower($record['country']['iso_code']), false) ? \strtolower($record['country']['iso_code']) : '--'; + $output[$i]['countryName'] = $locale->getText('countries.'.strtolower($record['country']['iso_code']), $locale->getText('locale.country.unknown')); } else { $output[$i]['countryCode'] = '--'; $output[$i]['countryName'] = $locale->getText('locale.country.unknown'); @@ -654,15 +656,16 @@ App::put('/v1/database/collections/:collectionId') } catch (AuthorizationException $exception) { throw new Exception('Unauthorized permissions', 401); } catch (StructureException $exception) { - throw new Exception('Bad structure. ' . $exception->getMessage(), 400); - } + throw new Exception('Bad structure. '.$exception->getMessage(), 400); + } $usage->setParam('database.collections.update', 1); $audits ->setParam('event', 'database.collections.update') - ->setParam('resource', 'database/collection/' . $collection->getId()) - ->setParam('data', $collection->getArrayCopy()); + ->setParam('resource', 'database/collection/'.$collection->getId()) + ->setParam('data', $collection->getArrayCopy()) + ; $response->dynamic($collection, Response::MODEL_COLLECTION); }); @@ -703,12 +706,14 @@ App::delete('/v1/database/collections/:collectionId') $usage->setParam('database.collections.delete', 1); $events - ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)); + ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)) + ; $audits ->setParam('event', 'database.collections.delete') - ->setParam('resource', 'database/collection/' . $collection->getId()) - ->setParam('data', $collection->getArrayCopy()); + ->setParam('resource', 'database/collection/'.$collection->getId()) + ->setParam('data', $collection->getArrayCopy()) + ; $response->noContent(); }); @@ -791,7 +796,7 @@ App::post('/v1/database/collections/:collectionId/attributes/email') 'required' => $required, 'default' => $default, 'array' => $array, - 'format' => \json_encode(['name' => 'email']), + 'format' => \json_encode(['name'=>'email']), ]), $response, $dbForExternal, $database, $audits, $usage); }); @@ -832,7 +837,7 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') 'required' => $required, 'default' => $default, 'array' => $array, - 'format' => \json_encode(['name' => 'ip']), + 'format' => \json_encode(['name'=>'ip']), ]), $response, $dbForExternal, $database, $audits, $usage); }); @@ -874,7 +879,7 @@ App::post('/v1/database/collections/:collectionId/attributes/url') 'required' => $required, 'default' => $default, 'array' => $array, - 'format' => \json_encode(['name' => 'url']), + 'format' => \json_encode(['name'=>'url']), ]), $response, $dbForExternal, $database, $audits, $usage); }); @@ -909,7 +914,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Stats\Stats $usage */ - + return $attributesCallback(new Document([ '$collection' => $collectionId, '$id' => $attributeId, @@ -919,7 +924,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') 'default' => $default, 'array' => $array, 'format' => \json_encode([ - 'name' => 'int-range', + 'name'=>'int-range', 'min' => $min, 'max' => $max, ]), @@ -966,7 +971,7 @@ App::post('/v1/database/collections/:collectionId/attributes/float') 'default' => $default, 'array' => $array, 'format' => \json_encode([ - 'name' => 'float-range', + 'name'=>'float-range', 'min' => $min, 'max' => $max, ]), @@ -1096,7 +1101,7 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId') ])]); $usage->setParam('database.collections.read', 1); - + $response->dynamic($attribute, Response::MODEL_ATTRIBUTE); }); @@ -1151,17 +1156,20 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') $database ->setParam('type', DATABASE_TYPE_DELETE_ATTRIBUTE) - ->setParam('document', $attribute); + ->setParam('document', $attribute) + ; $usage->setParam('database.collections.update', 1); $events - ->setParam('payload', $response->output($attribute, Response::MODEL_ATTRIBUTE)); + ->setParam('payload', $response->output($attribute, Response::MODEL_ATTRIBUTE)) + ; $audits ->setParam('event', 'database.attributes.delete') - ->setParam('resource', 'database/collection/' . $collection->getId()) - ->setParam('data', $attribute->getArrayCopy()); + ->setParam('resource', 'database/collection/'.$collection->getId()) + ->setParam('data', $attribute->getArrayCopy()) + ; $response->noContent(); }); @@ -1242,17 +1250,20 @@ App::post('/v1/database/collections/:collectionId/indexes') $database ->setParam('type', DATABASE_TYPE_CREATE_INDEX) - ->setParam('document', $index); + ->setParam('document', $index) + ; $usage->setParam('database.collections.update', 1); $audits ->setParam('event', 'database.indexes.create') - ->setParam('resource', 'database/collection/' . $collection->getId()) - ->setParam('data', $index->getArrayCopy()); + ->setParam('resource', 'database/collection/'.$collection->getId()) + ->setParam('data', $index->getArrayCopy()) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($index, Response::MODEL_INDEX); + }); App::get('/v1/database/collections/:collectionId/indexes') @@ -1338,7 +1349,7 @@ App::get('/v1/database/collections/:collectionId/indexes/:indexId') ])]); $usage->setParam('database.collections.read', 1); - + $response->dynamic($index, Response::MODEL_INDEX); }); @@ -1379,7 +1390,7 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') $indexes = $collection->getAttribute('indexes'); // find attribute in collection - $index = null; + $index= null; foreach ($indexes as $i) { if ($i->getId() === $indexId) { $index = $i->setAttribute('$collection', $collectionId); // set the collectionId @@ -1393,17 +1404,20 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') $database ->setParam('type', DATABASE_TYPE_DELETE_INDEX) - ->setParam('document', $index); + ->setParam('document', $index) + ; $usage->setParam('database.collections.update', 1); $events - ->setParam('payload', $response->output($index, Response::MODEL_INDEX)); + ->setParam('payload', $response->output($index, Response::MODEL_INDEX)) + ; $audits ->setParam('event', 'database.indexes.delete') - ->setParam('resource', 'database/collection/' . $collection->getId()) - ->setParam('data', $index->getArrayCopy()); + ->setParam('resource', 'database/collection/'.$collection->getId()) + ->setParam('data', $index->getArrayCopy()) + ; $response->noContent(); }); @@ -1436,7 +1450,7 @@ App::post('/v1/database/collections/:collectionId/documents') /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Stats\Stats $usage */ - + $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array if (empty($data)) { @@ -1446,7 +1460,7 @@ App::post('/v1/database/collections/:collectionId/documents') if (isset($data['$id'])) { throw new Exception('$id is not allowed for creating new documents, try update instead', 400); } - + $collection = $dbForExternal->getCollection($collectionId); if ($collection->isEmpty()) { @@ -1455,23 +1469,26 @@ App::post('/v1/database/collections/:collectionId/documents') $data['$collection'] = $collection->getId(); // Adding this param to make API easier for developers $data['$id'] = $documentId == 'unique()' ? $dbForExternal->getId() : $documentId; - $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user - $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? []; // By default set write permissions for user + $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? []; // By default set read permissions for user + $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? []; // By default set write permissions for user try { $document = $dbForExternal->createDocument($collectionId, new Document($data)); - } catch (StructureException $exception) { + } + catch (StructureException $exception) { throw new Exception($exception->getMessage(), 400); } $usage ->setParam('database.documents.create', 1) - ->setParam('collectionId', $collectionId); + ->setParam('collectionId', $collectionId) + ; $audits ->setParam('event', 'database.documents.create') - ->setParam('resource', 'database/document/' . $document->getId()) - ->setParam('data', $document->getArrayCopy()); + ->setParam('resource', 'database/document/'.$document->getId()) + ->setParam('data', $document->getArrayCopy()) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($document, Response::MODEL_DOCUMENT); @@ -1535,8 +1552,9 @@ App::get('/v1/database/collections/:collectionId/documents') $usage ->setParam('database.documents.read', 1) - ->setParam('collectionId', $collectionId); - + ->setParam('collectionId', $collectionId) + ; + $response->dynamic(new Document([ 'sum' => $dbForExternal->count($collectionId, $queries, APP_LIMIT_COUNT), 'documents' => $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null), @@ -1577,7 +1595,8 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.read', 1) - ->setParam('collectionId', $collectionId); + ->setParam('collectionId', $collectionId) + ; $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -1626,7 +1645,7 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') if (empty($data)) { throw new Exception('Missing payload', 400); } - + if (!\is_array($data)) { throw new Exception('Data param should be a valid JSON object', 400); } @@ -1640,20 +1659,24 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') try { $document = $dbForExternal->updateDocument($collection->getId(), $document->getId(), new Document($data)); - } catch (AuthorizationException $exception) { - throw new Exception('Unauthorized permissions', 401); - } catch (StructureException $exception) { - throw new Exception('Bad structure. ' . $exception->getMessage(), 400); } - + catch (AuthorizationException $exception) { + throw new Exception('Unauthorized permissions', 401); + } + catch (StructureException $exception) { + throw new Exception('Bad structure. '.$exception->getMessage(), 400); + } + $usage ->setParam('database.documents.update', 1) - ->setParam('collectionId', $collectionId); + ->setParam('collectionId', $collectionId) + ; $audits ->setParam('event', 'database.documents.update') - ->setParam('resource', 'database/document/' . $document->getId()) - ->setParam('data', $document->getArrayCopy()); + ->setParam('resource', 'database/document/'.$document->getId()) + ->setParam('data', $document->getArrayCopy()) + ; $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -1699,16 +1722,18 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.delete', 1) - ->setParam('collectionId', $collectionId); + ->setParam('collectionId', $collectionId) + ; $events - ->setParam('eventData', $response->output($document, Response::MODEL_DOCUMENT)); + ->setParam('eventData', $response->output($document, Response::MODEL_DOCUMENT)) + ; $audits ->setParam('event', 'database.documents.delete') - ->setParam('resource', 'database/document/' . $document->getId()) + ->setParam('resource', 'database/document/'.$document->getId()) ->setParam('data', $document->getArrayCopy()) // Audit document in case of malicious or disastrous action ; $response->noContent(); - }); \ No newline at end of file + }); From cd3939781ecd0c2cb136094f3d0df9244a5b1e51 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 24 Aug 2021 14:21:46 +0545 Subject: [PATCH 065/120] fix session provider usage --- app/tasks/usage.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index a6c9f3edac..fdbf1d3eb2 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -41,6 +41,7 @@ use Utopia\Database\Validator\Authorization; * users.update * users.delete * users.sessions.create + * users.sessions.{provider}.create * users.sessions.delete * * Functions @@ -163,6 +164,9 @@ $cli ], 'users.sessions.create' => [ 'table' => 'appwrite_usage_users_sessions_create', + ], + 'users.sessions.provider.create' => [ + 'table' => 'appwrite_usage_users_sessions_create', 'groupBy' => 'provider', ], 'users.sessions.delete' => [ From 218b332b0f92fef229f0970c6961faac8992adff Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 24 Aug 2021 15:36:45 +0545 Subject: [PATCH 066/120] fix get url --- app/controllers/shared/api.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index fc59d100ef..b7953da491 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -104,7 +104,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $events, $aud ->setParam('httpRequest', 1) ->setParam('httpUrl', $request->getHostname().$request->getURI()) ->setParam('httpMethod', $request->getMethod()) - ->setParam('httpPath', $route->getURL()) + ->setParam('httpPath', $route->getPath()) ->setParam('networkRequestSize', 0) ->setParam('networkResponseSize', 0) ->setParam('storage', 0) From 80fffe0ed49d85ffefc224c4458273c134a857a4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 24 Aug 2021 16:15:24 +0545 Subject: [PATCH 067/120] collections count --- app/controllers/api/projects.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 7d8f4ed54c..7042281679 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -290,6 +290,9 @@ App::get('/v1/projects/:projectId/usage') $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; + + $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.collections.count'])], 0, ['time'], [Database::ORDER_DESC]); + $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; $storageTotal = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.total'])], 0, ['time'], [Database::ORDER_DESC]); $storage = $storageTotal ? $storageTotal->getAttribute('value', 0) : 0; @@ -320,6 +323,10 @@ App::get('/v1/projects/:projectId/usage') 'data' => [], 'total' => $documentsTotal, ], + 'collections' => [ + 'data' => [], + 'total' => $collectionsTotal, + ], 'users' => [ 'data' => [], 'total' => $usersTotal, From 1af4b02034da811417ab5a551d91d8e50579ad2b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 25 Aug 2021 11:44:12 +0545 Subject: [PATCH 068/120] fix warning --- src/Appwrite/Stats/Stats.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index e7720088f5..0cdd19fee3 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -175,7 +175,7 @@ class Stats } if ($storage >= 1) { - $tags = ",projectId={$projectId},bucketId={($this->params['bucketId'] ?? '')}"; + $tags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? ''); $this->statsd->count('storage.all' . $tags, $storage); } From 95a20159a7684fe4a2cedbe354cf90cc1157ddb6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 25 Aug 2021 12:09:02 +0545 Subject: [PATCH 069/120] fix merge errors --- app/controllers/api/database.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 9dd7e712e8..d403a85084 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -186,7 +186,6 @@ App::get('/v1/database/collections') ->param('after', '', new UID(), 'ID of the collection used as the starting point for the query, excluding the collection itself. Should be used for efficient pagination when working with large sets of data.', true) ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') - ->inject('dbForExternal') ->inject('dbForInternal') ->inject('usage') ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) { @@ -228,7 +227,6 @@ App::get('/v1/database/collections/:collectionId') ->label('sdk.response.model', Response::MODEL_COLLECTION) ->param('collectionId', '', new UID(), 'Collection unique ID.') ->inject('response') - ->inject('dbForExternal') ->inject('dbForInternal') ->inject('usage') ->action(function ($collectionId, $response, $dbForInternal, $usage) { From 7775daa478a4d1f30cff6f3252d6d949f14ad568 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Aug 2021 16:30:03 +0530 Subject: [PATCH 070/120] feat(usage): added new response model --- app/controllers/api/database.php | 2 - composer.json | 2 +- composer.lock | 14 ++--- docker-compose.yml | 4 +- src/Appwrite/Utopia/Response.php | 1 + src/Appwrite/Utopia/Response/Model/Metric.php | 53 +++++++++++++++++++ 6 files changed, 64 insertions(+), 12 deletions(-) create mode 100644 src/Appwrite/Utopia/Response/Model/Metric.php diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 2a85899a25..edbb9aa5a6 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1,8 +1,6 @@ addRule('message', [ + 'type' => self::TYPE_STRING, + 'description' => 'Error message.', + 'default' => '', + 'example' => 'Not found', + ]) + ->addRule('code', [ + 'type' => self::TYPE_STRING, + 'description' => 'Error code.', + 'default' => '', + 'example' => '404', + ]) + ->addRule('version', [ + 'type' => self::TYPE_STRING, + 'description' => 'Server version number.', + 'default' => '', + 'example' => '1.0', + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'Metric'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_METRIC; + } +} \ No newline at end of file From 3d512e74bf61119ba4bc9aa2f2c2cb1c2fc7bc05 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Aug 2021 23:45:36 +0530 Subject: [PATCH 071/120] feat(usage): added response models for database api --- app/controllers/api/database.php | 157 +++++------------- app/controllers/api/storage.php | 27 ++- src/Appwrite/Utopia/Response.php | 7 + .../Utopia/Response/Model/CollectionUsage.php | 77 +++++++++ .../Utopia/Response/Model/DatabaseUsage.php | 112 +++++++++++++ src/Appwrite/Utopia/Response/Model/Metric.php | 26 ++- 6 files changed, 269 insertions(+), 137 deletions(-) create mode 100644 src/Appwrite/Utopia/Response/Model/CollectionUsage.php create mode 100644 src/Appwrite/Utopia/Response/Model/DatabaseUsage.php diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index edbb9aa5a6..a18c2e885c 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -245,13 +245,16 @@ App::get('/v1/database/collections/:collectionId') $response->dynamic($collection, Response::MODEL_COLLECTION); }); - App::get('/v1/database/usage') +App::get('/v1/database/usage') ->desc('Get usage stats for the database') ->groups(['api', 'database']) ->label('scope', 'collections.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'database') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_DATABASE_USAGE) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') @@ -261,11 +264,11 @@ App::get('/v1/database/collections/:collectionId') /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Registry\Registry $register */ - $stats = []; + $usage = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '30m', + 'period' => '15m', 'limit' => 48, ], '7d' => [ @@ -285,6 +288,8 @@ App::get('/v1/database/collections/:collectionId') Authorization::disable(); $metrics = [ + 'database.documents.count', + 'database.collections.count', 'database.collections.create', 'database.collections.read', 'database.collections.update', @@ -295,6 +300,7 @@ App::get('/v1/database/collections/:collectionId') 'database.documents.delete' ]; + $stats = []; foreach ($metrics as $metric) { $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), @@ -312,85 +318,24 @@ App::get('/v1/database/collections/:collectionId') $stats[$metric] = array_reverse($stats[$metric]); } - $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); - $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; - - $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.collections.count'])], 0, ['time'], [Database::ORDER_DESC]); - $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; - Authorization::reset(); - $documentsCreate = $stats["database.documents.create"] ?? []; - $documentsRead = $stats["database.documents.read"] ?? []; - $documentsUpdate = $stats["database.documents.update"] ?? []; - $documentsDelete = $stats["database.documents.delete"] ?? []; - $collectionsCreate = $stats["database.collections.create"] ?? []; - $collectionsRead = $stats["database.collections.read"] ?? []; - $collectionsUpdate = $stats["database.collections.update"] ?? []; - $collectionsDelete = $stats["database.collections.delete"] ?? []; - - $response->json([ + $usage = new Document([ 'range' => $range, - 'documents.create' => [ - 'data' => $documentsCreate, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $documentsCreate)), - ], - 'documents.read' => [ - 'data' => $documentsRead, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $documentsRead)), - ], - 'documents.update' => [ - 'data' => $documentsUpdate, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $documentsUpdate)), - ], - 'documents.delete' => [ - 'data' => $documentsDelete, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $documentsDelete)), - ], - 'collections.create' => [ - 'data' => $collectionsCreate, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $collectionsCreate)), - ], - 'collections.read' => [ - 'data' => $collectionsRead, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $collectionsRead)), - ], - 'collections.update' => [ - 'data' => $collectionsUpdate, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $collectionsUpdate)), - ], - 'collections.delete' => [ - 'data' => $collectionsDelete, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $collectionsDelete)), - ], - 'documentCount' => [ - 'data' => [], - 'total' => $documentsTotal, - ], - 'collectionCount' => [ - 'data' => [], - 'total' => $collectionsTotal, - ] + 'documents.count' => $stats["database.documents.count"], + 'collections.count' => $stats["database.collections.count"], + 'documents.create' => $stats["database.documents.create"], + 'documents.read' => $stats["database.documents.read"], + 'documents.update' => $stats["database.documents.update"], + 'documents.delete' => $stats["database.documents.delete"], + 'collections.create' => $stats["database.collections.create"], + 'collections.read' => $stats["database.collections.read"], + 'collections.update' => $stats["database.collections.update"], + 'collections.delete' => $stats["database.collections.delete"], ]); - } else { - $response->json([]); } + + $response->dynamic($usage, Response::MODEL_DATABASE_USAGE); }); App::get('/v1/database/:collectionId/usage') @@ -400,6 +345,9 @@ App::get('/v1/database/:collectionId/usage') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'database') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_COLLECTION_USAGE) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->param('collectionId', '', new UID(), 'Collection unique ID.') ->inject('response') @@ -417,11 +365,11 @@ App::get('/v1/database/:collectionId/usage') throw new Exception('Collection not found', 404); } - $stats = []; + $usage = []; if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '30m', + 'period' => '15m', 'limit' => 48, ], '7d' => [ @@ -441,12 +389,14 @@ App::get('/v1/database/:collectionId/usage') Authorization::disable(); $metrics = [ + "database.collections.$collectionId.documents.count", "database.collections.$collectionId.documents.create", "database.collections.$collectionId.documents.read", "database.collections.$collectionId.documents.update", "database.collections.$collectionId.documents.delete", ]; + $stats = []; foreach ($metrics as $metric) { $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), @@ -460,54 +410,22 @@ App::get('/v1/database/:collectionId/usage') 'date' => $requestDoc->getAttribute('time'), ]; } - $stats[$metric] = array_reverse($stats[$metric]); } - - $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ["database.collections.$collectionId.documents.count"])], 0, ['time'], [Database::ORDER_DESC]); - $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; Authorization::reset(); - - $create = $stats["database.collections.$collectionId.documents.create"] ?? []; - $read = $stats["database.collections.$collectionId.documents.read"] ?? []; - $update = $stats["database.collections.$collectionId.documents.update"] ?? []; - $delete = $stats["database.collections.$collectionId.documents.delete"] ?? []; - $response->json([ + $usage = new Document([ 'range' => $range, - 'create' => [ - 'data' => $create, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $create)), - ], - 'read' => [ - 'data' => $read, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $read)), - ], - 'update' => [ - 'data' => $update, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $update)), - ], - 'delete' => [ - 'data' => $delete, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $delete)), - ], - 'documentCount' => [ - 'data' => [], - 'total' => $documentsTotal, - ], + 'documents.count' => $stats["database.collections.$collectionId.documents.count"], + 'documents.create' => $stats["database.collections.$collectionId.documents.create"], + 'documents.read' => $stats["database.collections.$collectionId.documents.read"], + 'documents.update' => $stats["database.collections.$collectionId.documents.update"], + 'documents.delete' => $stats["database.collections.$collectionId.documents.delete"] ]); - } else { - $response->json([]); } + + $response->dynamic($usage, Response::MODEL_COLLECTION_USAGE); }); App::get('/v1/database/collections/:collectionId/logs') @@ -1458,6 +1376,7 @@ App::post('/v1/database/collections/:collectionId/documents') $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? []; // By default set read permissions for user $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? []; // By default set write permissions for user + var_dump($collectionId); try { $document = $dbForExternal->createDocument($collectionId, new Document($data)); } diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 3b433988e0..5589dbe95f 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -647,7 +647,7 @@ App::delete('/v1/storage/files/:fileId') App::get('/v1/storage/usage') ->desc('Get usage stats for storage') ->groups(['api', 'storage']) - ->label('scope', 'storage.read') + ->label('scope', 'files.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getUsage') @@ -705,7 +705,7 @@ App::get('/v1/storage/usage') App::get('/v1/storage/:bucketId/usage') ->desc('Get usage stats for a storage bucket') ->groups(['api', 'storage']) - ->label('scope', 'storage.read') + ->label('scope', 'files.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getUsage') @@ -772,6 +772,29 @@ App::get('/v1/storage/:bucketId/usage') $update = $stats["storage.buckets.$bucketId.files.update"] ?? []; $delete = $stats["storage.buckets.$bucketId.files.delete"] ?? []; + + // 'name' => [ + // [ + // [ + // 'value' => '', + // 'date' => 'unix timestamp' + // ] + // ] + // ] + + // $res = [ + // 'range' => '', + // 'name' => [ + // [ + // 'value' => , + // 'date' => + // ] + // ], + // 'nameTotal' => [ + + // ] + // ]; + $response->json([ 'range' => $range, 'create' => [ diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index f7217a5155..44799fdaa2 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -13,9 +13,11 @@ use Appwrite\Utopia\Response\Model\Any; use Appwrite\Utopia\Response\Model\Attribute; use Appwrite\Utopia\Response\Model\BaseList; use Appwrite\Utopia\Response\Model\Collection; +use Appwrite\Utopia\Response\Model\CollectionUsage; use Appwrite\Utopia\Response\Model\Continent; use Appwrite\Utopia\Response\Model\Country; use Appwrite\Utopia\Response\Model\Currency; +use Appwrite\Utopia\Response\Model\DatabaseUsage; use Appwrite\Utopia\Response\Model\Document as ModelDocument; use Appwrite\Utopia\Response\Model\Domain; use Appwrite\Utopia\Response\Model\Error; @@ -57,6 +59,7 @@ class Response extends SwooleResponse const MODEL_LOG_LIST = 'logList'; const MODEL_ERROR = 'error'; const MODEL_METRIC = 'metric'; + const MODEL_METRIC_LIST = 'metricList'; const MODEL_ERROR_DEV = 'errorDev'; const MODEL_BASE_LIST = 'baseList'; @@ -69,6 +72,8 @@ class Response extends SwooleResponse const MODEL_INDEX_LIST = 'indexList'; const MODEL_DOCUMENT = 'document'; const MODEL_DOCUMENT_LIST = 'documentList'; + const MODEL_DATABASE_USAGE = 'databaseUsage'; + const MODEL_COLLECTION_USAGE = 'collectionUsage'; // Users const MODEL_USER = 'user'; @@ -183,6 +188,8 @@ class Response extends SwooleResponse ->setModel(new Attribute()) ->setModel(new Index()) ->setModel(new ModelDocument()) + ->setModel(new DatabaseUsage()) + ->setModel(new CollectionUsage()) ->setModel(new Log()) ->setModel(new User()) ->setModel(new Preferences()) diff --git a/src/Appwrite/Utopia/Response/Model/CollectionUsage.php b/src/Appwrite/Utopia/Response/Model/CollectionUsage.php new file mode 100644 index 0000000000..2281b3d032 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/CollectionUsage.php @@ -0,0 +1,77 @@ +addRule('range', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The value of this metric at a timestamp.', + 'default' => 0, + 'example' => 1, + ]) + ->addRule('documents.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of documents.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents read.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents updated.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents deleted.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'CollectionUsage'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_COLLECTION_USAGE; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php b/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php new file mode 100644 index 0000000000..58d9575d70 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php @@ -0,0 +1,112 @@ +addRule('range', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The value of this metric at a timestamp.', + 'default' => 0, + 'example' => 1, + ]) + ->addRule('documents.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of documents.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of collections.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents read.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents updated.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents deleted.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections read.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections updated.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections delete.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'DatabaseUsage'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_DATABASE_USAGE; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/Metric.php b/src/Appwrite/Utopia/Response/Model/Metric.php index 73625eb1f3..26038af081 100644 --- a/src/Appwrite/Utopia/Response/Model/Metric.php +++ b/src/Appwrite/Utopia/Response/Model/Metric.php @@ -10,23 +10,17 @@ class Metric extends Model public function __construct() { $this - ->addRule('message', [ - 'type' => self::TYPE_STRING, - 'description' => 'Error message.', - 'default' => '', - 'example' => 'Not found', + ->addRule('value', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The value of this metric at the timestamp.', + 'default' => -1, + 'example' => 1, ]) - ->addRule('code', [ - 'type' => self::TYPE_STRING, - 'description' => 'Error code.', - 'default' => '', - 'example' => '404', - ]) - ->addRule('version', [ - 'type' => self::TYPE_STRING, - 'description' => 'Server version number.', - 'default' => '', - 'example' => '1.0', + ->addRule('timestamp', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The UNIX timestamp at which this metric was aggregated.', + 'default' => 0, + 'example' => 1592981250 ]) ; } From 6d6b0ee02418fe6ea7d70eb8b562ef46d256f023 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 27 Aug 2021 00:14:32 +0530 Subject: [PATCH 072/120] feat(usage): added response models for storage API --- app/controllers/api/storage.php | 212 ++++++++---------- src/Appwrite/Utopia/Response.php | 6 + .../Utopia/Response/Model/BucketsUsage.php | 70 ++++++ .../Utopia/Response/Model/CollectionUsage.php | 8 +- .../Utopia/Response/Model/DatabaseUsage.php | 8 +- .../Utopia/Response/Model/StorageUsage.php | 56 +++++ 6 files changed, 230 insertions(+), 130 deletions(-) create mode 100644 src/Appwrite/Utopia/Response/Model/BucketsUsage.php create mode 100644 src/Appwrite/Utopia/Response/Model/StorageUsage.php diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 5589dbe95f..69a7c18708 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -651,6 +651,9 @@ App::get('/v1/storage/usage') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_STORAGE_USAGE) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') @@ -658,71 +661,11 @@ App::get('/v1/storage/usage') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + $usage = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, - ], - '7d' => [ - 'period' => '1d', - 'limit' => 7, - ], - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ], - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; - - Authorization::disable(); - - $storageTotal = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.total'])], 0, ['time'], [Database::ORDER_DESC]); - $storage = $storageTotal ? $storageTotal->getAttribute('value', 0) : 0; - - $filesCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.files.count'])], 0, ['time'], [Database::ORDER_DESC]); - $filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0; - - Authorization::reset(); - - $response->json([ - 'range' => $range, - 'storage' => [ - 'total' => $storage, - ], - 'files' => [ - 'total' => $filesTotal, - ], - ]); - } else { - $response->json([]); - } - }); - -App::get('/v1/storage/:bucketId/usage') - ->desc('Get usage stats for a storage bucket') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) - ->label('sdk.namespace', 'storage') - ->label('sdk.method', 'getUsage') - ->param('bucketId', '', new UID(), 'Bucket unique ID.') - ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) - ->inject('response') - ->inject('dbForInternal') - ->action(function ($bucketId, $range, $response, $dbForInternal) { - /** @var Appwrite\Utopia\Response $response */ - /** @var Utopia\Database\Database $dbForInternal */ - - // TODO: Check if the storage bucket exists else throw 404 - $stats = []; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $period = [ - '24h' => [ - 'period' => '30m', + 'period' => '15m', 'limit' => 48, ], '7d' => [ @@ -742,12 +685,11 @@ App::get('/v1/storage/:bucketId/usage') Authorization::disable(); $metrics = [ - "storage.buckets.$bucketId.files.create", - "storage.buckets.$bucketId.files.read", - "storage.buckets.$bucketId.files.update", - "storage.buckets.$bucketId.files.delete" + "storage.total", + "storage.files.count" ]; + $stats = []; foreach ($metrics as $metric) { $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), @@ -761,68 +703,94 @@ App::get('/v1/storage/:bucketId/usage') 'date' => $requestDoc->getAttribute('time'), ]; } - $stats[$metric] = array_reverse($stats[$metric]); } + $usage = new Document([ + 'range' => $range, + 'storage' => $stats['storage.total'], + 'files' => $stats['storage.files.count'] + ]); + } + + $response->dynamic($usage, Response::MODEL_STORAGE_USAGE); + }); + +App::get('/v1/storage/:bucketId/usage') + ->desc('Get usage stats for a storage bucket') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'storage') + ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_BUCKETS_USAGE) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForInternal') + ->action(function ($bucketId, $range, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForInternal */ + + // TODO: Check if the storage bucket exists else throw 404 + + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + $metrics = [ + "storage.buckets.$bucketId.files.create", + "storage.buckets.$bucketId.files.read", + "storage.buckets.$bucketId.files.update", + "storage.buckets.$bucketId.files.delete" + ]; + + $stats = []; + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } Authorization::reset(); - $create = $stats["storage.buckets.$bucketId.files.create"] ?? []; - $read = $stats["storage.buckets.$bucketId.files.read"] ?? []; - $update = $stats["storage.buckets.$bucketId.files.update"] ?? []; - $delete = $stats["storage.buckets.$bucketId.files.delete"] ?? []; - - - // 'name' => [ - // [ - // [ - // 'value' => '', - // 'date' => 'unix timestamp' - // ] - // ] - // ] - - // $res = [ - // 'range' => '', - // 'name' => [ - // [ - // 'value' => , - // 'date' => - // ] - // ], - // 'nameTotal' => [ - - // ] - // ]; - - $response->json([ + $usage = new Document([ 'range' => $range, - 'create' => [ - 'data' => $create, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $create)), - ], - 'read' => [ - 'data' => $read, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $read)), - ], - 'update' => [ - 'data' => $update, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $update)), - ], - 'delete' => [ - 'data' => $delete, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $delete)), - ], + 'files.create' => $stats["storage.buckets.$bucketId.files.create"], + 'files.read' => $stats["storage.buckets.$bucketId.files.read"], + 'files.update' => $stats["storage.buckets.$bucketId.files.update"], + 'files.delete' => $stats["storage.buckets.$bucketId.files.delete"] ]); - } else { - $response->json([]); } + + $response->dynamic($usage, Response::MODEL_BUCKETS_USAGE); }); \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 44799fdaa2..811f07cb55 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -12,6 +12,7 @@ use Appwrite\Utopia\Response\Model\None; use Appwrite\Utopia\Response\Model\Any; use Appwrite\Utopia\Response\Model\Attribute; use Appwrite\Utopia\Response\Model\BaseList; +use Appwrite\Utopia\Response\Model\BucketsUsage; use Appwrite\Utopia\Response\Model\Collection; use Appwrite\Utopia\Response\Model\CollectionUsage; use Appwrite\Utopia\Response\Model\Continent; @@ -45,6 +46,7 @@ use Appwrite\Utopia\Response\Model\Token; use Appwrite\Utopia\Response\Model\Webhook; use Appwrite\Utopia\Response\Model\Preferences; use Appwrite\Utopia\Response\Model\Mock; // Keep last +use Appwrite\Utopia\Response\Model\StorageUsage; use stdClass; /** @@ -88,6 +90,8 @@ class Response extends SwooleResponse const MODEL_FILE = 'file'; const MODEL_FILE_LIST = 'fileList'; const MODEL_BUCKET = 'bucket'; // - Missing + const MODEL_BUCKETS_USAGE = 'bucketsUsage'; + const MODEL_STORAGE_USAGE = 'storageUsage'; // Locale const MODEL_LOCALE = 'locale'; @@ -198,6 +202,8 @@ class Response extends SwooleResponse ->setModel(new JWT()) ->setModel(new Locale()) ->setModel(new File()) + ->setModel(new StorageUsage()) + ->setModel(new BucketsUsage()) ->setModel(new Team()) ->setModel(new Membership()) ->setModel(new Func()) diff --git a/src/Appwrite/Utopia/Response/Model/BucketsUsage.php b/src/Appwrite/Utopia/Response/Model/BucketsUsage.php new file mode 100644 index 0000000000..0fb923e147 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/BucketsUsage.php @@ -0,0 +1,70 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('files.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('files.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files read.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('files.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files updated.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('files.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files deleted.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'BucketsUsage'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_BUCKETS_USAGE; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/CollectionUsage.php b/src/Appwrite/Utopia/Response/Model/CollectionUsage.php index 2281b3d032..a0b0a51c9b 100644 --- a/src/Appwrite/Utopia/Response/Model/CollectionUsage.php +++ b/src/Appwrite/Utopia/Response/Model/CollectionUsage.php @@ -12,10 +12,10 @@ class CollectionUsage extends Model { $this ->addRule('range', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'The value of this metric at a timestamp.', - 'default' => 0, - 'example' => 1, + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', ]) ->addRule('documents.count', [ 'type' => Response::MODEL_METRIC_LIST, diff --git a/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php b/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php index 58d9575d70..1c24625004 100644 --- a/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php +++ b/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php @@ -12,10 +12,10 @@ class DatabaseUsage extends Model { $this ->addRule('range', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'The value of this metric at a timestamp.', - 'default' => 0, - 'example' => 1, + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', ]) ->addRule('documents.count', [ 'type' => Response::MODEL_METRIC_LIST, diff --git a/src/Appwrite/Utopia/Response/Model/StorageUsage.php b/src/Appwrite/Utopia/Response/Model/StorageUsage.php new file mode 100644 index 0000000000..e26e0d4b3d --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/StorageUsage.php @@ -0,0 +1,56 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('storage', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for the occupied storage size.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('files', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of files.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'StorageUsage'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_STORAGE_USAGE; + } +} \ No newline at end of file From b893190ce52c7859bcf2d22fb17fefe4e5f8b275 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 27 Aug 2021 00:23:55 +0530 Subject: [PATCH 073/120] feat(usage): added response models for users API --- app/controllers/api/users.php | 77 ++++------------ src/Appwrite/Utopia/Response.php | 3 + .../Utopia/Response/Model/UsersUsage.php | 91 +++++++++++++++++++ 3 files changed, 114 insertions(+), 57 deletions(-) create mode 100644 src/Appwrite/Utopia/Response/Model/UsersUsage.php diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 6d82c4f472..5a0cb03209 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -18,6 +18,7 @@ use Utopia\Database\Exception\Duplicate; use Utopia\Database\Validator\UID; use DeviceDetector\DeviceDetector; use Appwrite\Database\Validator\CustomId; +use Appwrite\Utopia\Response\Model; use Utopia\Database\Database; use Utopia\Database\Query; @@ -616,6 +617,9 @@ App::get('/v1/users/usage') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USERS_USAGE) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') @@ -624,11 +628,11 @@ App::get('/v1/users/usage') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - $stats = []; + $usage = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '30m', + 'period' => '15m', 'limit' => 48, ], '7d' => [ @@ -646,8 +650,8 @@ App::get('/v1/users/usage') ]; Authorization::disable(); - $metrics = [ + "users.count", "users.create", "users.read", "users.update", @@ -656,6 +660,7 @@ App::get('/v1/users/usage') "users.sessions.delete" ]; + $stats = []; foreach ($metrics as $metric) { $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), @@ -672,63 +677,21 @@ App::get('/v1/users/usage') $stats[$metric] = array_reverse($stats[$metric]); } - - $usersCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); - $usersTotal = $usersCount ? $usersCount->getAttribute('value', 0) : 0; - + Authorization::reset(); - $create = $stats["users.create"] ?? []; - $read = $stats["users.read"] ?? []; - $update = $stats["users.update"] ?? []; - $delete = $stats["users.delete"] ?? []; - $sessionsCreate = $stats["users.sessions.create"] ?? []; - $sessionsDelete = $stats["users.sessions.delete"] ?? []; - - $response->json([ + $usage = new Document([ 'range' => $range, - 'users' => [ - 'data' => [], - 'total' => $usersTotal, - ], - 'create' => [ - 'data' => $create, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $create)), - ], - 'read' => [ - 'data' => $read, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $read)), - ], - 'update' => [ - 'data' => $update, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $update)), - ], - 'delete' => [ - 'data' => $delete, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $delete)), - ], - 'sessionsCreate' => [ - 'data' => $sessionsCreate, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $sessionsCreate)), - ], - 'sessionsDelete' => [ - 'data' => $sessionsDelete, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $sessionsDelete)), - ] + 'users.count' => $stats["users.count"], + 'users.create' => $stats["users.create"], + 'users.read' => $stats["users.read"], + 'users.update' => $stats["users.update"], + 'users.delete' => $stats["users.delete"], + 'sessions.create' => $stats["users.sessions.create"], + 'sessions.delete' => $stats["users.sessions.delete"] ]); - } else { - $response->json([]); + } + + $response->dynamic($usage, Response::MODEL_USERS_USAGE); }); \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 811f07cb55..728e52b9e0 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -47,6 +47,7 @@ use Appwrite\Utopia\Response\Model\Webhook; use Appwrite\Utopia\Response\Model\Preferences; use Appwrite\Utopia\Response\Model\Mock; // Keep last use Appwrite\Utopia\Response\Model\StorageUsage; +use Appwrite\Utopia\Response\Model\UsersUsage; use stdClass; /** @@ -80,6 +81,7 @@ class Response extends SwooleResponse // Users const MODEL_USER = 'user'; const MODEL_USER_LIST = 'userList'; + const MODEL_USERS_USAGE = 'usersUsage'; const MODEL_SESSION = 'session'; const MODEL_SESSION_LIST = 'sessionList'; const MODEL_TOKEN = 'token'; @@ -196,6 +198,7 @@ class Response extends SwooleResponse ->setModel(new CollectionUsage()) ->setModel(new Log()) ->setModel(new User()) + ->setModel(new UsersUsage()) ->setModel(new Preferences()) ->setModel(new Session()) ->setModel(new Token()) diff --git a/src/Appwrite/Utopia/Response/Model/UsersUsage.php b/src/Appwrite/Utopia/Response/Model/UsersUsage.php new file mode 100644 index 0000000000..bb3db30b26 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsersUsage.php @@ -0,0 +1,91 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('users.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of users.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('users.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('users.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users read.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('users.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users updated.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('users.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users deleted.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('sessions.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for sessions created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('sessions.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for sessions deleted.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'UsersUsage'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USERS_USAGE; + } +} \ No newline at end of file From 1e30cdba4b8270f4ec9869ba576e32d574981dae Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 27 Aug 2021 00:42:36 +0530 Subject: [PATCH 074/120] feat(usage): added response models for functions API --- app/controllers/api/functions.php | 165 ++---------------- src/Appwrite/Utopia/Response.php | 3 + .../Utopia/Response/Model/FunctionsUsage.php | 63 +++++++ 3 files changed, 84 insertions(+), 147 deletions(-) create mode 100644 src/Appwrite/Utopia/Response/Model/FunctionsUsage.php diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index ee3088ce18..9359d267f5 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -146,6 +146,9 @@ App::get('/v1/functions/:functionId/usage') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'functions') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_FUNCTIONS_USAGE) ->param('functionId', '', new UID(), 'Function unique ID.') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true) ->inject('response') @@ -164,130 +167,11 @@ App::get('/v1/functions/:functionId/usage') throw new Exception('Function not found', 404); } + $usage = []; if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-24 hours')), - 'end' => DateTime::createFromFormat('U', \strtotime('+1 hour')), - 'group' => '30m', - ], - '7d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-7 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', - ], - '30d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-30 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', - ], - '90d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-90 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', - ], - ]; - - $client = $register->get('influxdb'); - - $executions = []; - $failures = []; - $compute = []; - - if ($client) { - $start = $period[$range]['start']->format(DateTime::RFC3339); - $end = $period[$range]['end']->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); - - // Executions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \''.$start.'\' AND time < \''.$end.'\' AND "metric_type"=\'counter\' AND "project"=\''.$project->getId().'\' AND "functionId"=\''.$function->getId().'\' GROUP BY time('.$period[$range]['group'].') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $executions[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Failures - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \''.$start.'\' AND time < \''.$end.'\' AND "metric_type"=\'counter\' AND "project"=\''.$project->getId().'\' AND "functionId"=\''.$function->getId().'\' AND "functionStatus"=\'failed\' GROUP BY time('.$period[$range]['group'].') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $failures[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Compute - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_time" WHERE time > \''.$start.'\' AND time < \''.$end.'\' AND "metric_type"=\'counter\' AND "project"=\''.$project->getId().'\' AND "functionId"=\''.$function->getId().'\' GROUP BY time('.$period[$range]['group'].') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $compute[] = [ - 'value' => round((!empty($point['value'])) ? $point['value'] / 1000 : 0, 2), // minutes - 'date' => \strtotime($point['time']), - ]; - } - } - - $response->json([ - 'range' => $range, - 'executions' => [ - 'data' => $executions, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $executions)), - ], - 'failures' => [ - 'data' => $failures, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $failures)), - ], - 'compute' => [ - 'data' => $compute, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $compute)), - ], - ]); - } else { - $response->json([]); - } - }); - -App::get('/v1/functions/:functionId/usage2') - ->desc('Get Function Usage') - ->groups(['api', 'functions']) - ->label('scope', 'functions.read') - ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) - ->label('sdk.namespace', 'functions') - ->label('sdk.method', 'getUsage') - ->param('functionId', '', new UID(), 'Function unique ID.') - ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true) - ->inject('response') - ->inject('project') - ->inject('dbForInternal') - ->inject('register') - ->action(function ($functionId, $range, $response, $project, $dbForInternal, $register) { - /** @var Appwrite\Utopia\Response $response */ - /** @var Utopia\Database\Document $project */ - /** @var Utopia\Database\Database $dbForInternal */ - /** @var Utopia\Registry\Registry $register */ - - $function = $dbForInternal->getDocument('functions', $functionId); - - if ($function->isEmpty()) { - throw new Exception('Function not found', 404); - } - - if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $period = [ - '24h' => [ - 'period' => '30m', + 'period' => '15m', 'limit' => 48, ], '7d' => [ @@ -306,7 +190,13 @@ App::get('/v1/functions/:functionId/usage2') Authorization::disable(); - $metrics = ["functions.$functionId.executions", "functions.$functionId.failures", "functions.$functionId.compute"]; + $metrics = [ + "functions.$functionId.executions", + "functions.$functionId.failures", + "functions.$functionId.compute" + ]; + + $stats = []; foreach ($metrics as $metric) { $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), @@ -325,35 +215,16 @@ App::get('/v1/functions/:functionId/usage2') } Authorization::reset(); - - $executions = $stats["functions.$functionId.executions"] ?? []; - $failures = $stats["functions.$functionId.failures"] ?? []; - $compute = $stats["functions.$functionId.compute"] ?? []; - $response->json([ + $usage = new Document([ 'range' => $range, - 'executions' => [ - 'data' => $executions, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $executions)), - ], - 'failures' => [ - 'data' => $failures, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $failures)), - ], - 'compute' => [ - 'data' => $compute, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $compute)), - ], + 'functions.executions' => $stats["functions.$functionId.executions"], + 'functions.failures' => $stats["functions.$functionId.failures"], + 'functions.compute' => $stats["functions.$functionId.compute"] ]); - } else { - $response->json([]); } + + $response->dynamic($usage, Response::MODEL_FUNCTIONS_USAGE); }); App::put('/v1/functions/:functionId') diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 728e52b9e0..a1d0d404ed 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -26,6 +26,7 @@ use Appwrite\Utopia\Response\Model\ErrorDev; use Appwrite\Utopia\Response\Model\Execution; use Appwrite\Utopia\Response\Model\File; use Appwrite\Utopia\Response\Model\Func; +use Appwrite\Utopia\Response\Model\FunctionsUsage; use Appwrite\Utopia\Response\Model\Index; use Appwrite\Utopia\Response\Model\JWT; use Appwrite\Utopia\Response\Model\Key; @@ -117,6 +118,7 @@ class Response extends SwooleResponse // Functions const MODEL_FUNCTION = 'function'; const MODEL_FUNCTION_LIST = 'functionList'; + const MODEL_FUNCTIONS_USAGE = 'functionsUsage'; const MODEL_TAG = 'tag'; const MODEL_TAG_LIST = 'tagList'; const MODEL_EXECUTION = 'execution'; @@ -210,6 +212,7 @@ class Response extends SwooleResponse ->setModel(new Team()) ->setModel(new Membership()) ->setModel(new Func()) + ->setModel(new FunctionsUsage()) ->setModel(new Tag()) ->setModel(new Execution()) ->setModel(new Project()) diff --git a/src/Appwrite/Utopia/Response/Model/FunctionsUsage.php b/src/Appwrite/Utopia/Response/Model/FunctionsUsage.php new file mode 100644 index 0000000000..60f017705e --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/FunctionsUsage.php @@ -0,0 +1,63 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('functions.executions', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function executions.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('functions.failures', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function execution failures.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('functions.compute', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function execution duration.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'FunctionsUsage'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_FUNCTIONS_USAGE; + } +} \ No newline at end of file From 038aeb01fb39d41b3886c81452174f83f4c0f788 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 27 Aug 2021 01:01:37 +0530 Subject: [PATCH 075/120] feat(usage): added response models for projects API --- app/controllers/api/projects.php | 85 +++++++---------- src/Appwrite/Utopia/Response.php | 3 + .../Utopia/Response/Model/ProjectUsage.php | 91 +++++++++++++++++++ 3 files changed, 125 insertions(+), 54 deletions(-) create mode 100644 src/Appwrite/Utopia/Response/Model/ProjectUsage.php diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index d4a8f86846..e8551c29ce 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -223,6 +223,9 @@ App::get('/v1/projects/:projectId/usage') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'projects') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_PROJECT_USAGE) ->param('projectId', '', new UID(), 'Project unique ID.') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') @@ -241,11 +244,11 @@ App::get('/v1/projects/:projectId/usage') throw new Exception('Project not found', 404); } - $stats = []; + $usage = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '30m', + 'period' => '15m', 'limit' => 48, ], '7d' => [ @@ -266,7 +269,17 @@ App::get('/v1/projects/:projectId/usage') Authorization::disable(); - $metrics = ['requests', 'network', 'executions']; + $metrics = [ + 'requests', + 'network', + 'executions', + 'users.count', + 'database.documents.count', + 'database.collections.count', + 'storage.total' + ]; + + $stats = []; foreach ($metrics as $metric) { $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), @@ -282,59 +295,23 @@ App::get('/v1/projects/:projectId/usage') } $stats[$metric] = array_reverse($stats[$metric]); - } + } + + Authorization::reset(); + + $usage = new Document([ + 'range' => $range, + 'requests' => $stats['requests'], + 'network' => $stats['network'], + 'functions' => $stats['executions'], + 'documents' => $stats['database.documents.count'], + 'collections' => $stats['database.collections.count'], + 'users' => $stats['users.count'], + 'storage' => $stats['storage.total'] + ]); } - $usersCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); - $usersTotal = $usersCount ? $usersCount->getAttribute('value', 0) : 0; - - $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); - $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; - - $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.collections.count'])], 0, ['time'], [Database::ORDER_DESC]); - $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; - - $storageTotal = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.total'])], 0, ['time'], [Database::ORDER_DESC]); - $storage = $storageTotal ? $storageTotal->getAttribute('value', 0) : 0; - - Authorization::reset(); - - $response->json([ - 'range' => $range, - 'requests' => [ - 'data' => $stats['requests'] ?? [], - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $stats['requests'] ?? [])), - ], - 'network' => [ - 'data' => \array_map(function ($value) {return ['value' => \round($value['value'] / 1000000, 2), 'date' => $value['date']];}, $stats['network'] ?? []), // convert bytes to mb - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $stats['network'] ?? [])), - ], - 'functions' => [ - 'data' => $stats['executions'] ?? [], - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $stats['executions'] ?? [])), - ], - 'documents' => [ - 'data' => [], - 'total' => $documentsTotal, - ], - 'collections' => [ - 'data' => [], - 'total' => $collectionsTotal, - ], - 'users' => [ - 'data' => [], - 'total' => $usersTotal, - ], - 'storage' => [ - 'total' => $storage, - ], - ]); + $response->dynamic($usage, Response::MODEL_PROJECT_USAGE); }); App::patch('/v1/projects/:projectId') diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index a1d0d404ed..c87c924807 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -47,6 +47,7 @@ use Appwrite\Utopia\Response\Model\Token; use Appwrite\Utopia\Response\Model\Webhook; use Appwrite\Utopia\Response\Model\Preferences; use Appwrite\Utopia\Response\Model\Mock; // Keep last +use Appwrite\Utopia\Response\Model\ProjectUsage; use Appwrite\Utopia\Response\Model\StorageUsage; use Appwrite\Utopia\Response\Model\UsersUsage; use stdClass; @@ -127,6 +128,7 @@ class Response extends SwooleResponse // Project const MODEL_PROJECT = 'project'; const MODEL_PROJECT_LIST = 'projectList'; + const MODEL_PROJECT_USAGE = 'projectUsage'; const MODEL_WEBHOOK = 'webhook'; const MODEL_WEBHOOK_LIST = 'webhookList'; const MODEL_KEY = 'key'; @@ -216,6 +218,7 @@ class Response extends SwooleResponse ->setModel(new Tag()) ->setModel(new Execution()) ->setModel(new Project()) + ->setModel(new ProjectUsage()) ->setModel(new Webhook()) ->setModel(new Key()) ->setModel(new Domain()) diff --git a/src/Appwrite/Utopia/Response/Model/ProjectUsage.php b/src/Appwrite/Utopia/Response/Model/ProjectUsage.php new file mode 100644 index 0000000000..deee3b78b1 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/ProjectUsage.php @@ -0,0 +1,91 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('requests', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of requests.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('network', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for consumed bandwidth.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('functions', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function executions.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of documents.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of collections.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('users', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of users.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('functions', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for the occupied storage size.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'ProjectUsage'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_PROJECT_USAGE; + } +} \ No newline at end of file From d009b8ab89274e368fe9c87cc4284342921447ee Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 27 Aug 2021 17:22:52 +0545 Subject: [PATCH 076/120] delete older usage stats --- app/init.php | 1 + app/tasks/maintenance.php | 11 +++++++++++ app/workers/deletes.php | 26 ++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/app/init.php b/app/init.php index c8261ffd33..475bb3e40d 100644 --- a/app/init.php +++ b/app/init.php @@ -89,6 +89,7 @@ const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_CERTIFICATES = 'certificates'; +const DELETE_TYPE_USAGE_STATS = 'certificates'; // Mail Worker Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_RECOVERY = 'recovery'; diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index eccdf61b17..a5cf63f821 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -39,11 +39,22 @@ $cli ]); } + function notifyDeleteUsageStats(int $interval30m, int $interval1d) + { + Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ + 'type' => DELETE_TYPE_USAGE_STATS, + 'timestamp1d' => $interval1d, + 'timestamp30m' => $interval30m, + ]); + } + // # of days in seconds (1 day = 86400s) $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600'); $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400'); + $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600');//36 hours + $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention){ $time = date('d-m-Y H:i:s', time()); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index f0e66d2b46..fe0037056c 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -72,6 +72,9 @@ class DeletesV1 extends Worker $document = new Document($this->args['document']); $this->deleteCertificates($document); break; + + case DELETE_TYPE_USAGE_STATS: + $this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']); default: Console::error('No delete operation for type: '.$type); @@ -82,6 +85,29 @@ class DeletesV1 extends Worker public function shutdown(): void { } + + /** + * @param int $timestamp1d + * @param int $timestamp30m + */ + protected function deleteUsageStats(int $timestamp1d, int $timestamp30m) { + $this->deleteForProjectIds(function($projectId) use ($timestamp1d, $timestamp30m) { + if (!($dbForInternal = $this->getInternalDB($projectId))) { + throw new Exception('Failed to get projectDB for project '.$projectId); + } + + // Delete Usage stats + $this->deleteByGroup('stats', [ + new Query('time', Query::TYPE_LESSER, [$timestamp1d]), + new Query('period', Query::TYPE_EQUAL, ['1d', '15m']), + ], $dbForInternal); + + $this->deleteByGroup('stats', [ + new Query('time', Query::TYPE_LESSER, [$timestamp30m]), + new Query('period', Query::TYPE_EQUAL, ['30m']), + ], $dbForInternal); + }); + } /** * @param Document $document teams document From ad99f8bd2404a2f505125c0998b8bdab32ba8834 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 27 Aug 2021 22:15:08 +0530 Subject: [PATCH 077/120] feat(usage): rename all usage response models --- app/controllers/api/database.php | 1 - docker-compose.yml | 2 +- src/Appwrite/Utopia/Response.php | 41 ++++++++++--------- .../{BucketsUsage.php => UsageBuckets.php} | 8 ++-- ...ollectionUsage.php => UsageCollection.php} | 8 ++-- .../{DatabaseUsage.php => UsageDatabase.php} | 8 ++-- ...{FunctionsUsage.php => UsageFunctions.php} | 8 ++-- .../{ProjectUsage.php => UsageProject.php} | 8 ++-- .../{StorageUsage.php => UsageStorage.php} | 6 +-- .../Model/{UsersUsage.php => UsageUsers.php} | 6 +-- 10 files changed, 48 insertions(+), 48 deletions(-) rename src/Appwrite/Utopia/Response/Model/{BucketsUsage.php => UsageBuckets.php} (93%) rename src/Appwrite/Utopia/Response/Model/{CollectionUsage.php => UsageCollection.php} (93%) rename src/Appwrite/Utopia/Response/Model/{DatabaseUsage.php => UsageDatabase.php} (96%) rename src/Appwrite/Utopia/Response/Model/{FunctionsUsage.php => UsageFunctions.php} (91%) rename src/Appwrite/Utopia/Response/Model/{ProjectUsage.php => UsageProject.php} (95%) rename src/Appwrite/Utopia/Response/Model/{StorageUsage.php => UsageStorage.php} (92%) rename src/Appwrite/Utopia/Response/Model/{UsersUsage.php => UsageUsers.php} (96%) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index a18c2e885c..86e8304d8b 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1376,7 +1376,6 @@ App::post('/v1/database/collections/:collectionId/documents') $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? []; // By default set read permissions for user $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? []; // By default set write permissions for user - var_dump($collectionId); try { $document = $dbForExternal->createDocument($collectionId, new Document($data)); } diff --git a/docker-compose.yml b/docker-compose.yml index 910c07f094..12bc51b802 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,7 +63,7 @@ services: - ./psalm.xml:/usr/src/code/psalm.xml - ./tests:/usr/src/code/tests - ./app:/usr/src/code/app - - ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database + # - ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database - ./docs:/usr/src/code/docs - ./public:/usr/src/code/public - ./src:/usr/src/code/src diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index c87c924807..0c4c6bf0aa 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -12,13 +12,10 @@ use Appwrite\Utopia\Response\Model\None; use Appwrite\Utopia\Response\Model\Any; use Appwrite\Utopia\Response\Model\Attribute; use Appwrite\Utopia\Response\Model\BaseList; -use Appwrite\Utopia\Response\Model\BucketsUsage; use Appwrite\Utopia\Response\Model\Collection; -use Appwrite\Utopia\Response\Model\CollectionUsage; use Appwrite\Utopia\Response\Model\Continent; use Appwrite\Utopia\Response\Model\Country; use Appwrite\Utopia\Response\Model\Currency; -use Appwrite\Utopia\Response\Model\DatabaseUsage; use Appwrite\Utopia\Response\Model\Document as ModelDocument; use Appwrite\Utopia\Response\Model\Domain; use Appwrite\Utopia\Response\Model\Error; @@ -26,7 +23,6 @@ use Appwrite\Utopia\Response\Model\ErrorDev; use Appwrite\Utopia\Response\Model\Execution; use Appwrite\Utopia\Response\Model\File; use Appwrite\Utopia\Response\Model\Func; -use Appwrite\Utopia\Response\Model\FunctionsUsage; use Appwrite\Utopia\Response\Model\Index; use Appwrite\Utopia\Response\Model\JWT; use Appwrite\Utopia\Response\Model\Key; @@ -47,8 +43,13 @@ use Appwrite\Utopia\Response\Model\Token; use Appwrite\Utopia\Response\Model\Webhook; use Appwrite\Utopia\Response\Model\Preferences; use Appwrite\Utopia\Response\Model\Mock; // Keep last -use Appwrite\Utopia\Response\Model\ProjectUsage; -use Appwrite\Utopia\Response\Model\StorageUsage; +use Appwrite\Utopia\Response\Model\UsageBuckets; +use Appwrite\Utopia\Response\Model\UsageCollection; +use Appwrite\Utopia\Response\Model\UsageDatabase; +use Appwrite\Utopia\Response\Model\UsageFunctions; +use Appwrite\Utopia\Response\Model\UsageProject; +use Appwrite\Utopia\Response\Model\UsageStorage; +use Appwrite\Utopia\Response\Model\UsageUsers; use Appwrite\Utopia\Response\Model\UsersUsage; use stdClass; @@ -67,6 +68,13 @@ class Response extends SwooleResponse const MODEL_METRIC_LIST = 'metricList'; const MODEL_ERROR_DEV = 'errorDev'; const MODEL_BASE_LIST = 'baseList'; + const MODEL_USAGE_DATABASE = 'usageDatabase'; + const MODEL_USAGE_COLLECTION = 'usageCollection'; + const MODEL_USAGE_USERS = 'usageUsers'; + const MODEL_USAGE_BUCKETS = 'usageBuckets'; + const MODEL_USAGE_STORAGE = 'usageStorage'; + const MODEL_USAGE_FUNCTIONS = 'usageFunctions'; + const MODEL_USAGE_PROJECT = 'usageProject'; // Database const MODEL_COLLECTION = 'collection'; @@ -77,13 +85,10 @@ class Response extends SwooleResponse const MODEL_INDEX_LIST = 'indexList'; const MODEL_DOCUMENT = 'document'; const MODEL_DOCUMENT_LIST = 'documentList'; - const MODEL_DATABASE_USAGE = 'databaseUsage'; - const MODEL_COLLECTION_USAGE = 'collectionUsage'; // Users const MODEL_USER = 'user'; const MODEL_USER_LIST = 'userList'; - const MODEL_USERS_USAGE = 'usersUsage'; const MODEL_SESSION = 'session'; const MODEL_SESSION_LIST = 'sessionList'; const MODEL_TOKEN = 'token'; @@ -94,8 +99,6 @@ class Response extends SwooleResponse const MODEL_FILE = 'file'; const MODEL_FILE_LIST = 'fileList'; const MODEL_BUCKET = 'bucket'; // - Missing - const MODEL_BUCKETS_USAGE = 'bucketsUsage'; - const MODEL_STORAGE_USAGE = 'storageUsage'; // Locale const MODEL_LOCALE = 'locale'; @@ -119,7 +122,6 @@ class Response extends SwooleResponse // Functions const MODEL_FUNCTION = 'function'; const MODEL_FUNCTION_LIST = 'functionList'; - const MODEL_FUNCTIONS_USAGE = 'functionsUsage'; const MODEL_TAG = 'tag'; const MODEL_TAG_LIST = 'tagList'; const MODEL_EXECUTION = 'execution'; @@ -128,7 +130,6 @@ class Response extends SwooleResponse // Project const MODEL_PROJECT = 'project'; const MODEL_PROJECT_LIST = 'projectList'; - const MODEL_PROJECT_USAGE = 'projectUsage'; const MODEL_WEBHOOK = 'webhook'; const MODEL_WEBHOOK_LIST = 'webhookList'; const MODEL_KEY = 'key'; @@ -198,27 +199,20 @@ class Response extends SwooleResponse ->setModel(new Attribute()) ->setModel(new Index()) ->setModel(new ModelDocument()) - ->setModel(new DatabaseUsage()) - ->setModel(new CollectionUsage()) ->setModel(new Log()) ->setModel(new User()) - ->setModel(new UsersUsage()) ->setModel(new Preferences()) ->setModel(new Session()) ->setModel(new Token()) ->setModel(new JWT()) ->setModel(new Locale()) ->setModel(new File()) - ->setModel(new StorageUsage()) - ->setModel(new BucketsUsage()) ->setModel(new Team()) ->setModel(new Membership()) ->setModel(new Func()) - ->setModel(new FunctionsUsage()) ->setModel(new Tag()) ->setModel(new Execution()) ->setModel(new Project()) - ->setModel(new ProjectUsage()) ->setModel(new Webhook()) ->setModel(new Key()) ->setModel(new Domain()) @@ -228,6 +222,13 @@ class Response extends SwooleResponse ->setModel(new Language()) ->setModel(new Currency()) ->setModel(new Phone()) + ->setModel(new UsageDatabase()) + ->setModel(new UsageCollection()) + ->setModel(new UsageUsers()) + ->setModel(new UsageStorage()) + ->setModel(new UsageBuckets()) + ->setModel(new UsageFunctions()) + ->setModel(new UsageProject()) // Verification // Recovery // Tests (keep last) diff --git a/src/Appwrite/Utopia/Response/Model/BucketsUsage.php b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php similarity index 93% rename from src/Appwrite/Utopia/Response/Model/BucketsUsage.php rename to src/Appwrite/Utopia/Response/Model/UsageBuckets.php index 0fb923e147..11fb2e2f8c 100644 --- a/src/Appwrite/Utopia/Response/Model/BucketsUsage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php @@ -6,7 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use stdClass; -class BucketsUsage extends Model +class UsageBuckets extends Model { public function __construct() { @@ -55,16 +55,16 @@ class BucketsUsage extends Model */ public function getName():string { - return 'BucketsUsage'; + return 'UsageBuckets'; } /** - * Get Collection + * Get Type * * @return string */ public function getType():string { - return Response::MODEL_BUCKETS_USAGE; + return Response::MODEL_USAGE_BUCKETS; } } \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/CollectionUsage.php b/src/Appwrite/Utopia/Response/Model/UsageCollection.php similarity index 93% rename from src/Appwrite/Utopia/Response/Model/CollectionUsage.php rename to src/Appwrite/Utopia/Response/Model/UsageCollection.php index a0b0a51c9b..f2815831a2 100644 --- a/src/Appwrite/Utopia/Response/Model/CollectionUsage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageCollection.php @@ -6,7 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use stdClass; -class CollectionUsage extends Model +class UsageCollection extends Model { public function __construct() { @@ -62,16 +62,16 @@ class CollectionUsage extends Model */ public function getName():string { - return 'CollectionUsage'; + return 'UsageCollection'; } /** - * Get Collection + * Get Type * * @return string */ public function getType():string { - return Response::MODEL_COLLECTION_USAGE; + return Response::MODEL_USAGE_COLLECTION; } } \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php similarity index 96% rename from src/Appwrite/Utopia/Response/Model/DatabaseUsage.php rename to src/Appwrite/Utopia/Response/Model/UsageDatabase.php index 1c24625004..2ebe0b64c0 100644 --- a/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php @@ -6,7 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use stdClass; -class DatabaseUsage extends Model +class UsageDatabase extends Model { public function __construct() { @@ -97,16 +97,16 @@ class DatabaseUsage extends Model */ public function getName():string { - return 'DatabaseUsage'; + return 'UsageDatabase'; } /** - * Get Collection + * Get Type * * @return string */ public function getType():string { - return Response::MODEL_DATABASE_USAGE; + return Response::MODEL_USAGE_DATABASE; } } \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/FunctionsUsage.php b/src/Appwrite/Utopia/Response/Model/UsageFunctions.php similarity index 91% rename from src/Appwrite/Utopia/Response/Model/FunctionsUsage.php rename to src/Appwrite/Utopia/Response/Model/UsageFunctions.php index 60f017705e..625bdab71c 100644 --- a/src/Appwrite/Utopia/Response/Model/FunctionsUsage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageFunctions.php @@ -6,7 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use stdClass; -class FunctionsUsage extends Model +class UsageFunctions extends Model { public function __construct() { @@ -48,16 +48,16 @@ class FunctionsUsage extends Model */ public function getName():string { - return 'FunctionsUsage'; + return 'UsageFunctions'; } /** - * Get Collection + * Get Type * * @return string */ public function getType():string { - return Response::MODEL_FUNCTIONS_USAGE; + return Response::MODEL_USAGE_FUNCTIONS; } } \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/ProjectUsage.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php similarity index 95% rename from src/Appwrite/Utopia/Response/Model/ProjectUsage.php rename to src/Appwrite/Utopia/Response/Model/UsageProject.php index deee3b78b1..95bb3cd42d 100644 --- a/src/Appwrite/Utopia/Response/Model/ProjectUsage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -6,7 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use stdClass; -class ProjectUsage extends Model +class UsageProject extends Model { public function __construct() { @@ -76,16 +76,16 @@ class ProjectUsage extends Model */ public function getName():string { - return 'ProjectUsage'; + return 'UsageProject'; } /** - * Get Collection + * Get Type * * @return string */ public function getType():string { - return Response::MODEL_PROJECT_USAGE; + return Response::MODEL_USAGE_PROJECT; } } \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/StorageUsage.php b/src/Appwrite/Utopia/Response/Model/UsageStorage.php similarity index 92% rename from src/Appwrite/Utopia/Response/Model/StorageUsage.php rename to src/Appwrite/Utopia/Response/Model/UsageStorage.php index e26e0d4b3d..66eaed602c 100644 --- a/src/Appwrite/Utopia/Response/Model/StorageUsage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageStorage.php @@ -6,7 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use stdClass; -class StorageUsage extends Model +class UsageStorage extends Model { public function __construct() { @@ -45,12 +45,12 @@ class StorageUsage extends Model } /** - * Get Collection + * Get Type * * @return string */ public function getType():string { - return Response::MODEL_STORAGE_USAGE; + return Response::MODEL_USAGE_STORAGE; } } \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/UsersUsage.php b/src/Appwrite/Utopia/Response/Model/UsageUsers.php similarity index 96% rename from src/Appwrite/Utopia/Response/Model/UsersUsage.php rename to src/Appwrite/Utopia/Response/Model/UsageUsers.php index bb3db30b26..db8056f5c6 100644 --- a/src/Appwrite/Utopia/Response/Model/UsersUsage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageUsers.php @@ -6,7 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use stdClass; -class UsersUsage extends Model +class UsageUsers extends Model { public function __construct() { @@ -76,7 +76,7 @@ class UsersUsage extends Model */ public function getName():string { - return 'UsersUsage'; + return 'UsageUsers'; } /** @@ -86,6 +86,6 @@ class UsersUsage extends Model */ public function getType():string { - return Response::MODEL_USERS_USAGE; + return Response::MODEL_USAGE_USERS; } } \ No newline at end of file From 6835124af8f387f0df364991d39f0ef2e76db473 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 27 Aug 2021 23:04:43 +0530 Subject: [PATCH 078/120] feat(usage): rename all usage response models --- app/controllers/api/database.php | 8 ++++---- app/controllers/api/functions.php | 4 ++-- app/controllers/api/projects.php | 4 ++-- app/controllers/api/storage.php | 8 ++++---- app/controllers/api/users.php | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 86e8304d8b..1ff2e9b76e 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -254,7 +254,7 @@ App::get('/v1/database/usage') ->label('sdk.method', 'getUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_DATABASE_USAGE) + ->label('sdk.response.model', Response::MODEL_USAGE_DATABASE) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') @@ -335,7 +335,7 @@ App::get('/v1/database/usage') ]); } - $response->dynamic($usage, Response::MODEL_DATABASE_USAGE); + $response->dynamic($usage, Response::MODEL_USAGE_DATABASE); }); App::get('/v1/database/:collectionId/usage') @@ -347,7 +347,7 @@ App::get('/v1/database/:collectionId/usage') ->label('sdk.method', 'getUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_COLLECTION_USAGE) + ->label('sdk.response.model', Response::MODEL_USAGE_COLLECTION) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->param('collectionId', '', new UID(), 'Collection unique ID.') ->inject('response') @@ -425,7 +425,7 @@ App::get('/v1/database/:collectionId/usage') ]); } - $response->dynamic($usage, Response::MODEL_COLLECTION_USAGE); + $response->dynamic($usage, Response::MODEL_USAGE_COLLECTION); }); App::get('/v1/database/collections/:collectionId/logs') diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 9359d267f5..dec4fa92bf 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -148,7 +148,7 @@ App::get('/v1/functions/:functionId/usage') ->label('sdk.method', 'getUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_FUNCTIONS_USAGE) + ->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS) ->param('functionId', '', new UID(), 'Function unique ID.') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true) ->inject('response') @@ -224,7 +224,7 @@ App::get('/v1/functions/:functionId/usage') ]); } - $response->dynamic($usage, Response::MODEL_FUNCTIONS_USAGE); + $response->dynamic($usage, Response::MODEL_USAGE_FUNCTIONS); }); App::put('/v1/functions/:functionId') diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index e8551c29ce..2d8311845a 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -225,7 +225,7 @@ App::get('/v1/projects/:projectId/usage') ->label('sdk.method', 'getUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_PROJECT_USAGE) + ->label('sdk.response.model', Response::MODEL_USAGE_PROJECT) ->param('projectId', '', new UID(), 'Project unique ID.') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') @@ -311,7 +311,7 @@ App::get('/v1/projects/:projectId/usage') ]); } - $response->dynamic($usage, Response::MODEL_PROJECT_USAGE); + $response->dynamic($usage, Response::MODEL_USAGE_PROJECT); }); App::patch('/v1/projects/:projectId') diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 69a7c18708..c3804b2f60 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -653,7 +653,7 @@ App::get('/v1/storage/usage') ->label('sdk.method', 'getUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_STORAGE_USAGE) + ->label('sdk.response.model', Response::MODEL_USAGE_STORAGE) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') @@ -713,7 +713,7 @@ App::get('/v1/storage/usage') ]); } - $response->dynamic($usage, Response::MODEL_STORAGE_USAGE); + $response->dynamic($usage, Response::MODEL_USAGE_STORAGE); }); App::get('/v1/storage/:bucketId/usage') @@ -725,7 +725,7 @@ App::get('/v1/storage/:bucketId/usage') ->label('sdk.method', 'getUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_BUCKETS_USAGE) + ->label('sdk.response.model', Response::MODEL_USAGE_BUCKETS) ->param('bucketId', '', new UID(), 'Bucket unique ID.') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') @@ -792,5 +792,5 @@ App::get('/v1/storage/:bucketId/usage') ]); } - $response->dynamic($usage, Response::MODEL_BUCKETS_USAGE); + $response->dynamic($usage, Response::MODEL_USAGE_BUCKETS); }); \ No newline at end of file diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 5a0cb03209..1895098bf2 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -619,7 +619,7 @@ App::get('/v1/users/usage') ->label('sdk.method', 'getUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_USERS_USAGE) + ->label('sdk.response.model', Response::MODEL_USAGE_USERS) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') @@ -693,5 +693,5 @@ App::get('/v1/users/usage') } - $response->dynamic($usage, Response::MODEL_USERS_USAGE); + $response->dynamic($usage, Response::MODEL_USAGE_USERS); }); \ No newline at end of file From 1b074a3c38b6aa49b85c8a26dcfaa19952ea686b Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sat, 28 Aug 2021 21:38:50 +0530 Subject: [PATCH 079/120] feat(model): added better description for rule --- src/Appwrite/Utopia/Response/Model/UsageProject.php | 4 ++-- src/Appwrite/Utopia/Response/Model/UsageStorage.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index 95bb3cd42d..c44af4cf6d 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -59,9 +59,9 @@ class UsageProject extends Model 'example' => new stdClass, 'array' => true ]) - ->addRule('functions', [ + ->addRule('storage', [ 'type' => Response::MODEL_METRIC_LIST, - 'description' => 'Aggregated stats for the occupied storage size.', + 'description' => 'Aggregated stats for the occupied storage size (in bytes).', 'default' => [], 'example' => new stdClass, 'array' => true diff --git a/src/Appwrite/Utopia/Response/Model/UsageStorage.php b/src/Appwrite/Utopia/Response/Model/UsageStorage.php index 66eaed602c..4462a219a9 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageStorage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageStorage.php @@ -19,7 +19,7 @@ class UsageStorage extends Model ]) ->addRule('storage', [ 'type' => Response::MODEL_METRIC_LIST, - 'description' => 'Aggregated stats for the occupied storage size.', + 'description' => 'Aggregated stats for the occupied storage size (in bytes).', 'default' => [], 'example' => new stdClass, 'array' => true From f7d657cc38ff6418f002fb5a7f11e9dec914a388 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sat, 28 Aug 2021 21:55:48 +0530 Subject: [PATCH 080/120] feat(model): use the new authorization skip methods --- app/controllers/api/database.php | 73 +++++++++++++++---------------- app/controllers/api/functions.php | 36 +++++++-------- app/controllers/api/projects.php | 36 +++++++-------- app/controllers/api/storage.php | 68 ++++++++++++++-------------- app/controllers/api/users.php | 37 ++++++++-------- 5 files changed, 122 insertions(+), 128 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 3b3e477d90..e2af571bf7 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1,6 +1,5 @@ find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $stats[$metric] = array_reverse($stats[$metric]); - } - - Authorization::reset(); + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); $usage = new Document([ 'range' => $range, @@ -387,8 +385,6 @@ App::get('/v1/database/:collectionId/usage') 'limit' => 90, ], ]; - - Authorization::disable(); $metrics = [ "database.collections.$collectionId.documents.count", @@ -399,23 +395,24 @@ App::get('/v1/database/:collectionId/usage') ]; $stats = []; - foreach ($metrics as $metric) { - $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - $stats[$metric] = array_reverse($stats[$metric]); - } - - Authorization::reset(); + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); $usage = new Document([ 'range' => $range, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index dec4fa92bf..fadc323832 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -188,8 +188,6 @@ App::get('/v1/functions/:functionId/usage') ], ]; - Authorization::disable(); - $metrics = [ "functions.$functionId.executions", "functions.$functionId.failures", @@ -197,24 +195,24 @@ App::get('/v1/functions/:functionId/usage') ]; $stats = []; - foreach ($metrics as $metric) { - $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $stats[$metric] = array_reverse($stats[$metric]); - } - - Authorization::reset(); + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); $usage = new Document([ 'range' => $range, diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 2d8311845a..0d1c3b713a 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -267,8 +267,6 @@ App::get('/v1/projects/:projectId/usage') $dbForInternal->setNamespace('project_' . $projectId . '_internal'); - Authorization::disable(); - $metrics = [ 'requests', 'network', @@ -280,24 +278,24 @@ App::get('/v1/projects/:projectId/usage') ]; $stats = []; - foreach ($metrics as $metric) { - $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $stats[$metric] = array_reverse($stats[$metric]); - } - - Authorization::reset(); + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); $usage = new Document([ 'range' => $range, diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index c3804b2f60..a11d50b5e3 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -10,7 +10,7 @@ use Utopia\Validator\HexColor; use Utopia\Cache\Cache; use Utopia\Cache\Adapter\Filesystem; use Appwrite\ClamAV\Network; -use Appwrite\Database\Validator\Authorization; +use Utopia\Database\Validator\Authorization; use Appwrite\Database\Validator\CustomId; use Utopia\Database\Document; use Utopia\Database\Validator\UID; @@ -682,29 +682,30 @@ App::get('/v1/storage/usage') ], ]; - Authorization::disable(); - $metrics = [ "storage.total", "storage.files.count" ]; $stats = []; - foreach ($metrics as $metric) { - $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - $stats[$metric] = array_reverse($stats[$metric]); - } + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); $usage = new Document([ 'range' => $range, @@ -757,7 +758,6 @@ App::get('/v1/storage/:bucketId/usage') ], ]; - Authorization::disable(); $metrics = [ "storage.buckets.$bucketId.files.create", "storage.buckets.$bucketId.files.read", @@ -766,22 +766,24 @@ App::get('/v1/storage/:bucketId/usage') ]; $stats = []; - foreach ($metrics as $metric) { - $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - $stats[$metric] = array_reverse($stats[$metric]); - } - Authorization::reset(); + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); $usage = new Document([ 'range' => $range, diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 1895098bf2..7541487ba0 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -2,7 +2,6 @@ use Appwrite\Auth\Auth; use Appwrite\Auth\Validator\Password; -use Appwrite\Database\Validator\Authorization; use Appwrite\Utopia\Response; use Utopia\App; use Utopia\Exception; @@ -21,6 +20,7 @@ use Appwrite\Database\Validator\CustomId; use Appwrite\Utopia\Response\Model; use Utopia\Database\Database; use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; App::post('/v1/users') ->desc('Create User') @@ -649,7 +649,6 @@ App::get('/v1/users/usage') ], ]; - Authorization::disable(); $metrics = [ "users.count", "users.create", @@ -661,24 +660,24 @@ App::get('/v1/users/usage') ]; $stats = []; - foreach ($metrics as $metric) { - $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $stats[$metric] = array_reverse($stats[$metric]); - } - - Authorization::reset(); + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); $usage = new Document([ 'range' => $range, From a4e2aeb5e79615f544a3f4bbd2533021624e7adf Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 29 Aug 2021 11:00:37 +0530 Subject: [PATCH 081/120] feat(usage): added provider metrics to endpoints --- app/controllers/api/users.php | 5 ++++- src/Appwrite/Utopia/Response/Model/UsageUsers.php | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 7541487ba0..7fcdffcc8b 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -621,10 +621,11 @@ App::get('/v1/users/usage') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USAGE_USERS) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->param('provider', '', new WhiteList(['email', 'anonymous', 'oauth2-google', 'oauth2-apple'], true), 'Provider Name.', true) ->inject('response') ->inject('dbForInternal') ->inject('register') - ->action(function ($range, $response, $dbForInternal) { + ->action(function ($range, $provider, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -656,6 +657,7 @@ App::get('/v1/users/usage') "users.update", "users.delete", "users.sessions.create", + "users.sessions.$provider.create", "users.sessions.delete" ]; @@ -687,6 +689,7 @@ App::get('/v1/users/usage') 'users.update' => $stats["users.update"], 'users.delete' => $stats["users.delete"], 'sessions.create' => $stats["users.sessions.create"], + 'sessions.provider.create' => $stats["users.sessions.$provider.create"], 'sessions.delete' => $stats["users.sessions.delete"] ]); diff --git a/src/Appwrite/Utopia/Response/Model/UsageUsers.php b/src/Appwrite/Utopia/Response/Model/UsageUsers.php index db8056f5c6..913c4f33fe 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageUsers.php +++ b/src/Appwrite/Utopia/Response/Model/UsageUsers.php @@ -59,6 +59,13 @@ class UsageUsers extends Model 'example' => new stdClass, 'array' => true ]) + ->addRule('sessions.provider.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for sessions created for a provider ( email, anonymous or oauth2 ).', + 'default' => null, + 'example' => new stdClass, + 'array' => true + ]) ->addRule('sessions.delete', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for sessions deleted.', From 58a5320cf11f04ccebd35e92a71b846bdb9bed15 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 29 Aug 2021 13:20:52 +0530 Subject: [PATCH 082/120] feat(tests): added tests for database usage --- tests/e2e/Scopes/SideConsole.php | 23 +++++ .../Services/Database/DatabaseConsoleTest.php | 89 +++++++++++++++++ .../Functions/FunctionsConsoleTest.php | 97 +++++++++++++++++++ 3 files changed, 209 insertions(+) create mode 100644 tests/e2e/Scopes/SideConsole.php create mode 100644 tests/e2e/Services/Database/DatabaseConsoleTest.php create mode 100644 tests/e2e/Services/Functions/FunctionsConsoleTest.php diff --git a/tests/e2e/Scopes/SideConsole.php b/tests/e2e/Scopes/SideConsole.php new file mode 100644 index 0000000000..ada99a6e6a --- /dev/null +++ b/tests/e2e/Scopes/SideConsole.php @@ -0,0 +1,23 @@ + 'http://localhost', + 'cookie' => 'a_session_console='. $this->getRoot()['session'], + 'x-appwrite-mode' => 'admin' + ]; + } + + /** + * @return string + */ + public function getSide() + { + return 'console'; + } +} diff --git a/tests/e2e/Services/Database/DatabaseConsoleTest.php b/tests/e2e/Services/Database/DatabaseConsoleTest.php new file mode 100644 index 0000000000..0921d2c50a --- /dev/null +++ b/tests/e2e/Services/Database/DatabaseConsoleTest.php @@ -0,0 +1,89 @@ +client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'collectionId' => 'unique()', + 'name' => 'Movies', + 'read' => ['role:all'], + 'write' => ['role:all'], + 'permission' => 'document', + ]); + + $this->assertEquals($movies['headers']['status-code'], 201); + $this->assertEquals($movies['body']['name'], 'Movies'); + + return ['moviesId' => $movies['body']['$id']]; + } + + public function testGetDatabaseUsage() + { + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['documents.count']); + $this->assertIsArray($response['body']['collections.count']); + $this->assertIsArray($response['body']['documents.create']); + $this->assertIsArray($response['body']['documents.read']); + $this->assertIsArray($response['body']['documents.update']); + $this->assertIsArray($response['body']['documents.delete']); + $this->assertIsArray($response['body']['collections.create']); + $this->assertIsArray($response['body']['collections.read']); + $this->assertIsArray($response['body']['collections.update']); + $this->assertIsArray($response['body']['collections.delete']); + } + + + /** + * @depends testCreateCollection + */ + public function testGetCollectionUsage(array $data) + { + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['documents.count']); + $this->assertIsArray($response['body']['documents.create']); + $this->assertIsArray($response['body']['documents.read']); + $this->assertIsArray($response['body']['documents.update']); + $this->assertIsArray($response['body']['documents.delete']); + } +} \ No newline at end of file diff --git a/tests/e2e/Services/Functions/FunctionsConsoleTest.php b/tests/e2e/Services/Functions/FunctionsConsoleTest.php new file mode 100644 index 0000000000..99dd4d315d --- /dev/null +++ b/tests/e2e/Services/Functions/FunctionsConsoleTest.php @@ -0,0 +1,97 @@ + 'http://localhost', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + 'x-appwrite-mode' => 'admin' + ]; + } + + public function testCreateCollection():array + { + /** + * Test for SUCCESS + */ + $movies = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'collectionId' => 'unique()', + 'name' => 'Movies', + 'read' => ['role:all'], + 'write' => ['role:all'], + 'permission' => 'document', + ]); + + $this->assertEquals($movies['headers']['status-code'], 201); + $this->assertEquals($movies['body']['name'], 'Movies'); + + return ['moviesId' => $movies['body']['$id']]; + } + + public function testGetDatabaseUsage() + { + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['documents.count']); + $this->assertIsArray($response['body']['collections.count']); + $this->assertIsArray($response['body']['documents.create']); + $this->assertIsArray($response['body']['documents.read']); + $this->assertIsArray($response['body']['documents.update']); + $this->assertIsArray($response['body']['documents.delete']); + $this->assertIsArray($response['body']['collections.create']); + $this->assertIsArray($response['body']['collections.read']); + $this->assertIsArray($response['body']['collections.update']); + $this->assertIsArray($response['body']['collections.delete']); + } + + + /** + * @depends testCreateCollection + */ + public function testGetCollectionUsage(array $data) + { + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['documents.count']); + $this->assertIsArray($response['body']['documents.create']); + $this->assertIsArray($response['body']['documents.read']); + $this->assertIsArray($response['body']['documents.update']); + $this->assertIsArray($response['body']['documents.delete']); + } + +} \ No newline at end of file From 94a17e9ce06b65cd408cb7ff57e80426a8506b96 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 29 Aug 2021 13:29:08 +0530 Subject: [PATCH 083/120] feat(tests): added tests for functions usage --- .../Services/Database/DatabaseConsoleTest.php | 36 +++++- .../Functions/FunctionsConsoleTest.php | 109 ++++++++---------- 2 files changed, 86 insertions(+), 59 deletions(-) diff --git a/tests/e2e/Services/Database/DatabaseConsoleTest.php b/tests/e2e/Services/Database/DatabaseConsoleTest.php index 0921d2c50a..f8d50188ce 100644 --- a/tests/e2e/Services/Database/DatabaseConsoleTest.php +++ b/tests/e2e/Services/Database/DatabaseConsoleTest.php @@ -36,6 +36,19 @@ class DatabaseConsoleTest extends Scope public function testGetDatabaseUsage() { + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + /** * Test for SUCCESS */ @@ -68,9 +81,30 @@ class DatabaseConsoleTest extends Scope public function testGetCollectionUsage(array $data) { /** - * Test for SUCCESS + * Test for FAILURE */ + $response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + $response = $this->client->call(Client::METHOD_GET, '/database/randomCollectionId/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 404); + + /** + * Test for SUCCESS + */ $response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'] diff --git a/tests/e2e/Services/Functions/FunctionsConsoleTest.php b/tests/e2e/Services/Functions/FunctionsConsoleTest.php index 99dd4d315d..669a42c2ee 100644 --- a/tests/e2e/Services/Functions/FunctionsConsoleTest.php +++ b/tests/e2e/Services/Functions/FunctionsConsoleTest.php @@ -5,80 +5,75 @@ namespace Tests\E2E\Services\Database; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Client; +use Tests\E2E\Scopes\SideConsole; -class DatabaseConsoleTest extends Scope +class FunctionsConsoleTest extends Scope { use ProjectCustom; + use SideConsole; - public function getHeaders():array + public function testCreateFunction():array { - return [ - 'origin' => 'http://localhost', - 'cookie' => 'a_session_console=' . $this->getRoot()['session'], - 'x-appwrite-mode' => 'admin' - ]; - } - - public function testCreateCollection():array - { - /** - * Test for SUCCESS - */ - $movies = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ + $function = $this->client->call(Client::METHOD_POST, '/functions', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ - 'collectionId' => 'unique()', - 'name' => 'Movies', - 'read' => ['role:all'], - 'write' => ['role:all'], - 'permission' => 'document', + 'functionId' => 'unique()', + 'name' => 'Test', + 'execute' => ['user:'.$this->getUser()['$id']], + 'runtime' => 'php-8.0', + 'vars' => [ + 'funcKey1' => 'funcValue1', + 'funcKey2' => 'funcValue2', + 'funcKey3' => 'funcValue3', + ], + 'events' => [ + 'account.create', + 'account.delete', + ], + 'schedule' => '0 0 1 1 *', + 'timeout' => 10, ]); - $this->assertEquals($movies['headers']['status-code'], 201); - $this->assertEquals($movies['body']['name'], 'Movies'); + $this->assertEquals(201, $function['headers']['status-code']); - return ['moviesId' => $movies['body']['$id']]; + return [ + 'functionId' => $function['body']['$id'] + ]; } - public function testGetDatabaseUsage() - { - /** - * Test for SUCCESS - */ - - $response = $this->client->call(Client::METHOD_GET, '/database/usage', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'] - ], $this->getHeaders()), [ - 'range' => '24h' - ]); - - $this->assertEquals($response['headers']['status-code'], 200); - $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['documents.count']); - $this->assertIsArray($response['body']['collections.count']); - $this->assertIsArray($response['body']['documents.create']); - $this->assertIsArray($response['body']['documents.read']); - $this->assertIsArray($response['body']['documents.update']); - $this->assertIsArray($response['body']['documents.delete']); - $this->assertIsArray($response['body']['collections.create']); - $this->assertIsArray($response['body']['collections.read']); - $this->assertIsArray($response['body']['collections.update']); - $this->assertIsArray($response['body']['collections.delete']); - } - - /** - * @depends testCreateCollection + * @depends testCreateFunction */ public function testGetCollectionUsage(array $data) { + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '232h' + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_GET, '/functions/randomFunctionId/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals(404, $response['headers']['status-code']); + /** * Test for SUCCESS */ - $response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/usage', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'] ], $this->getHeaders()), [ @@ -87,11 +82,9 @@ class DatabaseConsoleTest extends Scope $this->assertEquals($response['headers']['status-code'], 200); $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['documents.count']); - $this->assertIsArray($response['body']['documents.create']); - $this->assertIsArray($response['body']['documents.read']); - $this->assertIsArray($response['body']['documents.update']); - $this->assertIsArray($response['body']['documents.delete']); + $this->assertIsArray($response['body']['functions.executions']); + $this->assertIsArray($response['body']['functions.failures']); + $this->assertIsArray($response['body']['functions.compute']); } } \ No newline at end of file From 9eea6d55f737f41d199245a6e37327d31aae8e2d Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 29 Aug 2021 13:41:29 +0530 Subject: [PATCH 084/120] feat(tests): added tests for projects usage --- ...Test.php => DatabaseConsoleClientTest.php} | 4 ++- ...est.php => FunctionsConsoleClientTest.php} | 3 ++- .../Projects/ProjectsConsoleClientTest.php | 26 +++++++------------ 3 files changed, 14 insertions(+), 19 deletions(-) rename tests/e2e/Services/Database/{DatabaseConsoleTest.php => DatabaseConsoleClientTest.php} (96%) rename tests/e2e/Services/Functions/{FunctionsConsoleTest.php => FunctionsConsoleClientTest.php} (96%) diff --git a/tests/e2e/Services/Database/DatabaseConsoleTest.php b/tests/e2e/Services/Database/DatabaseConsoleClientTest.php similarity index 96% rename from tests/e2e/Services/Database/DatabaseConsoleTest.php rename to tests/e2e/Services/Database/DatabaseConsoleClientTest.php index f8d50188ce..625f26107b 100644 --- a/tests/e2e/Services/Database/DatabaseConsoleTest.php +++ b/tests/e2e/Services/Database/DatabaseConsoleClientTest.php @@ -7,7 +7,7 @@ use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Client; use Tests\E2E\Scopes\SideConsole; -class DatabaseConsoleTest extends Scope +class DatabaseConsoleClientTest extends Scope { use ProjectCustom; use SideConsole; @@ -61,6 +61,7 @@ class DatabaseConsoleTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 11); $this->assertEquals($response['body']['range'], '24h'); $this->assertIsArray($response['body']['documents.count']); $this->assertIsArray($response['body']['collections.count']); @@ -113,6 +114,7 @@ class DatabaseConsoleTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 6); $this->assertEquals($response['body']['range'], '24h'); $this->assertIsArray($response['body']['documents.count']); $this->assertIsArray($response['body']['documents.create']); diff --git a/tests/e2e/Services/Functions/FunctionsConsoleTest.php b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php similarity index 96% rename from tests/e2e/Services/Functions/FunctionsConsoleTest.php rename to tests/e2e/Services/Functions/FunctionsConsoleClientTest.php index 669a42c2ee..d8bdb30b65 100644 --- a/tests/e2e/Services/Functions/FunctionsConsoleTest.php +++ b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php @@ -7,7 +7,7 @@ use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Client; use Tests\E2E\Scopes\SideConsole; -class FunctionsConsoleTest extends Scope +class FunctionsConsoleClientTest extends Scope { use ProjectCustom; use SideConsole; @@ -81,6 +81,7 @@ class FunctionsConsoleTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 4); $this->assertEquals($response['body']['range'], '24h'); $this->assertIsArray($response['body']['functions.executions']); $this->assertIsArray($response['body']['functions.failures']); diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 4d83efb416..1842891492 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -215,24 +215,16 @@ class ProjectsConsoleClientTest extends Scope ], $this->getHeaders())); $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(count($response['body']), 8); $this->assertNotEmpty($response['body']); - $this->assertArrayHasKey('collections', $response['body']); - $this->assertArrayHasKey('documents', $response['body']); - $this->assertArrayHasKey('network', $response['body']); - $this->assertArrayHasKey('requests', $response['body']); - $this->assertArrayHasKey('storage', $response['body']); - $this->assertArrayHasKey('users', $response['body']); - $this->assertIsArray($response['body']['collections']['data']); - $this->assertIsInt($response['body']['collections']['total']); - $this->assertIsArray($response['body']['documents']['data']); - $this->assertIsInt($response['body']['documents']['total']); - $this->assertIsArray($response['body']['network']['data']); - $this->assertIsInt($response['body']['network']['total']); - $this->assertIsArray($response['body']['requests']['data']); - $this->assertIsInt($response['body']['requests']['total']); - $this->assertIsInt($response['body']['storage']['total']); - $this->assertIsArray($response['body']['users']['data']); - $this->assertIsInt($response['body']['users']['total']); + $this->assertEquals('30d', $response['body']['range']); + $this->assertIsArray($response['body']['requests']); + $this->assertIsArray($response['body']['network']); + $this->assertIsArray($response['body']['functions']); + $this->assertIsArray($response['body']['documents']); + $this->assertIsArray($response['body']['collections']); + $this->assertIsArray($response['body']['users']); + $this->assertIsArray($response['body']['storage']); /** * Test for FAILURE From 4454a518b78b891917332d18f39a266a047def2f Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 29 Aug 2021 13:49:25 +0530 Subject: [PATCH 085/120] feat(tests): added tests for storage usage --- .../Storage/StorageConsoleClientTest.php | 86 ++++++++++++++++++- 1 file changed, 82 insertions(+), 4 deletions(-) diff --git a/tests/e2e/Services/Storage/StorageConsoleClientTest.php b/tests/e2e/Services/Storage/StorageConsoleClientTest.php index b2c9582960..6960661777 100644 --- a/tests/e2e/Services/Storage/StorageConsoleClientTest.php +++ b/tests/e2e/Services/Storage/StorageConsoleClientTest.php @@ -2,13 +2,91 @@ namespace Tests\E2E\Services\Storage; +use Tests\E2E\Client; use Tests\E2E\Scopes\Scope; -use Tests\E2E\Scopes\ProjectConsole; -use Tests\E2E\Scopes\SideClient; +use Tests\E2E\Scopes\ProjectCustom; +use Tests\E2E\Scopes\SideConsole; class StorageConsoleClientTest extends Scope { + use SideConsole; use StorageBase; - use ProjectConsole; - use SideClient; + use ProjectCustom; + + public function testGetStorageUsage() + { + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/storage/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/storage/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 3); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['storage']); + $this->assertIsArray($response['body']['files']); + } + + public function testGetStorageBucketUsage() + { + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/storage/default/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + // TODO: Uncomment once we implement check for missing bucketId in the usage endpoint. + + // $response = $this->client->call(Client::METHOD_GET, '/storage/randomBucketId/usage', array_merge([ + // 'content-type' => 'application/json', + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()), [ + // 'range' => '24h' + // ]); + + // $this->assertEquals($response['headers']['status-code'], 404); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/storage/default/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 5); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['files.create']); + $this->assertIsArray($response['body']['files.read']); + $this->assertIsArray($response['body']['files.update']); + $this->assertIsArray($response['body']['files.delete']); + } } \ No newline at end of file From 3aedad2061d020a2fc392273c167a6b7ab4d9812 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 29 Aug 2021 14:13:14 +0530 Subject: [PATCH 086/120] feat(tests): added tests for users usage --- .../Utopia/Response/Model/UsageUsers.php | 2 +- .../Functions/FunctionsConsoleClientTest.php | 2 +- .../Services/Users/UsersConsoleClientTest.php | 83 +++++++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/Services/Users/UsersConsoleClientTest.php diff --git a/src/Appwrite/Utopia/Response/Model/UsageUsers.php b/src/Appwrite/Utopia/Response/Model/UsageUsers.php index 913c4f33fe..8813615062 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageUsers.php +++ b/src/Appwrite/Utopia/Response/Model/UsageUsers.php @@ -62,7 +62,7 @@ class UsageUsers extends Model ->addRule('sessions.provider.create', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for sessions created for a provider ( email, anonymous or oauth2 ).', - 'default' => null, + 'default' => [], 'example' => new stdClass, 'array' => true ]) diff --git a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php index d8bdb30b65..ee15a81dcb 100644 --- a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php @@ -1,6 +1,6 @@ client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h', + 'provider' => 'email' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h', + 'provider' => 'some-random-provider' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h', + 'provider' => 'email' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 9); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['users.count']); + $this->assertIsArray($response['body']['users.create']); + $this->assertIsArray($response['body']['users.read']); + $this->assertIsArray($response['body']['users.update']); + $this->assertIsArray($response['body']['users.delete']); + $this->assertIsArray($response['body']['sessions.create']); + $this->assertIsArray($response['body']['sessions.provider.create']); + $this->assertIsArray($response['body']['sessions.delete']); + + $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 9); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['users.count']); + $this->assertIsArray($response['body']['users.create']); + $this->assertIsArray($response['body']['users.read']); + $this->assertIsArray($response['body']['users.update']); + $this->assertIsArray($response['body']['users.delete']); + $this->assertIsArray($response['body']['sessions.create']); + $this->assertIsArray($response['body']['sessions.provider.create']); + $this->assertIsArray($response['body']['sessions.delete']); + } +} \ No newline at end of file From 66294445ce47a2a7010ff63dc1c75ae25f47fb17 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 29 Aug 2021 14:24:50 +0530 Subject: [PATCH 087/120] feat(tests): added tests for users usage --- app/controllers/api/storage.php | 2 ++ src/Appwrite/Utopia/Response/Model/UsageBuckets.php | 7 +++++++ tests/e2e/Services/Storage/StorageConsoleClientTest.php | 4 ++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index a11d50b5e3..ee6210d25d 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -759,6 +759,7 @@ App::get('/v1/storage/:bucketId/usage') ]; $metrics = [ + "storage.buckets.$bucketId.files.count", "storage.buckets.$bucketId.files.create", "storage.buckets.$bucketId.files.read", "storage.buckets.$bucketId.files.update", @@ -787,6 +788,7 @@ App::get('/v1/storage/:bucketId/usage') $usage = new Document([ 'range' => $range, + 'files.count' => $stats["storage.buckets.$bucketId.files.count"], 'files.create' => $stats["storage.buckets.$bucketId.files.create"], 'files.read' => $stats["storage.buckets.$bucketId.files.read"], 'files.update' => $stats["storage.buckets.$bucketId.files.update"], diff --git a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php index 11fb2e2f8c..8c66ff47c4 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php +++ b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php @@ -17,6 +17,13 @@ class UsageBuckets extends Model 'default' => '', 'example' => '30d', ]) + ->addRule('files.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of files in this bucket.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) ->addRule('files.create', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for files created.', diff --git a/tests/e2e/Services/Storage/StorageConsoleClientTest.php b/tests/e2e/Services/Storage/StorageConsoleClientTest.php index 6960661777..4ec1a6ce9a 100644 --- a/tests/e2e/Services/Storage/StorageConsoleClientTest.php +++ b/tests/e2e/Services/Storage/StorageConsoleClientTest.php @@ -30,7 +30,6 @@ class StorageConsoleClientTest extends Scope /** * Test for SUCCESS */ - $response = $this->client->call(Client::METHOD_GET, '/storage/usage', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'] @@ -82,8 +81,9 @@ class StorageConsoleClientTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 200); - $this->assertEquals(count($response['body']), 5); + $this->assertEquals(count($response['body']), 6); $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['files.count']); $this->assertIsArray($response['body']['files.create']); $this->assertIsArray($response['body']['files.read']); $this->assertIsArray($response['body']['files.update']); From c84dc0fa92c774cd4d6e94d34721a53003edf47b Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 30 Aug 2021 10:19:29 +0300 Subject: [PATCH 088/120] init --- app/tasks/usage.php | 222 ++++++++++++++++++++++++++------------------ 1 file changed, 130 insertions(+), 92 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index fdbf1d3eb2..2de5c09377 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -11,15 +11,19 @@ use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; /** * Metrics We collect * + * General + * * requests * network * executions + * + * Database + * * database.collections.create * database.collections.read * database.collections.update @@ -32,10 +36,16 @@ use Utopia\Database\Validator\Authorization; * database.collections.{collectionId}.documents.read * database.collections.{collectionId}.documents.update * database.collections.{collectionId}.documents.delete + * + * Storage + * * storage.buckets.{bucketId}.files.create * storage.buckets.{bucketId}.files.read * storage.buckets.{bucketId}.files.update * storage.buckets.{bucketId}.files.delete + * + * Users + * * users.create * users.read * users.update @@ -71,7 +81,7 @@ $cli Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); - $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); //30 seconds + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) $periods = [ [ 'key' => '30m', @@ -189,16 +199,18 @@ $cli ], ]; + // TODO Maybe move this to the setResource method, and reuse in the http.php file $attempts = 0; $max = 10; $sleep = 1; + do { // connect to db try { $attempts++; $db = $register->get('db'); $redis = $register->get('cache'); break; // leave the do-while if successful - } catch (\Exception$e) { + } catch (\Exception $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); if ($attempts >= $max) { throw new \Exception('Failed to connect to database: ' . $e->getMessage()); @@ -207,6 +219,7 @@ $cli } } while ($attempts < $max); + // TODO use inject $cacheAdapter = new Cache(new Redis($redis)); $dbForProject = new Database(new MariaDB($db), $cacheAdapter); $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); @@ -223,9 +236,13 @@ $cli $loopStart = microtime(true); + /** + * Aggregate InfluxDB every 30 seconds + */ $client = $register->get('influxdb'); if ($client) { $database = $client->selectDB('telegraf'); + // sync data foreach ($globalMetrics as $metric => $options) { //for each metrics foreach ($periods as $period) { // aggregate data for each period @@ -250,12 +267,11 @@ $cli $points = $result->getPoints(); foreach ($points as $point) { $projectId = $point['projectId']; + if (!empty($projectId) && $projectId != 'console') { $dbForProject->setNamespace('project_' . $projectId . '_internal'); - if($metric == 'functions.functionId.executions') { - var_dump($points); - } $metricUpdated = $metric; + if (!empty($groupBy)) { $groupedBy = $point[$options['groupBy']] ?? ''; if (empty($groupedBy)) { @@ -263,9 +279,11 @@ $cli } $metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric); } + $time = \strtotime($point['time']); $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //construct unique id for each metric using time, period and metric $value = (!empty($point['value'])) ? $point['value'] : 0; + try { $document = $dbForProject->getDocument('stats', $id); if ($document->isEmpty()) { @@ -279,12 +297,11 @@ $cli ])); } else { $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $value)); + $document->setAttribute('value', $value)); } $latestTime[$metric][$period['key']] = $time; - } catch (\Exception$e) { - // if projects are deleted this might fail - Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}"); + } catch (\Exception $e) { // if projects are deleted this might fail + Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}: {$e->getMessage()}"); } } } @@ -292,119 +309,140 @@ $cli } } - if ($iterations % 30 == 0) { //every 15 minutes - // aggregate number of objects in database - // get count of all the documents per collection - - // buckets will have the same + /** + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ + if ($iterations % 30 == 0) { // Every 15 minutes + // Aggregate number of objects in database + // Get count of all the documents per collection - + // Buckets will have the same + $latestProject = null; + do { $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); - if (!empty($projects)) { - $latestProject = $projects[array_key_last($projects)]; + + if (empty($projects)) { + continue; + } - foreach ($projects as $project) { - $id = $project->getId(); + $latestProject = $projects[array_key_last($projects)]; - // get total storage - $dbForProject->setNamespace('project_' . $id . '_internal'); - $storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size'); + foreach ($projects as $project) { + $id = $project->getId(); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'period' => '15m', - 'time' => time(), - 'metric' => 'storage.total', - 'value' => $storageTotal, - 'type' => 1, - ])); + // Get total storage + $dbForProject->setNamespace('project_' . $id . '_internal'); + $storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size'); - $collections = [ - 'users' => [ - 'namespace' => 'internal', - ], - 'collections' => [ - 'metricPrefix' => 'database', - 'namespace' => 'internal', - 'subCollections' => [ - 'documents' => [ - 'namespace' => 'external', - ], + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'period' => '15m', + 'time' => time(), + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + + $collections = [ + 'users' => [ + 'namespace' => 'internal', + ], + 'collections' => [ + 'metricPrefix' => 'database', + 'namespace' => 'internal', + 'subCollections' => [ // TODO better document this key + 'documents' => [ + 'namespace' => 'external', ], ], - 'files' => [ - 'metricPrefix' => 'storage', - 'namespace' => 'internal', - ], - ]; - foreach ($collections as $collection => $options) { - try { + ], + 'files' => [ + 'metricPrefix' => 'storage', + 'namespace' => 'internal', + ], + ]; + + foreach ($collections as $collection => $options) { + try { + $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $count = $dbForProject->count($collection); + $dbForProject->setNamespace("project_{$id}_internal"); + $metricPrefix = $options['metricPrefix'] ?? ''; + $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; + + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + + $subCollections = $options['subCollections'] ?? []; + + if (empty($subCollections)) { + continue; + } + + $latestParent = null; + $subCollectionCounts = []; //total project level count of sub collections + + do { $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); - $count = $dbForProject->count($collection); - $dbForProject->setNamespace("project_{$id}_internal"); - $metricPrefix = $options['metricPrefix'] ?? ''; - $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => $metric, - 'value' => $count, - 'type' => 1, - ])); + $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); - $subCollections = $options['subCollections'] ?? []; - if (!empty($subCollections)) { - $latestParent = null; - $subCollectionCounts = []; //total project level count of sub collections - do { - $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); - $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); - if (!empty($parents)) { - $latestParent = $parents[array_key_last($parents)]; - foreach ($parents as $parent) { - foreach ($subCollections as $subCollection => $subOptions) { - $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); - $count = $dbForProject->count($parent->getId()); - $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; + if (empty($parents)) { + continue; + } - $dbForProject->setNamespace("project_{$id}_internal"); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", - 'value' => $count, - 'type' => 1, - ])); - } - } - } - } while (!empty($parents)); + $latestParent = $parents[array_key_last($parents)]; + + foreach ($parents as $parent) { + foreach ($subCollections as $subCollection => $subOptions) { + $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); + $count = $dbForProject->count($parent->getId()); + $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; - foreach ($subCollectionsCounts as $subCollection => $count) { $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ '$id' => $dbForProject->getId(), 'time' => time(), 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", + 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", 'value' => $count, 'type' => 1, ])); } } - } catch (\Exception$e) { - Console::warning("Failed to save database counters data for project {$collection}"); + } while (!empty($parents)); + + foreach ($subCollectionsCounts as $subCollection => $count) { + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", + 'value' => $count, + 'type' => 1, + ])); } + } catch (\Exception $e) { + Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}"); } } } - } while (!empty($projects)); } + $iterations++; $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); - }); + }); \ No newline at end of file From 723bddf95ab876521be0803e9869407b21484745 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 30 Aug 2021 13:14:45 +0530 Subject: [PATCH 089/120] feat(compose): disable debug mode --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 12bc51b802..539799aa9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,7 +39,7 @@ services: build: context: . args: - - DEBUG=true + - DEBUG=false - TESTING=true - VERSION=dev ports: From 2477b80978534a797d5a69118b47a0537c805856 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 30 Aug 2021 15:20:32 +0545 Subject: [PATCH 090/120] update counters also as time series data --- app/tasks/usage.php | 250 +++++++++++++++++++++++++++++++------------- 1 file changed, 179 insertions(+), 71 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 2de5c09377..3fa90a3f43 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -17,13 +17,13 @@ use Utopia\Database\Validator\Authorization; * Metrics We collect * * General - * + * * requests * network * executions - * + * * Database - * + * * database.collections.create * database.collections.read * database.collections.update @@ -36,16 +36,16 @@ use Utopia\Database\Validator\Authorization; * database.collections.{collectionId}.documents.read * database.collections.{collectionId}.documents.update * database.collections.{collectionId}.documents.delete - * + * * Storage - * + * * storage.buckets.{bucketId}.files.create * storage.buckets.{bucketId}.files.read * storage.buckets.{bucketId}.files.update * storage.buckets.{bucketId}.files.delete - * + * * Users - * + * * users.create * users.read * users.update @@ -203,7 +203,7 @@ $cli $attempts = 0; $max = 10; $sleep = 1; - + do { // connect to db try { $attempts++; @@ -219,7 +219,7 @@ $cli } } while ($attempts < $max); - // TODO use inject + // TODO use inject $cacheAdapter = new Cache(new Redis($redis)); $dbForProject = new Database(new MariaDB($db), $cacheAdapter); $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); @@ -242,7 +242,7 @@ $cli $client = $register->get('influxdb'); if ($client) { $database = $client->selectDB('telegraf'); - + // sync data foreach ($globalMetrics as $metric => $options) { //for each metrics foreach ($periods as $period) { // aggregate data for each period @@ -252,10 +252,10 @@ $cli } $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); - $table = $options['table']; //which influxdb table to query for this metric - $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //some sub level metrics may be grouped by other tags like collectionId, bucketId, etc + $table = $options['table']; //Which influxdb table to query for this metric + $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc - $filters = $options['filters'] ?? []; + $filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status if (!empty($filters)) { $filters = ' AND ' . implode(' AND ', array_map(function ($filter, $value) { return '"' . $filter . '"=\'' . $value . '\''; @@ -263,7 +263,7 @@ $cli } $result = $database->query('SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\'' . (empty($filters) ? '' : $filters) . ' GROUP BY time(' . $period['key'] . '), "projectId"' . $groupBy . ' FILL(null)'); - + $points = $result->getPoints(); foreach ($points as $point) { $projectId = $point['projectId']; @@ -271,7 +271,7 @@ $cli if (!empty($projectId) && $projectId != 'console') { $dbForProject->setNamespace('project_' . $projectId . '_internal'); $metricUpdated = $metric; - + if (!empty($groupBy)) { $groupedBy = $point[$options['groupBy']] ?? ''; if (empty($groupedBy)) { @@ -281,9 +281,9 @@ $cli } $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //construct unique id for each metric using time, period and metric + $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //Construct unique id for each metric using time, period and metric $value = (!empty($point['value'])) ? $point['value'] : 0; - + try { $document = $dbForProject->getDocument('stats', $id); if ($document->isEmpty()) { @@ -310,19 +310,19 @@ $cli } /** - * Aggregate MariaDB every 15 minutes - * Some of the queries here might contain full-table scans. - */ + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ if ($iterations % 30 == 0) { // Every 15 minutes // Aggregate number of objects in database // Get count of all the documents per collection - // Buckets will have the same $latestProject = null; - + do { $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); - + if (empty($projects)) { continue; } @@ -330,20 +330,45 @@ $cli $latestProject = $projects[array_key_last($projects)]; foreach ($projects as $project) { - $id = $project->getId(); + $projectId = $project->getId(); // Get total storage - $dbForProject->setNamespace('project_' . $id . '_internal'); + $dbForProject->setNamespace('project_' . $projectId . '_internal'); $storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size'); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'period' => '15m', - 'time' => time(), - 'metric' => 'storage.total', - 'value' => $storageTotal, - 'type' => 1, - ])); + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_storage.total'); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '30m', + 'time' => $time, + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $storageTotal)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_storage.total'); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '1d', + 'time' => $time, + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $storageTotal)); + } $collections = [ 'users' => [ @@ -352,7 +377,7 @@ $cli 'collections' => [ 'metricPrefix' => 'database', 'namespace' => 'internal', - 'subCollections' => [ // TODO better document this key + 'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting 'documents' => [ 'namespace' => 'external', ], @@ -366,20 +391,45 @@ $cli foreach ($collections as $collection => $options) { try { - $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); $count = $dbForProject->count($collection); - $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->setNamespace("project_{$projectId}_internal"); $metricPrefix = $options['metricPrefix'] ?? ''; $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; - - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => $metric, - 'value' => $count, - 'type' => 1, - ])); + + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } $subCollections = $options['subCollections'] ?? []; @@ -391,8 +441,8 @@ $cli $subCollectionCounts = []; //total project level count of sub collections do { - $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); - $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); + $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); + $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections if (empty($parents)) { continue; @@ -401,37 +451,95 @@ $cli $latestParent = $parents[array_key_last($parents)]; foreach ($parents as $parent) { - foreach ($subCollections as $subCollection => $subOptions) { - $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); + foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count + $dbForProject->setNamespace("project_{$projectId}_{$subOptions['namespace']}"); $count = $dbForProject->count($parent->getId()); - $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; - $dbForProject->setNamespace("project_{$id}_internal"); + $subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", - 'value' => $count, - 'type' => 1, - ])); + $dbForProject->setNamespace("project_{$projectId}_internal"); + + $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count"; + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } } } } while (!empty($parents)); - foreach ($subCollectionsCounts as $subCollection => $count) { - $dbForProject->setNamespace("project_{$id}_internal"); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", - 'value' => $count, - 'type' => 1, - ])); + /** + * Inserting project level counts for sub collections like database.documents.count + */ + foreach ($subCollectionCounts as $subCollection => $count) { + $dbForProject->setNamespace("project_{$projectId}_internal"); + + $metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count"; + + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } } - } catch (\Exception $e) { + } catch (\Exception$e) { Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}"); } } @@ -442,7 +550,7 @@ $cli $iterations++; $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); - + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); - }); \ No newline at end of file + }); From 87311797f2ece141467966599175cc3ef5313e58 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 30 Aug 2021 15:23:11 +0545 Subject: [PATCH 091/120] more comments --- app/tasks/usage.php | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 3fa90a3f43..e66bec113f 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -310,17 +310,14 @@ $cli } /** - * Aggregate MariaDB every 15 minutes - * Some of the queries here might contain full-table scans. - */ - if ($iterations % 30 == 0) { // Every 15 minutes - // Aggregate number of objects in database - // Get count of all the documents per collection - - // Buckets will have the same + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ + if ($iterations % 30 == 0) { // Every 15 minutes aggregate number of objects in database $latestProject = null; - do { + do { // Loop over all the projects $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); if (empty($projects)) { @@ -440,7 +437,7 @@ $cli $latestParent = null; $subCollectionCounts = []; //total project level count of sub collections - do { + do { // Loop over all the parent collection document for each sub collection $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections From ad81e414d99787b4516bdad56f5c3e7f59748506 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 1 Sep 2021 00:34:01 +0530 Subject: [PATCH 092/120] feat(response): add metric and metric list to response models --- src/Appwrite/Utopia/Response.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 0c4c6bf0aa..7e012bc6c0 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -33,6 +33,7 @@ use Appwrite\Utopia\Response\Model\Team; use Appwrite\Utopia\Response\Model\Locale; use Appwrite\Utopia\Response\Model\Log; use Appwrite\Utopia\Response\Model\Membership; +use Appwrite\Utopia\Response\Model\Metric; use Appwrite\Utopia\Response\Model\Permissions; use Appwrite\Utopia\Response\Model\Phone; use Appwrite\Utopia\Response\Model\Platform; @@ -194,6 +195,7 @@ class Response extends SwooleResponse ->setModel(new BaseList('Languages List', self::MODEL_LANGUAGE_LIST, 'languages', self::MODEL_LANGUAGE)) ->setModel(new BaseList('Currencies List', self::MODEL_CURRENCY_LIST, 'currencies', self::MODEL_CURRENCY)) ->setModel(new BaseList('Phones List', self::MODEL_PHONE_LIST, 'phones', self::MODEL_PHONE)) + ->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metric', self::MODEL_METRIC)) // Entities ->setModel(new Collection()) ->setModel(new Attribute()) @@ -229,6 +231,7 @@ class Response extends SwooleResponse ->setModel(new UsageBuckets()) ->setModel(new UsageFunctions()) ->setModel(new UsageProject()) + ->setModel(new Metric()) // Verification // Recovery // Tests (keep last) From ac0ff8ea0fcfc60375b61001a72ed6a4dded9bb0 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 1 Sep 2021 02:25:13 +0530 Subject: [PATCH 093/120] feat(response): use recursive function for swagger/openapi spec generation --- app/controllers/api/database.php | 2 +- app/controllers/api/storage.php | 2 +- app/controllers/api/users.php | 3 +- .../Specification/Format/OpenAPI3.php | 28 ++++++++++++--- .../Specification/Format/Swagger2.php | 34 ++++++++++++++----- src/Appwrite/Utopia/Response.php | 5 ++- 6 files changed, 55 insertions(+), 19 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index e2af571bf7..be0a346047 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -344,7 +344,7 @@ App::get('/v1/database/:collectionId/usage') ->label('scope', 'collections.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'database') - ->label('sdk.method', 'getUsage') + ->label('sdk.method', 'getCollectionUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USAGE_COLLECTION) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index ee6210d25d..69ed5ba536 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -723,7 +723,7 @@ App::get('/v1/storage/:bucketId/usage') ->label('scope', 'files.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'storage') - ->label('sdk.method', 'getUsage') + ->label('sdk.method', 'getBucketUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USAGE_BUCKETS) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 7fcdffcc8b..48c67daa82 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -18,6 +18,7 @@ use Utopia\Database\Validator\UID; use DeviceDetector\DeviceDetector; use Appwrite\Database\Validator\CustomId; use Appwrite\Utopia\Response\Model; +use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; @@ -621,7 +622,7 @@ App::get('/v1/users/usage') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USAGE_USERS) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) - ->param('provider', '', new WhiteList(['email', 'anonymous', 'oauth2-google', 'oauth2-apple'], true), 'Provider Name.', true) + ->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(function($value) { return "oauth-".$value; }, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true) ->inject('response') ->inject('dbForInternal') ->inject('register') diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index 353369eb65..32e2347c0e 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -21,6 +21,28 @@ class OpenAPI3 extends Format return 'Open API 3'; } + /** + * Get Used Models + * + * Recursively get all used models + * + * @param object $model + * @param array $models + * + * @return void + */ + protected function getUsedModels($model, array &$usedModels) + { + if (is_string($model) && !in_array($model, ['string', 'integer', 'boolean', 'json', 'float'])) { + $usedModels[] = $model; + return; + } + if (!is_object($model)) return; + foreach ($model->getRules() as $rule) { + $this->getUsedModels($rule['type'], $usedModels); + } + } + /** * Parse * @@ -352,11 +374,7 @@ class OpenAPI3 extends Format $output['paths'][$url][\strtolower($route->getMethod())] = $temp; } foreach ($this->models as $model) { - foreach ($model->getRules() as $rule) { - if (!in_array($rule['type'], ['string', 'integer', 'boolean', 'json', 'float'])) { - $usedModels[] = $rule['type']; - } - } + $this->getUsedModels($model, $usedModels); } foreach ($this->models as $model) { if (!in_array($model->getType(), $usedModels) && $model->getType() !== 'error') { diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index d357283a02..75cac416d7 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -21,6 +21,28 @@ class Swagger2 extends Format return 'Swagger 2'; } + /** + * Get Used Models + * + * Recursively get all used models + * + * @param object $model + * @param array $models + * + * @return void + */ + protected function getUsedModels($model, array &$usedModels) + { + if (is_string($model) && !in_array($model, ['string', 'integer', 'boolean', 'json', 'float'])) { + $usedModels[] = $model; + return; + } + if (!is_object($model)) return; + foreach ($model->getRules() as $rule) { + $this->getUsedModels($rule['type'], $usedModels); + } + } + /** * Parse * @@ -354,15 +376,11 @@ class Swagger2 extends Format $output['paths'][$url][\strtolower($route->getMethod())] = $temp; } foreach ($this->models as $model) { - foreach ($model->getRules() as $rule) { - if ( - in_array($model->getType(), $usedModels) - && !in_array($rule['type'], ['string', 'integer', 'boolean', 'json', 'float']) - ) { - $usedModels[] = $rule['type']; - } - } + $this->getUsedModels($model, $usedModels); } + + // var_dump($usedModels); + foreach ($this->models as $model) { if (!in_array($model->getType(), $usedModels)) { continue; diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 7e012bc6c0..329dc86432 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -51,7 +51,6 @@ use Appwrite\Utopia\Response\Model\UsageFunctions; use Appwrite\Utopia\Response\Model\UsageProject; use Appwrite\Utopia\Response\Model\UsageStorage; use Appwrite\Utopia\Response\Model\UsageUsers; -use Appwrite\Utopia\Response\Model\UsersUsage; use stdClass; /** @@ -195,7 +194,7 @@ class Response extends SwooleResponse ->setModel(new BaseList('Languages List', self::MODEL_LANGUAGE_LIST, 'languages', self::MODEL_LANGUAGE)) ->setModel(new BaseList('Currencies List', self::MODEL_CURRENCY_LIST, 'currencies', self::MODEL_CURRENCY)) ->setModel(new BaseList('Phones List', self::MODEL_PHONE_LIST, 'phones', self::MODEL_PHONE)) - ->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metric', self::MODEL_METRIC)) + ->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metrics', self::MODEL_METRIC, true, false)) // Entities ->setModel(new Collection()) ->setModel(new Attribute()) @@ -224,6 +223,7 @@ class Response extends SwooleResponse ->setModel(new Language()) ->setModel(new Currency()) ->setModel(new Phone()) + ->setModel(new Metric()) ->setModel(new UsageDatabase()) ->setModel(new UsageCollection()) ->setModel(new UsageUsers()) @@ -231,7 +231,6 @@ class Response extends SwooleResponse ->setModel(new UsageBuckets()) ->setModel(new UsageFunctions()) ->setModel(new UsageProject()) - ->setModel(new Metric()) // Verification // Recovery // Tests (keep last) From dd9cddc525b8bfe8178d68daffce673f0184b0f4 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 1 Sep 2021 16:54:36 +0530 Subject: [PATCH 094/120] feat(review): fix review comments --- src/Appwrite/Specification/Format/OpenAPI3.php | 2 +- src/Appwrite/Specification/Format/Swagger2.php | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index 32e2347c0e..5095c5395f 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -27,7 +27,7 @@ class OpenAPI3 extends Format * Recursively get all used models * * @param object $model - * @param array $models + * @param array $models * * @return void */ diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index 75cac416d7..2a3be45d17 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -27,7 +27,7 @@ class Swagger2 extends Format * Recursively get all used models * * @param object $model - * @param array $models + * @param array $models * * @return void */ @@ -379,8 +379,6 @@ class Swagger2 extends Format $this->getUsedModels($model, $usedModels); } - // var_dump($usedModels); - foreach ($this->models as $model) { if (!in_array($model->getType(), $usedModels)) { continue; From 3a29d6a565487d535a96860312a420ceb83c85c4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 2 Sep 2021 12:16:03 +0545 Subject: [PATCH 095/120] fix deletes, remove 15m period after refactor --- app/workers/deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 4d5bd0cacc..3142065fa5 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -105,7 +105,7 @@ class DeletesV1 extends Worker // Delete Usage stats $this->deleteByGroup('stats', [ new Query('time', Query::TYPE_LESSER, [$timestamp1d]), - new Query('period', Query::TYPE_EQUAL, ['1d', '15m']), + new Query('period', Query::TYPE_EQUAL, ['1d']), ], $dbForInternal); $this->deleteByGroup('stats', [ From 334a59e3c1e226a648c4f15f7e5aa28d6e2773ba Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 2 Sep 2021 12:17:32 +0545 Subject: [PATCH 096/120] schedule usage stats deletion --- app/tasks/maintenance.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index a5cf63f821..f30c2a9a42 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -56,11 +56,12 @@ $cli $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600');//36 hours $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days - Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention){ + Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { $time = date('d-m-Y H:i:s', time()); Console::info("[{$time}] Notifying deletes workers every {$interval} seconds"); notifyDeleteExecutionLogs($executionLogsRetention); notifyDeleteAbuseLogs($abuseLogsRetention); notifyDeleteAuditLogs($auditLogRetention); + notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d); }, $interval); }); \ No newline at end of file From 84e9881a6a4638f9253ca409d685442178ce3a39 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 2 Sep 2021 12:45:03 -0400 Subject: [PATCH 097/120] Delete from attribute/index tables on deleteCollection --- app/controllers/api/database.php | 5 +++++ app/workers/deletes.php | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 7df3172484..191ed38e5f 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -426,6 +426,11 @@ App::delete('/v1/database/collections/:collectionId') throw new Exception('Failed to remove collection from DB', 500); } + $deletes + ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $collection) + ; + $events ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)) ; diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 1894aaf76b..759409d286 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -44,6 +44,9 @@ class DeletesV1 extends Worker switch ($document->getCollection()) { // TODO@kodumbeats define these as constants somewhere + case 'collections': + $this->deleteCollection($document, $projectId); + break; case 'projects': $this->deleteProject($document); break; @@ -88,6 +91,25 @@ class DeletesV1 extends Worker public function shutdown(): void { } + + /** + * @param Document $document teams document + * @param string $projectId + */ + protected function deleteCollection(Document $document, string $projectId): void + { + $collectionId = $document->getId(); + + $dbForInternal = $this->getInternalDB($projectId); + + $this->deleteByGroup('attributes', [ + new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) + ], $dbForInternal); + + $this->deleteByGroup('indexes', [ + new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) + ], $dbForInternal); + } /** * @param Document $document teams document From 8c3ca6808c8a1d9c48b095b66f669d53fe91144d Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 2 Sep 2021 12:46:05 -0400 Subject: [PATCH 098/120] Test deleteCollection removal of attributes/indexes --- tests/e2e/Services/Database/DatabaseCustomServerTest.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index e5a75fa685..79750318b2 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -403,5 +403,13 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals(400, $tooMany['headers']['status-code']); $this->assertEquals('Index limit exceeded', $tooMany['body']['message']); + + $collection = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $collectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(204, $collection['headers']['status-code']); } } \ No newline at end of file From 1a21c8fdbbf53fd0ff94622244895a470016325e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 6 Sep 2021 11:48:44 +0545 Subject: [PATCH 099/120] fix constant value --- app/init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index 475bb3e40d..78c44076e5 100644 --- a/app/init.php +++ b/app/init.php @@ -89,7 +89,7 @@ const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_CERTIFICATES = 'certificates'; -const DELETE_TYPE_USAGE_STATS = 'certificates'; +const DELETE_TYPE_USAGE_STATS = 'usageStats'; // Mail Worker Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_RECOVERY = 'recovery'; From eccc19e5dc46fa6c087106a77b10da899d19339a Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 6 Sep 2021 12:13:20 +0530 Subject: [PATCH 100/120] Apply suggestions from code review Co-authored-by: Damodar Lohani --- app/controllers/api/database.php | 4 ++-- app/controllers/api/functions.php | 2 +- app/controllers/api/projects.php | 2 +- app/controllers/api/storage.php | 2 +- app/controllers/api/users.php | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index be0a346047..5a8a599900 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -270,7 +270,7 @@ App::get('/v1/database/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '15m', + 'period' => '30m', 'limit' => 48, ], '7d' => [ @@ -369,7 +369,7 @@ App::get('/v1/database/:collectionId/usage') if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '15m', + 'period' => '30m', 'limit' => 48, ], '7d' => [ diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index fadc323832..2a67c6d4c6 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -171,7 +171,7 @@ App::get('/v1/functions/:functionId/usage') if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '15m', + 'period' => '30m', 'limit' => 48, ], '7d' => [ diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 0d1c3b713a..3c7dbb8e5e 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -248,7 +248,7 @@ App::get('/v1/projects/:projectId/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '15m', + 'period' => '30m', 'limit' => 48, ], '7d' => [ diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 69ed5ba536..b9b2b8e8a1 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -665,7 +665,7 @@ App::get('/v1/storage/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '15m', + 'period' => '30m', 'limit' => 48, ], '7d' => [ diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 48c67daa82..c55f5382f5 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -634,7 +634,7 @@ App::get('/v1/users/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '15m', + 'period' => '30m', 'limit' => 48, ], '7d' => [ From 9f93ceec13cdb962d4c604bdc6ced7d046eefc34 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 6 Sep 2021 15:24:36 +0545 Subject: [PATCH 101/120] missing break --- app/workers/deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 3142065fa5..f9571949f2 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -81,7 +81,7 @@ class DeletesV1 extends Worker case DELETE_TYPE_USAGE_STATS: $this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']); - + break; default: Console::error('No delete operation for type: '.$type); break; From 1ecd9eecffab2f804140e89c21a8bdc548063950 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 7 Sep 2021 11:03:30 +0545 Subject: [PATCH 102/120] fix deletes interval --- app/tasks/maintenance.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index f30c2a9a42..fdfd7c942b 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -43,8 +43,8 @@ $cli { Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ 'type' => DELETE_TYPE_USAGE_STATS, - 'timestamp1d' => $interval1d, - 'timestamp30m' => $interval30m, + 'timestamp1d' => time() - $interval1d, + 'timestamp30m' => time() - $interval30m, ]); } From b38c77255e0d03e4e5206a5da09ede133d7ec508 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 8 Sep 2021 12:40:48 +0545 Subject: [PATCH 103/120] Update app/init.php Co-authored-by: Eldad A. Fux --- app/init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index 78c44076e5..4c1e6cf219 100644 --- a/app/init.php +++ b/app/init.php @@ -89,7 +89,7 @@ const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_CERTIFICATES = 'certificates'; -const DELETE_TYPE_USAGE_STATS = 'usageStats'; +const DELETE_TYPE_USAGE = 'usage'; // Mail Worker Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_RECOVERY = 'recovery'; From 5802212dd8d56d61ea2cf518fdc64d32ed5b9813 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 10 Sep 2021 16:08:49 +0545 Subject: [PATCH 104/120] fix usage daemon first time startup --- app/tasks/usage.php | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index e66bec113f..4ff5e8cad9 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -238,10 +238,27 @@ $cli /** * Aggregate InfluxDB every 30 seconds + * @var InfluxDB\Client $client */ $client = $register->get('influxdb'); if ($client) { + $attempts = 0; + $max = 10; + $sleep = 1; + $database = $client->selectDB('telegraf'); + do { // check if telegraf database is ready + $attempts++; + if(!in_array('telegraf', $client->listDatabases())) { + Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); + if($attempts >= $max) { + throw new \Exception('InfluxDB database not ready yet'); + } + sleep($sleep); + } else { + break; // leave the do-while if successful + } + } while ($attempts < $max); // sync data foreach ($globalMetrics as $metric => $options) { //for each metrics @@ -318,7 +335,23 @@ $cli $latestProject = null; do { // Loop over all the projects - $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + $attempts = 0; + $max = 10; + $sleep = 1; + + do { // list projects + try { + $attempts++; + $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + break; // leave the do-while if successful + } catch (\Exception $e) { + Console::warning("Console DB not ready yet. Retrying ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('Failed access console db: ' . $e->getMessage()); + } + sleep($sleep); + } + } while ($attempts < $max); if (empty($projects)) { continue; From 8cde715d04027c4f7b9559c15b65ff19f4d7d7e1 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 15 Sep 2021 11:28:40 -0400 Subject: [PATCH 105/120] Use correct constant when deleting usage stats --- app/tasks/maintenance.php | 2 +- app/workers/deletes.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index fdfd7c942b..e0ef1adc01 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -42,7 +42,7 @@ $cli function notifyDeleteUsageStats(int $interval30m, int $interval1d) { Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ - 'type' => DELETE_TYPE_USAGE_STATS, + 'type' => DELETE_TYPE_USAGE, 'timestamp1d' => time() - $interval1d, 'timestamp30m' => time() - $interval30m, ]); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index f9571949f2..7016244f92 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -79,7 +79,7 @@ class DeletesV1 extends Worker $this->deleteCertificates($document); break; - case DELETE_TYPE_USAGE_STATS: + case DELETE_TYPE_USAGE: $this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']); break; default: From e90036f67451b3a52bdedc473b240f80c154deb7 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 15 Sep 2021 11:46:08 -0400 Subject: [PATCH 106/120] Exit tests if a container has status exited --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index a90890423c..325865fcd2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,12 @@ install: script: - docker ps -a +# Tests should fail if any container is in exited status +- ALL_UP=`docker ps -aq --filter "status=exited"` +- > + if [[ "$ALL_UP" != "" ]]; then + exit 1 + fi - docker-compose logs appwrite - docker-compose logs mariadb - docker-compose logs appwrite-worker-functions From bfdf6de15f101ec1cfd62373e646cb0862ab3eb9 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 15 Sep 2021 14:02:18 -0400 Subject: [PATCH 107/120] Delete collection document from internal collections table --- app/workers/deletes.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 78ceeb45a6..f557a3a549 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -112,6 +112,8 @@ class DeletesV1 extends Worker $this->deleteByGroup('indexes', [ new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) ], $dbForInternal); + + $dbForInternal->deleteDocument('collections', $collectionId); } /** From 43d5d67f4321bd5e33af515228e152ca1e058302 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 10:46:59 +0530 Subject: [PATCH 108/120] feat(tests): check for failing test --- phpunit.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index f0a2813f5a..56885de032 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -19,7 +19,7 @@ ./tests/e2e/Client.php ./tests/e2e/General ./tests/e2e/Scopes - ./tests/e2e/Services/Account + ./tests/e2e/Services/Functions/FunctionsBase.php ./tests/e2e/Services/Functions/FunctionsCustomServerTest.php ./tests/e2e/Services/Functions/FunctionsCustomClientTest.php From f8067b9cfbb72ba2d5368a2daa3c5e9bdee38f79 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 11:08:21 +0530 Subject: [PATCH 109/120] feat(tests): check for failing test --- phpunit.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index 56885de032..57e374c80c 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -13,14 +13,14 @@ - ./tests/unit/ + - ./tests/e2e/Client.php + + ./tests/e2e/Services/Account + - ./tests/e2e/Services/Functions/FunctionsBase.php + \ No newline at end of file From 3f4a0756ad60c104f6e459285ca6f26dfc3604b7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 12:36:24 +0530 Subject: [PATCH 110/120] feat(tests): check for failing test --- phpunit.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index 57e374c80c..572dceaeea 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -19,10 +19,10 @@ - ./tests/e2e/Services/Account - + ./tests/e2e/Services/Database - ./tests/e2e/Services/Health + - ./tests/e2e/Services/Database - + ./tests/e2e/Services/Health ./tests/e2e/Services/Locale - ./tests/e2e/Services/Projects - ./tests/e2e/Services/Storage - ./tests/e2e/Services/Teams - ./tests/e2e/Services/Users - ./tests/e2e/Services/Workers - ./tests/e2e/Services/Webhooks --> + + + + + + From b580810c7ddef61cd0573cb0b259257a39e6c041 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 13:29:20 +0530 Subject: [PATCH 112/120] feat(tests): check for failing test --- .travis.yml | 3 +++ phpunit.xml | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a90890423c..f5e7ee4681 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,9 @@ arch: os: linux +vm: + size: x-large + language: shell notifications: diff --git a/phpunit.xml b/phpunit.xml index 44cc631d0e..bd38ab510a 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -19,11 +19,11 @@ - + ./tests/e2e/Services/Account - ./tests/e2e/Services/Health - ./tests/e2e/Services/Locale + From cad09af1bdb37c8d31a072ae641b83702f306884 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 13:49:18 +0530 Subject: [PATCH 113/120] feat(tests): check for failing test --- .travis.yml | 2 +- phpunit.xml | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index f5e7ee4681..2fa9ff9357 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ arch: os: linux vm: - size: x-large + size: large language: shell diff --git a/phpunit.xml b/phpunit.xml index bd38ab510a..f0a2813f5a 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -13,26 +13,26 @@ - + ./tests/unit/ - + ./tests/e2e/Scopes ./tests/e2e/Services/Account - - - - - - - - - - + ./tests/e2e/Services/Functions/FunctionsCustomClientTest.php \ No newline at end of file From 086f6312116cb1eea975e36286738b1c4f9ae955 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 17:04:23 +0530 Subject: [PATCH 114/120] feat(tests): check for failing test --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2fa9ff9357..fe7bc3098e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,9 +6,10 @@ arch: os: linux +# Small change to check build failure vm: - size: large - + size: large + language: shell notifications: From 351b252c2cf2a1aa6e64242c0bc9b8a046f6cad0 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 17:35:53 +0530 Subject: [PATCH 115/120] feat(tests): check for failing test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fe7bc3098e..077c8cc787 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ arch: os: linux -# Small change to check build failure +# Small change to check build vm: size: large From 2853b7f52a5958a293b6877491cc9ad122529d33 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 18:10:18 +0530 Subject: [PATCH 116/120] feat(tests): check for failing test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 077c8cc787..d2f93874b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ arch: os: linux -# Small change to check build +# Small change to check vm: size: large From c03dc64adb8d1c524119abaac5026c75acd50b33 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 18:48:23 +0530 Subject: [PATCH 117/120] feat(tests): check for failing test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d2f93874b5..89dad4bbab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ arch: os: linux -# Small change to check +# Small change to vm: size: large From 13024e97f2ebe8dc244b9f1a0fd20fb4be73ae3f Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 19:27:12 +0530 Subject: [PATCH 118/120] feat(tests): check for failing test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 89dad4bbab..a1a2f6bd65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ arch: os: linux -# Small change to +# Small change vm: size: large From b390c663a456bd855ef93847dacff78615a21e84 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 16 Sep 2021 20:57:25 -0400 Subject: [PATCH 119/120] Upgrade travis instance from medium to large --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index a90890423c..2fa9ff9357 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,9 @@ arch: os: linux +vm: + size: large + language: shell notifications: From cb7c810207c79eddf8daac344eadc9b8ed4092e4 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Sun, 19 Sep 2021 11:49:24 -0400 Subject: [PATCH 120/120] Drop collection table in deletes worker --- app/controllers/api/database.php | 2 -- app/workers/deletes.php | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index dcef7105ea..26d93ae5be 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -620,8 +620,6 @@ App::delete('/v1/database/collections/:collectionId') throw new Exception('Collection not found', 404); } - $dbForExternal->deleteCollection($collectionId); // TDOD move to DB worker - if (!$dbForInternal->deleteDocument('collections', $collectionId)) { throw new Exception('Failed to remove collection from DB', 500); } diff --git a/app/workers/deletes.php b/app/workers/deletes.php index f557a3a549..8382e14b14 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -104,6 +104,7 @@ class DeletesV1 extends Worker $collectionId = $document->getId(); $dbForInternal = $this->getInternalDB($projectId); + $dbForExternal = $this->getExternalDB($projectId); $this->deleteByGroup('attributes', [ new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) @@ -113,7 +114,7 @@ class DeletesV1 extends Worker new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) ], $dbForInternal); - $dbForInternal->deleteDocument('collections', $collectionId); + $dbForExternal->deleteCollection($collectionId); } /**