diff --git a/.env b/.env index 65fb54cb04..c616312e50 100644 --- a/.env +++ b/.env @@ -76,8 +76,8 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 _APP_MAINTENANCE_RETENTION_ABUSE=86400 _APP_MAINTENANCE_RETENTION_AUDIT=1209600 -_APP_USAGE_TIMESERIES_INTERVAL=2 -_APP_USAGE_DATABASE_INTERVAL=15 +_APP_USAGE_AGGREGATION_INTERVAL=5 +_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 _APP_USAGE_STATS=enabled _APP_LOGGING_PROVIDER= _APP_LOGGING_CONFIG= diff --git a/Dockerfile b/Dockerfile index f88168ba67..b3a408e722 100755 --- a/Dockerfile +++ b/Dockerfile @@ -243,13 +243,13 @@ ENV _APP_SERVER=swoole \ _APP_SETUP=self-hosted \ _APP_VERSION=$VERSION \ _APP_USAGE_STATS=enabled \ - _APP_USAGE_TIMESERIES_INTERVAL=30 \ - _APP_USAGE_DATABASE_INTERVAL=900 \ + _APP_USAGE_AGGREGATION_INTERVAL=30 \ # 14 Days = 1209600 s _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 \ _APP_MAINTENANCE_RETENTION_AUDIT=1209600 \ # 1 Day = 86400 s _APP_MAINTENANCE_RETENTION_ABUSE=86400 \ + _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 \ _APP_MAINTENANCE_INTERVAL=86400 \ _APP_LOGGING_PROVIDER= \ _APP_LOGGING_CONFIG= diff --git a/app/config/variables.php b/app/config/variables.php index 9f3bc018e8..40bb99f8f2 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -170,8 +170,8 @@ return [ ], [ 'name' => '_APP_USAGE_AGGREGATION_INTERVAL', - 'description' => 'Deprecated since 1.0.0, use `_APP_USAGE_TIMESERIES_INTERVAL` and `_APP_USAGE_DATABASE_INTERVAL` instead.', - 'introduction' => '0.10.0', + 'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats and syncing it to Database from TimeSeries data. The default value is 30 seconds. Reintroduced in 1.1.0.', + 'introduction' => '1.1.0', 'default' => '30', 'required' => false, 'question' => '', @@ -179,7 +179,7 @@ return [ ], [ 'name' => '_APP_USAGE_TIMESERIES_INTERVAL', - 'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats and syncing it to Appwrite Database from Timeseries Database. The default value is 30 seconds.', + 'description' => 'Deprecated since 1.1.0 use _APP_USAGE_AGGREGATION_INTERVAL instead.', 'introduction' => '1.0.0', 'default' => '30', 'required' => false, @@ -188,7 +188,7 @@ return [ ], [ 'name' => '_APP_USAGE_DATABASE_INTERVAL', - 'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats from data in Appwrite Database. The default value is 15 minutes.', + 'description' => 'Deprecated since 1.1.0 use _APP_USAGE_AGGREGATION_INTERVAL instead.', 'introduction' => '1.0.0', 'default' => '900', 'required' => false, @@ -857,7 +857,16 @@ return [ 'required' => false, 'question' => '', 'filter' => '' - ] + ], + [ + 'name' => '_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', + 'description' => 'The maximum duration (in seconds) upto which to retain hourly usage metrics. The default value is 8640000 seconds (100 days).', + 'introduction' => '', + 'default' => '8640000', + 'required' => false, + 'question' => '', + 'filter' => '' + ], ], ], ]; diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 19264454e0..acec695233 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2467,8 +2467,8 @@ App::get('/v1/databases/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -2529,7 +2529,7 @@ App::get('/v1/databases/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ @@ -2586,8 +2586,8 @@ App::get('/v1/databases/:databaseId/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -2643,7 +2643,7 @@ App::get('/v1/databases/:databaseId/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ @@ -2706,8 +2706,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -2758,7 +2758,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 7632e10926..3ab9558914 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -237,8 +237,8 @@ App::get('/v1/functions/:functionId/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -292,7 +292,7 @@ App::get('/v1/functions/:functionId/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ @@ -340,8 +340,8 @@ App::get('/v1/functions/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -395,7 +395,7 @@ App::get('/v1/functions/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index a5c9177f82..46a6f09e58 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -271,8 +271,8 @@ App::get('/v1/projects/:projectId/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -328,7 +328,7 @@ App::get('/v1/projects/:projectId/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index f236285749..176d32081f 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1454,8 +1454,8 @@ App::get('/v1/storage/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -1513,7 +1513,7 @@ App::get('/v1/storage/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ @@ -1571,8 +1571,8 @@ App::get('/v1/storage/:bucketId/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -1624,7 +1624,7 @@ App::get('/v1/storage/:bucketId/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index c95105b775..6211176a05 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1116,8 +1116,8 @@ App::get('/v1/users/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $periods = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, + 'period' => '1h', + 'limit' => 24, ], '7d' => [ 'period' => '1d', @@ -1171,7 +1171,7 @@ App::get('/v1/users/usage') while ($backfill > 0) { $last = $limit - $backfill - 1; // array index of last added metric $diff = match ($period) { // convert period to seconds for unix timestamp math - '30m' => 1800, + '1h' => 3600, '1d' => 86400, }; $stats[$metric][] = [ diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index e07f405140..261f709d4b 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -16,6 +16,7 @@ use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapters\TimeLimit; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; +use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; @@ -47,6 +48,47 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar return $label; }; +$databaseListener = function (string $event, Document $document, Stats $usage) { + $multiplier = 1; + if ($event === Database::EVENT_DOCUMENT_DELETE) { + $multiplier = -1; + } + + $collection = $document->getCollection(); + switch ($collection) { + case 'users': + $usage->setParam('users.{scope}.count.total', 1 * $multiplier); + break; + case 'databases': + $usage->setParam('databases.{scope}.count.total', 1 * $multiplier); + break; + case 'buckets': + $usage->setParam('buckets.{scope}.count.total', 1 * $multiplier); + break; + case 'deployments': + $usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size') * $multiplier); + break; + default: + if (strpos($collection, 'bucket_') === 0) { + $usage + ->setParam('bucketId', $document->getAttribute('bucketId')) + ->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal') * $multiplier) + ->setParam('files.{scope}.count.total', 1 * $multiplier); + } elseif (strpos($collection, 'database_') === 0) { + $usage + ->setParam('databaseId', $document->getAttribute('databaseId')); + if (strpos($collection, '_collection_') !== false) { + $usage + ->setParam('collectionId', $document->getAttribute('$collectionId')) + ->setParam('documents.{scope}.count.total', 1 * $multiplier); + } else { + $usage->setParam('collections.{scope}.count.total', 1 * $multiplier); + } + } + break; + } +}; + App::init() ->groups(['api']) ->inject('utopia') @@ -62,7 +104,7 @@ App::init() ->inject('database') ->inject('dbForProject') ->inject('mode') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Mail $mails, Stats $usage, Delete $deletes, EventDatabase $database, Database $dbForProject, string $mode) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Mail $mails, Stats $usage, Delete $deletes, EventDatabase $database, Database $dbForProject, string $mode) use ($databaseListener) { $route = $utopia->match($request); @@ -149,6 +191,7 @@ App::init() ->setUser($user); $usage + ->setParam('projectInternalId', $project->getInternalId()) ->setParam('projectId', $project->getId()) ->setParam('project.{scope}.network.requests', 1) ->setParam('httpMethod', $request->getMethod()) @@ -158,6 +201,10 @@ App::init() $deletes->setProject($project); $database->setProject($project); + $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, fn ($event, Document $document) => $databaseListener($event, $document, $usage)); + + $dbForProject->on(Database::EVENT_DOCUMENT_DELETE, fn ($event, Document $document) => $databaseListener($event, $document, $usage)); + $useCache = $route->getLabel('cache', false); if ($useCache) { @@ -404,7 +451,6 @@ App::shutdown() if ( App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled' && $project->getId() - && $mode !== APP_MODE_ADMIN // TODO: add check to make sure user is admin && !empty($route->getLabel('sdk.namespace', null)) ) { // Don't calculate console usage on admin mode $metric = $route->getLabel('usage.metric', ''); diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index 42b5ed00dc..92f260ce46 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -78,12 +78,11 @@ $cli ->trigger(); } - function notifyDeleteUsageStats(int $interval30m, int $interval1d) + function notifyDeleteUsageStats(int $usageStatsRetentionHourly) { (new Delete()) ->setType(DELETE_TYPE_USAGE) - ->setDateTime1d(DateTime::addSeconds(new \DateTime(), -1 * $interval1d)) - ->setDateTime30m(DateTime::addSeconds(new \DateTime(), -1 * $interval30m)) + ->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly)) ->trigger(); } @@ -144,11 +143,11 @@ $cli $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600'); $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400'); - $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600'); //36 hours - $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days + $usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days + $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days - Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d, $cacheRetention) { + Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetentionHourly, $cacheRetention) { $database = getConsoleDB(); $time = DateTime::now(); @@ -157,7 +156,7 @@ $cli notifyDeleteExecutionLogs($executionLogsRetention); notifyDeleteAbuseLogs($abuseLogsRetention); notifyDeleteAuditLogs($auditLogRetention); - notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d); + notifyDeleteUsageStats($usageStatsRetentionHourly); notifyDeleteConnections(); notifyDeleteExpiredSessions(); renewCertificates($database); diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 6028781644..d940dd5e49 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -2,10 +2,6 @@ global $cli, $register; -use Appwrite\Stats\Usage; -use Appwrite\Stats\UsageDB; -use Appwrite\Usage\Calculators\Aggregator; -use Appwrite\Usage\Calculators\Database; use Appwrite\Usage\Calculators\TimeSeries; use InfluxDB\Database as InfluxDatabase; use Utopia\App; @@ -114,65 +110,29 @@ $logError = function (Throwable $error, string $action = 'syncUsageStats') use ( Console::warning($error->getTraceAsString()); }; - -function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void -{ - $interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default) - $region = App::getEnv('region', 'default'); - $usage = new TimeSeries($region, $database, $influxDB, $logError); - - Console::loop(function () use ($interval, $usage) { - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds"); - $loopStart = microtime(true); - - $usage->collect(); - - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregation took {$loopTook} seconds"); - }, $interval); -} - -function aggregateDatabase(UtopiaDatabase $database, callable $logError): void -{ - $interval = (int) App::getEnv('_APP_USAGE_DATABASE_INTERVAL', '900'); // 15 minutes (by default) - $region = App::getEnv('region', 'default'); - $usage = new Database($region, $database, $logError); - $aggregrator = new Aggregator($region, $database, $logError); - - Console::loop(function () use ($interval, $usage, $aggregrator) { - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregating database usage every {$interval} seconds."); - $loopStart = microtime(true); - $usage->collect(); - $aggregrator->collect(); - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - - Console::info("[{$now}] Aggregation took {$loopTook} seconds"); - }, $interval); -} - $cli ->task('usage') - ->param('type', 'timeseries', new WhiteList(['timeseries', 'database'])) ->desc('Schedules syncing data from influxdb to Appwrite console db') - ->action(function (string $type) use ($register, $logError) { + ->action(function () use ($register, $logError) { Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); $database = getDatabase($register, '_console'); $influxDB = getInfluxDB($register); - switch ($type) { - case 'timeseries': - aggregateTimeseries($database, $influxDB, $logError); - break; - case 'database': - aggregateDatabase($database, $logError); - break; - default: - Console::error("Unsupported usage aggregation type"); - } + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) + $region = App::getEnv('region', 'default'); + $usage = new TimeSeries($region, $database, $influxDB, $logError); + + Console::loop(function () use ($interval, $usage) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds"); + $loopStart = microtime(true); + + $usage->collect(); + + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); + }, $interval); }); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 0442d8456b..e14e434a84 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -150,6 +150,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_SMS_PROVIDER - _APP_SMS_FROM @@ -549,13 +550,12 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - appwrite-usage-timeseries: + appwrite-usage: image: /: - entrypoint: - - usage - - --type=timeseries - container_name: appwrite-usage-timeseries + entrypoint: usage + container_name: appwrite-usage <<: *x-logging restart: unless-stopped networks: @@ -573,40 +573,7 @@ services: - _APP_DB_PASS - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_LOGGING_PROVIDER - - _APP_LOGGING_CONFIG - - appwrite-usage-database: - image: /: - entrypoint: - - usage - - --type=database - container_name: appwrite-usage-database - <<: *x-logging - restart: unless-stopped - networks: - - appwrite - depends_on: - - influxdb - - mariadb - environment: - - _APP_ENV - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL + - _APP_USAGE_AGGREGATION_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b015043b1d..44ae1136a8 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -105,7 +105,7 @@ class DeletesV1 extends Worker break; case DELETE_TYPE_USAGE: - $this->deleteUsageStats($this->args['dateTime1d'], $this->args['dateTime30m']); + $this->deleteUsageStats($this->args['dateTime1d'], $this->args['hourlyUsageRetentionDatetime']); break; case DELETE_TYPE_CACHE_BY_RESOURCE: @@ -215,21 +215,15 @@ class DeletesV1 extends Worker /** * @param string $datetime1d - * @param string $datetime30m + * @param string $hourlyUsageRetentionDatetime */ - protected function deleteUsageStats(string $datetime1d, string $datetime30m) + protected function deleteUsageStats(string $hourlyUsageRetentionDatetime) { - $this->deleteForProjectIds(function (string $projectId) use ($datetime1d, $datetime30m) { + $this->deleteForProjectIds(function (string $projectId) use ($hourlyUsageRetentionDatetime) { $dbForProject = $this->getProjectDB($projectId); - // Delete Usage stats $this->deleteByGroup('stats', [ - Query::lessThan('time', $datetime1d), - Query::equal('period', ['1d']), - ], $dbForProject); - - $this->deleteByGroup('stats', [ - Query::lessThan('time', $datetime30m), - Query::equal('period', ['30m']), + Query::lessThan('time', $hourlyUsageRetentionDatetime), + Query::equal('period', ['1h']), ], $dbForProject); }); } diff --git a/composer.lock b/composer.lock index 34609a4285..3b4148a950 100644 --- a/composer.lock +++ b/composer.lock @@ -2959,16 +2959,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.15.1", + "version": "v4.15.2", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "0ef6c55a3f47f89d7a374e6f835197a0b5fcf900" + "reference": "f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/0ef6c55a3f47f89d7a374e6f835197a0b5fcf900", - "reference": "0ef6c55a3f47f89d7a374e6f835197a0b5fcf900", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc", + "reference": "f59bbe44bf7d96f24f3e2b4ddc21cd52c1d2adbc", "shasum": "" }, "require": { @@ -3009,9 +3009,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.1" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.15.2" }, - "time": "2022-09-04T07:30:47+00:00" + "time": "2022-11-12T15:38:23+00:00" }, { "name": "phar-io/manifest", @@ -4841,16 +4841,16 @@ }, { "name": "symfony/polyfill-ctype", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", - "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4" + "reference": "5bbc823adecdae860bb64756d639ecfec17b050a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", - "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/5bbc823adecdae860bb64756d639ecfec17b050a", + "reference": "5bbc823adecdae860bb64756d639ecfec17b050a", "shasum": "" }, "require": { @@ -4865,7 +4865,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "1.26-dev" + "dev-main": "1.27-dev" }, "thanks": { "name": "symfony/polyfill", @@ -4903,7 +4903,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/v1.26.0" + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.27.0" }, "funding": [ { @@ -4919,20 +4919,20 @@ "type": "tidelift" } ], - "time": "2022-05-24T11:49:31+00:00" + "time": "2022-11-03T14:55:06+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.26.0", + "version": "v1.27.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e" + "reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", - "reference": "9344f9cb97f3b19424af1a21a3b0e75b0a7d8d7e", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/8ad114f6b39e2c98a8b0e3bd907732c207c2b534", + "reference": "8ad114f6b39e2c98a8b0e3bd907732c207c2b534", "shasum": "" }, "require": { @@ -4947,7 +4947,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "1.26-dev" + "dev-main": "1.27-dev" }, "thanks": { "name": "symfony/polyfill", @@ -4986,7 +4986,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.26.0" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.27.0" }, "funding": [ { @@ -5002,7 +5002,7 @@ "type": "tidelift" } ], - "time": "2022-05-24T11:49:31+00:00" + "time": "2022-11-03T14:55:06+00:00" }, { "name": "textalk/websocket", diff --git a/docker-compose.yml b/docker-compose.yml index ea9241a9d3..18ea38f658 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -171,6 +171,7 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_SMS_PROVIDER - _APP_SMS_FROM @@ -602,13 +603,12 @@ services: - _APP_MAINTENANCE_RETENTION_CACHE - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - appwrite-usage-timeseries: - entrypoint: - - usage - - --type=timeseries + appwrite-usage: + entrypoint: usage <<: *x-logging - container_name: appwrite-usage-timeseries + container_name: appwrite-usage image: appwrite-dev networks: - appwrite @@ -629,43 +629,7 @@ services: - _APP_DB_PASS - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - - _APP_LOGGING_PROVIDER - - _APP_LOGGING_CONFIG - - appwrite-usage-database: - entrypoint: - - usage - - --type=database - <<: *x-logging - container_name: appwrite-usage-database - image: appwrite-dev - networks: - - appwrite - volumes: - - ./app:/usr/src/code/app - - ./src:/usr/src/code/src - - ./dev:/usr/local/dev - depends_on: - - influxdb - - mariadb - environment: - - _APP_ENV - - _APP_OPENSSL_KEY_V1 - - _APP_DB_HOST - - _APP_DB_PORT - - _APP_DB_SCHEMA - - _APP_DB_USER - - _APP_DB_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL + - _APP_USAGE_AGGREGATION_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER diff --git a/src/Appwrite/Event/Delete.php b/src/Appwrite/Event/Delete.php index 72ace2a86d..d1519121a6 100644 --- a/src/Appwrite/Event/Delete.php +++ b/src/Appwrite/Event/Delete.php @@ -11,8 +11,7 @@ class Delete extends Event protected ?Document $document = null; protected ?string $resource = null; protected ?string $datetime = null; - protected ?string $dateTime30m = null; - protected ?string $dateTime1d = null; + protected ?string $hourlyUsageRetentionDatetime = null; public function __construct() @@ -56,26 +55,14 @@ class Delete extends Event } /** - * Set datetime for 1 day interval. + * Sets datetime for 1h interval. * * @param string $datetime * @return self */ - public function setDateTime1d(string $datetime): self + public function setUsageRetentionHourlyDateTime(string $datetime): self { - $this->dateTime1d = $datetime; - return $this; - } - - /** - * Sets datetime for 30m interval. - * - * @param string $datetime - * @return self - */ - public function setDateTime30m(string $datetime): self - { - $this->dateTime30m = $datetime; + $this->hourlyUsageRetentionDatetime = $datetime; return $this; } @@ -140,8 +127,7 @@ class Delete extends Event 'document' => $this->document, 'resource' => $this->resource, 'datetime' => $this->datetime, - 'dateTime1d' => $this->dateTime1d, - 'dateTime30m' => $this->dateTime30m, + 'hourlyUsageRetentionDatetime' => $this->hourlyUsageRetentionDatetime, ]); } } diff --git a/src/Appwrite/Usage/Calculators/Aggregator.php b/src/Appwrite/Usage/Calculators/Aggregator.php deleted file mode 100644 index 67cb18fe56..0000000000 --- a/src/Appwrite/Usage/Calculators/Aggregator.php +++ /dev/null @@ -1,231 +0,0 @@ -database->setNamespace('_' . $projectId); - - $databasesGeneralMetrics = [ - 'databases.$all.requests.create', - 'databases.$all.requests.read', - 'databases.$all.requests.update', - 'databases.$all.requests.delete', - 'collections.$all.requests.create', - 'collections.$all.requests.read', - 'collections.$all.requests.update', - 'collections.$all.requests.delete', - 'documents.$all.requests.create', - 'documents.$all.requests.read', - 'documents.$all.requests.update', - 'documents.$all.requests.delete' - ]; - - foreach ($databasesGeneralMetrics as $metric) { - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - - $databasesDatabaseMetrics = [ - 'collections.databaseId.requests.create', - 'collections.databaseId.requests.read', - 'collections.databaseId.requests.update', - 'collections.databaseId.requests.delete', - 'documents.databaseId.requests.create', - 'documents.databaseId.requests.read', - 'documents.databaseId.requests.update', - 'documents.databaseId.requests.delete', - ]; - - $this->foreachDocument($projectId, 'databases', [], function (Document $database) use ($databasesDatabaseMetrics, $projectId) { - $databaseId = $database->getId(); - foreach ($databasesDatabaseMetrics as $metric) { - $metric = str_replace('databaseId', $databaseId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - - $databasesCollectionMetrics = [ - 'documents.' . $databaseId . '/collectionId.requests.create', - 'documents.' . $databaseId . '/collectionId.requests.read', - 'documents.' . $databaseId . '/collectionId.requests.update', - 'documents.' . $databaseId . '/collectionId.requests.delete', - ]; - - $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function (Document $collection) use ($databasesCollectionMetrics, $projectId) { - $collectionId = $collection->getId(); - foreach ($databasesCollectionMetrics as $metric) { - $metric = str_replace('collectionId', $collectionId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - }); - }); - } - - protected function aggregateStorageMetrics(string $projectId): void - { - $this->database->setNamespace('_' . $projectId); - - $storageGeneralMetrics = [ - 'buckets.$all.requests.create', - 'buckets.$all.requests.read', - 'buckets.$all.requests.update', - 'buckets.$all.requests.delete', - 'files.$all.requests.create', - 'files.$all.requests.read', - 'files.$all.requests.update', - 'files.$all.requests.delete', - ]; - - foreach ($storageGeneralMetrics as $metric) { - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - - $storageBucketMetrics = [ - 'files.bucketId.requests.create', - 'files.bucketId.requests.read', - 'files.bucketId.requests.update', - 'files.bucketId.requests.delete', - ]; - - $this->foreachDocument($projectId, 'buckets', [], function (Document $bucket) use ($storageBucketMetrics, $projectId) { - $bucketId = $bucket->getId(); - foreach ($storageBucketMetrics as $metric) { - $metric = str_replace('bucketId', $bucketId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - }); - } - - protected function aggregateFunctionMetrics(string $projectId): void - { - $this->database->setNamespace('_' . $projectId); - - $functionsGeneralMetrics = [ - 'project.$all.compute.total', - 'project.$all.compute.time', - 'executions.$all.compute.total', - 'executions.$all.compute.success', - 'executions.$all.compute.failure', - 'executions.$all.compute.time', - 'builds.$all.compute.total', - 'builds.$all.compute.success', - 'builds.$all.compute.failure', - 'builds.$all.compute.time', - ]; - - foreach ($functionsGeneralMetrics as $metric) { - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - - $functionMetrics = [ - 'executions.functionId.compute.total', - 'executions.functionId.compute.success', - 'executions.functionId.compute.failure', - 'executions.functionId.compute.time', - 'builds.functionId.compute.total', - 'builds.functionId.compute.success', - 'builds.functionId.compute.failure', - 'builds.functionId.compute.time', - ]; - - $this->foreachDocument($projectId, 'functions', [], function (Document $function) use ($functionMetrics, $projectId) { - $functionId = $function->getId(); - foreach ($functionMetrics as $metric) { - $metric = str_replace('functionId', $functionId, $metric); - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - }); - } - - protected function aggregateUsersMetrics(string $projectId): void - { - $metrics = [ - 'users.$all.requests.create', - 'users.$all.requests.read', - 'users.$all.requests.update', - 'users.$all.requests.delete', - 'sessions.$all.requests.create', - 'sessions.$all.requests.delete' - ]; - - foreach ($metrics as $metric) { - $this->aggregateDailyMetric($projectId, $metric); - $this->aggregateMonthlyMetric($projectId, $metric); - } - } - - protected function aggregateGeneralMetrics(string $projectId): void - { - $this->aggregateDailyMetric($projectId, 'project.$all.network.requests'); - $this->aggregateDailyMetric($projectId, 'project.$all.network.bandwidth'); - $this->aggregateDailyMetric($projectId, 'project.$all.network.inbound'); - $this->aggregateDailyMetric($projectId, 'project.$all.network.outbound'); - $this->aggregateMonthlyMetric($projectId, 'project.$all.network.requests'); - $this->aggregateMonthlyMetric($projectId, 'project.$all.network.bandwidth'); - $this->aggregateMonthlyMetric($projectId, 'project.$all.network.inbound'); - $this->aggregateMonthlyMetric($projectId, 'project.$all.network.outbound'); - } - - protected function aggregateDailyMetric(string $projectId, string $metric): void - { - $beginOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T00:00:00.000'))->format(DateTime::RFC3339); - $endOfDay = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-d\T23:59:59.999'))->format(DateTime::RFC3339); - - $this->database->setNamespace('_' . $projectId); - $value = (int) $this->database->sum('stats', 'value', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['30m']), - Query::greaterThanEqual('time', $beginOfDay), - Query::lessThanEqual('time', $endOfDay), - ]); - $this->createOrUpdateMetric($projectId, $metric, '1d', $beginOfDay, $value); - } - - protected function aggregateMonthlyMetric(string $projectId, string $metric): void - { - $beginOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339); - $endOfMonth = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-t\T23:59:59.999'))->format(DateTime::RFC3339); - $this->database->setNamespace('_' . $projectId); - $value = (int) $this->database->sum('stats', 'value', [ - Query::equal('metric', [$metric]), - Query::equal('period', ['1d']), - Query::greaterThanEqual('time', $beginOfMonth), - Query::lessThanEqual('time', $endOfMonth), - ]); - $this->createOrUpdateMetric($projectId, $metric, '1mo', $beginOfMonth, $value); - } - - /** - * Collect Stats - * Collect all database related stats - * - * @return void - */ - public function collect(): void - { - $this->foreachDocument('console', 'projects', [], function (Document $project) { - $projectId = $project->getInternalId(); - - // Aggregate new metrics from already collected usage metrics - // for lower time period (1day and 1 month metric from 30 minute metrics) - $this->aggregateGeneralMetrics($projectId); - $this->aggregateFunctionMetrics($projectId); - $this->aggregateDatabaseMetrics($projectId); - $this->aggregateStorageMetrics($projectId); - $this->aggregateUsersMetrics($projectId); - }); - } -} diff --git a/src/Appwrite/Usage/Calculators/Database.php b/src/Appwrite/Usage/Calculators/Database.php deleted file mode 100644 index 64447a0ad7..0000000000 --- a/src/Appwrite/Usage/Calculators/Database.php +++ /dev/null @@ -1,364 +0,0 @@ - '30m', - 'multiplier' => 1800, - ], - [ - 'key' => '1d', - 'multiplier' => 86400, - ], - ]; - - public function __construct(string $region, UtopiaDatabase $database, callable $errorHandler = null) - { - parent::__construct($region); - $this->database = $database; - $this->errorHandler = $errorHandler; - } - - /** - * Create Per Period Metric - * - * Create given metric for each defined period - * - * @param string $projectId - * @param string $metric - * @param int $value - * @param bool $monthly - * @return void - * @throws Authorization - * @throws Structure - */ - protected function createPerPeriodMetric(string $projectId, string $metric, int $value, bool $monthly = false): void - { - foreach ($this->periods as $options) { - $period = $options['key']; - $date = new \DateTime(); - if ($period === '30m') { - $minutes = $date->format('i') >= '30' ? "30" : "00"; - $time = $date->format('Y-m-d H:' . $minutes . ':00'); - } elseif ($period === '1d') { - $time = $date->format('Y-m-d 00:00:00'); - } else { - throw new Exception("Period type not found", 500); - } - $this->createOrUpdateMetric($projectId, $metric, $period, $time, $value); - } - - // Required for billing - if ($monthly) { - $time = DateTime::createFromFormat('Y-m-d\TH:i:s.v', \date('Y-m-01\T00:00:00.000'))->format(DateTime::RFC3339); - $this->createOrUpdateMetric($projectId, $metric, '1mo', $time, $value); - } - } - - /** - * Create or Update Metric - * - * Create or update each metric in the stats collection for the given project - * - * @param string $projectId - * @param string $metric - * @param string $period - * @param string $time - * @param int $value - * - * @return void - * @throws Authorization - * @throws Structure - */ - protected function createOrUpdateMetric(string $projectId, string $metric, string $period, string $time, int $value): void - { - $id = \md5("{$time}_{$period}_{$metric}"); - $this->database->setNamespace('_' . $projectId); - - try { - $document = $this->database->getDocument('stats', $id); - if ($document->isEmpty()) { - $this->database->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $metric, - 'value' => $value, - 'region' => $this->region, - 'type' => 2, // these are cumulative metrics - ])); - } else { - $this->database->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $value) - ); - } - } catch (\Exception$e) { // if projects are deleted this might fail - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); - } else { - throw $e; - } - } - } - - /** - * Foreach Document - * - * Call provided callback for each document in the collection - * - * @param string $projectId - * @param string $collection - * @param array $queries - * @param callable $callback - * - * @return void - * @throws Exception - */ - protected function foreachDocument(string $projectId, string $collection, array $queries, callable $callback): void - { - $limit = 50; - $results = []; - $sum = $limit; - $latestDocument = null; - - while ($sum === $limit) { - try { - $paginationQueries = [Query::limit($limit)]; - if ($latestDocument !== null) { - $paginationQueries[] = Query::cursorAfter($latestDocument); - } - - $this->database->setNamespace('_' . $projectId); - $results = $this->database->find($collection, \array_merge($paginationQueries, $queries)); - } catch (\Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "fetch_documents_project_{$projectId}_collection_{$collection}"); - return; - } else { - throw $e; - } - } - if (empty($results)) { - return; - } - - $sum = count($results); - - foreach ($results as $document) { - if (is_callable($callback)) { - $callback($document); - } - } - $latestDocument = $results[array_key_last($results)]; - } - } - - /** - * Sum - * - * Calculate sum of an attribute of documents in collection - * - * @param string $projectId - * @param string $collection - * @param string $attribute - * @param string|null $metric - * @param int $multiplier - * @return int - * @throws Exception - */ - private function sum(string $projectId, string $collection, string $attribute, string $metric = null, int $multiplier = 1): int - { - $this->database->setNamespace('_' . $projectId); - - try { - $sum = $this->database->sum($collection, $attribute); - $sum = (int) ($sum * $multiplier); - - if (!is_null($metric)) { - $this->createPerPeriodMetric($projectId, $metric, $sum); - } - return $sum; - } catch (Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "fetch_sum_project_{$projectId}_collection_{$collection}"); - } else { - throw $e; - } - } - return 0; - } - - /** - * Count - * - * Count number of documents in collection - * - * @param string $projectId - * @param string $collection - * @param ?string $metric - * - * @return int - * @throws Exception - */ - private function count(string $projectId, string $collection, ?string $metric = null): int - { - $this->database->setNamespace('_' . $projectId); - - try { - $count = $this->database->count($collection); - if (!is_null($metric)) { - $this->createPerPeriodMetric($projectId, (string) $metric, $count); - } - return $count; - } catch (Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "fetch_count_project_{$projectId}_collection_{$collection}"); - } else { - throw $e; - } - } - return 0; - } - - /** - * Deployments Total - * - * Total sum of storage used by deployments - * - * @param string $projectId - * - * @return int - * @throws Exception - */ - private function deploymentsTotal(string $projectId): int - { - return $this->sum($projectId, 'deployments', 'size', 'deployments.$all.storage.size'); - } - - /** - * Users Stats - * - * Metric: users.count - * - * @param string $projectId - * - * @return void - * @throws Exception - */ - private function usersStats(string $projectId): void - { - $this->count($projectId, 'users', 'users.$all.count.total'); - } - - /** - * Storage Stats - * - * Metrics: buckets.$all.count.total, files.$all.count.total, files.bucketId,count.total, - * files.$all.storage.size, files.bucketId.storage.size, project.$all.storage.size - * - * @param string $projectId - * - * @return void - * @throws Authorization - * @throws Structure - */ - private function storageStats(string $projectId): void - { - $projectFilesTotal = 0; - $projectFilesCount = 0; - - $metric = 'buckets.$all.count.total'; - $this->count($projectId, 'buckets', $metric); - - $this->foreachDocument($projectId, 'buckets', [], function ($bucket) use (&$projectFilesCount, &$projectFilesTotal, $projectId,) { - $metric = "files.{$bucket->getId()}.count.total"; - $count = $this->count($projectId, 'bucket_' . $bucket->getInternalId(), $metric); - $projectFilesCount += $count; - - $metric = "files.{$bucket->getId()}.storage.size"; - $sum = $this->sum($projectId, 'bucket_' . $bucket->getInternalId(), 'sizeOriginal', $metric); - $projectFilesTotal += $sum; - }); - - $this->createPerPeriodMetric($projectId, 'files.$all.count.total', $projectFilesCount); - $this->createPerPeriodMetric($projectId, 'files.$all.storage.size', $projectFilesTotal); - - $deploymentsTotal = $this->deploymentsTotal($projectId); - $this->createPerPeriodMetric($projectId, 'project.$all.storage.size', $projectFilesTotal + $deploymentsTotal); - } - - /** - * Database Stats - * - * Collect all database stats - * Metrics: databases.$all.count.total, collections.$all.count.total, collections.databaseId.count.total, - * documents.$all.count.all, documents.databaseId.count.total, documents.databaseId/collectionId.count.total - * - * @param string $projectId - * - * @return void - * @throws Authorization - * @throws Structure - */ - private function databaseStats(string $projectId): void - { - $projectDocumentsCount = 0; - $projectCollectionsCount = 0; - - $this->count($projectId, 'databases', 'databases.$all.count.total'); - - $this->foreachDocument($projectId, 'databases', [], function ($database) use (&$projectDocumentsCount, &$projectCollectionsCount, $projectId) { - $metric = "collections.{$database->getId()}.count.total"; - $count = $this->count($projectId, 'database_' . $database->getInternalId(), $metric); - $projectCollectionsCount += $count; - $databaseDocumentsCount = 0; - - $this->foreachDocument($projectId, 'database_' . $database->getInternalId(), [], function ($collection) use (&$projectDocumentsCount, &$databaseDocumentsCount, $projectId, $database) { - $metric = "documents.{$database->getId()}/{$collection->getId()}.count.total"; - - $count = $this->count($projectId, 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $metric); - $projectDocumentsCount += $count; - $databaseDocumentsCount += $count; - }); - - $this->createPerPeriodMetric($projectId, "documents.{$database->getId()}.count.total", $databaseDocumentsCount); - }); - - $this->createPerPeriodMetric($projectId, 'collections.$all.count.total', $projectCollectionsCount); - $this->createPerPeriodMetric($projectId, 'documents.$all.count.total', $projectDocumentsCount); - } - - /** - * Collect Stats - * - * Collect all database related stats - * - * @return void - * @throws Exception - */ - public function collect(): void - { - $this->foreachDocument('console', 'projects', [], function (Document $project) { - $projectId = $project->getInternalId(); - - $this->usersStats($projectId); - $this->databaseStats($projectId); - $this->storageStats($projectId); - }); - } -} diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 2ceb39a596..7b9497e66c 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -11,12 +11,54 @@ use DateTime; class TimeSeries extends Calculator { + /** + * InfluxDB + * + * @var InfluxDatabase + */ protected InfluxDatabase $influxDB; + + /** + * Utopia Database + * + * @var Database + */ protected Database $database; + + /** + * Error Handler Callback + * + * @var callable + */ protected $errorHandler; + + /** + * Latest times for metric that was synced to the database + * + * @var array + */ private array $latestTime = []; - // all the mertics that we are collecting + /** + * Periods the metrics are collected for + * @var array + */ + protected array $periods = [ + [ + 'key' => '1h', + 'startTime' => '-24 hours' + ], + [ + 'key' => '1d', + 'startTime' => '-30 days' + ] + ]; + + /** + * All the metrics that we are collecting + * + * @var array + */ protected array $metrics = [ 'project.$all.network.requests' => [ 'table' => 'appwrite_usage_project_{scope}_network_requests', @@ -190,12 +232,6 @@ class TimeSeries extends Calculator 'executions.$all.compute.total' => [ 'table' => 'appwrite_usage_executions_{scope}_compute', ], - 'builds.$all.compute.time' => [ - 'table' => 'appwrite_usage_executions_{scope}_compute_time', - ], - 'executions.$all.compute.time' => [ - 'table' => 'appwrite_usage_executions_{scope}_compute_time', - ], 'builds.$all.compute.total' => [ 'table' => 'appwrite_usage_builds_{scope}_compute', ], @@ -231,14 +267,7 @@ class TimeSeries extends Calculator 'table' => 'appwrite_usage_builds_{scope}_compute', 'groupBy' => ['functionId'], ], - 'executions.functionId.compute.time' => [ - 'table' => 'appwrite_usage_executions_{scope}_compute_time', - 'groupBy' => ['functionId'], - ], - 'builds.functionId.compute.time' => [ - 'table' => 'appwrite_usage_builds_{scope}_compute_time', - 'groupBy' => ['functionId'], - ], + 'executions.functionId.compute.failure' => [ 'table' => 'appwrite_usage_executions_{scope}_compute', 'groupBy' => ['functionId'], @@ -268,15 +297,89 @@ class TimeSeries extends Calculator ], ], + // counters + 'users.$all.count.total' => [ + 'table' => 'appwrite_usage_users_{scope}_count_total', + ], + 'buckets.$all.count.total' => [ + 'table' => 'appwrite_usage_buckets_{scope}_count_total', + ], + 'files.$all.count.total' => [ + 'table' => 'appwrite_usage_files_{scope}_count_total', + ], + 'files.bucketId.count.total' => [ + 'table' => 'appwrite_usage_files_{scope}_count_total', + 'groupBy' => ['bucketId'] + ], + 'databases.$all.count.total' => [ + 'table' => 'appwrite_usage_databases_{scope}_count_total', + ], + 'collections.$all.count.total' => [ + 'table' => 'appwrite_usage_collections_{scope}_count_total', + ], + 'documents.$all.count.total' => [ + 'table' => 'appwrite_usage_documents_{scope}_count_total', + ], + 'collections.databaseId.count.total' => [ + 'table' => 'appwrite_usage_collections_{scope}_count_total', + 'groupBy' => ['databaseId'] + ], + 'documents.databaseId.count.total' => [ + 'table' => 'appwrite_usage_documents_{scope}_count_total', + 'groupBy' => ['databaseId'] + ], + 'documents.databaseId/collectionId.count.total' => [ + 'table' => 'appwrite_usage_documents_{scope}_count_total', + 'groupBy' => ['databaseId', 'collectionId'] + ], + 'deployments.$all.storage.size' => [ + 'table' => 'appwrite_usage_deployments_{scope}_storage_size', + ], + 'project.$all.storage.size' => [ + 'table' => 'appwrite_usage_project_{scope}_storage_size', + ], + 'files.$all.storage.size' => [ + 'table' => 'appwrite_usage_files_{scope}_storage_size', + ], + 'files.$bucketId.storage.size' => [ + 'table' => 'appwrite_usage_files_{scope}_storage_size', + 'groupBy' => ['bucketId'] + ], + + 'builds.$all.compute.time' => [ + 'table' => 'appwrite_usage_executions_{scope}_compute_time', + ], + 'executions.$all.compute.time' => [ + 'table' => 'appwrite_usage_executions_{scope}_compute_time', + ], + + 'executions.functionId.compute.time' => [ + 'table' => 'appwrite_usage_executions_{scope}_compute_time', + 'groupBy' => ['functionId'], + ], + 'builds.functionId.compute.time' => [ + 'table' => 'appwrite_usage_builds_{scope}_compute_time', + 'groupBy' => ['functionId'], + ], + 'project.$all.compute.time' => [ // Built time + execution time 'table' => 'appwrite_usage_project_{scope}_compute_time', 'groupBy' => ['functionId'], ], - ]; - protected array $period = [ - 'key' => '30m', - 'startTime' => '-24 hours', + 'deployments.$all.storage.size' => [ + 'table' => 'appwrite_usage_deployments_{scope}_storage_size' + ], + 'project.$all.storage.size' => [ + 'table' => 'appwrite_usage_project_{scope}_storage_size' + ], + 'files.$all.storage.size' => [ + 'table' => 'appwrite_usage_files_{scope}_storage_size' + ], + 'files.bucketId.storage.size' => [ + 'table' => 'appwrite_usage_files_{scope}_storage_size', + 'groupBy' => ['bucketId'] + ] ]; public function __construct(string $region, Database $database, InfluxDatabase $influxDB, callable $errorHandler = null) @@ -303,9 +406,7 @@ class TimeSeries extends Calculator private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void { $id = \md5("{$time}_{$period}_{$metric}"); - $this->database->setNamespace('_console'); - $project = $this->database->getDocument('projects', $projectId); - $this->database->setNamespace('_' . $project->getInternalId()); + $this->database->setNamespace('_' . $projectId); try { $document = $this->database->getDocument('stats', $id); @@ -368,7 +469,7 @@ class TimeSeries extends Calculator $query .= "WHERE \"time\" > '{$start}' "; $query .= "AND \"time\" < '{$end}' "; $query .= "AND \"metric_type\"='counter' {$filters} "; - $query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} "; + $query .= "GROUP BY time({$period['key']}), \"projectId\", \"projectInternalId\" {$groupBy} "; $query .= "FILL(null)"; try { @@ -390,9 +491,11 @@ class TimeSeries extends Calculator } $value = (!empty($point['value'])) ? $point['value'] : 0; - + if (empty($point['projectInternalId'] ?? null)) { + return; + } $this->createOrUpdateMetric( - $projectId, + $point['projectInternalId'], $point['time'], $period['key'], $metricUpdated, @@ -419,14 +522,16 @@ class TimeSeries extends Calculator */ public function collect(): void { - foreach ($this->metrics as $metric => $options) { //for each metrics - try { - $this->syncFromInfluxDB($metric, $options, $this->period); - } catch (\Exception $e) { - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e); - } else { - throw $e; + foreach ($this->periods as $period) { + foreach ($this->metrics as $metric => $options) { //for each metrics + try { + $this->syncFromInfluxDB($metric, $options, $period); + } catch (\Exception $e) { + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e); + } else { + throw $e; + } } } } diff --git a/src/Appwrite/Usage/Stats.php b/src/Appwrite/Usage/Stats.php index 00298238f6..e6e0056664 100644 --- a/src/Appwrite/Usage/Stats.php +++ b/src/Appwrite/Usage/Stats.php @@ -76,11 +76,14 @@ class Stats /** * Submit data to StatsD. + * Send various metrics to StatsD based on the parameters that are set + * @return void */ public function submit(): void { $projectId = $this->params['projectId'] ?? ''; - $tags = ",projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); + $projectInternalId = $this->params['projectInternalId']; + $tags = ",projectInternalId={$projectInternalId},projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); // the global namespace is prepended to every key (optional) $this->statsd->setNamespace($this->namespace); @@ -91,8 +94,8 @@ class Stats $this->statsd->increment('project.{scope}.network.requests' . $tags . ',method=' . \strtolower($httpMethod)); } - $inbound = $this->params['networkRequestSize'] ?? 0; - $outbound = $this->params['networkResponseSize'] ?? 0; + $inbound = $this->params['project.{scope}.network.inbound'] ?? 0; + $outbound = $this->params['project.{scope}.network.outbound'] ?? 0; $this->statsd->count('project.{scope}.network.inbound' . $tags, $inbound); $this->statsd->count('project.{scope}.network.outbound' . $tags, $outbound); $this->statsd->count('project.{scope}.network.bandwidth' . $tags, $inbound + $outbound); @@ -102,12 +105,13 @@ class Stats 'users.{scope}.requests.read', 'users.{scope}.requests.update', 'users.{scope}.requests.delete', + 'users.{scope}.count.total', ]; foreach ($usersMetrics as $metric) { $value = $this->params[$metric] ?? 0; - if ($value >= 1) { - $this->statsd->increment($metric . $tags); + if ($value === 1 || $value === -1) { + $this->statsd->count($metric . $tags, $value); } } @@ -124,13 +128,16 @@ class Stats 'documents.{scope}.requests.read', 'documents.{scope}.requests.update', 'documents.{scope}.requests.delete', + 'databases.{scope}.count.total', + 'collections.{scope}.count.total', + 'documents.{scope}.count.total' ]; foreach ($dbMetrics as $metric) { $value = $this->params[$metric] ?? 0; - if ($value >= 1) { + if ($value === 1 || $value === -1) { $dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? ''); - $this->statsd->increment($metric . $dbTags); + $this->statsd->count($metric . $dbTags, $value); } } @@ -143,13 +150,16 @@ class Stats 'files.{scope}.requests.read', 'files.{scope}.requests.update', 'files.{scope}.requests.delete', + 'buckets.{scope}.count.total', + 'files.{scope}.count.total', + 'files.{scope}.storage.size' ]; foreach ($storageMertics as $metric) { $value = $this->params[$metric] ?? 0; - if ($value >= 1) { + if ($value !== 0) { $storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? ''); - $this->statsd->increment($metric . $storageTags); + $this->statsd->count($metric . $storageTags, $value); } } @@ -176,19 +186,30 @@ class Stats $functionBuildTime = ($this->params['buildTime'] ?? 0) * 1000; // ms $functionBuildStatus = $this->params['buildStatus'] ?? ''; $functionCompute = $functionExecutionTime + $functionBuildTime; + $functionTags = $tags . ',functionId=' . $functionId; + + $deploymentSize = $this->params['deployment.{scope}.storage.size'] ?? 0; + $storageSize = $this->params['files.{scope}.storage.size'] ?? 0; + if ($deploymentSize + $storageSize > 0 || $deploymentSize + $storageSize <= -1) { + $this->statsd->count('project.{scope}.storage.size' . $tags, $deploymentSize + $storageSize); + } + + if ($deploymentSize !== 0) { + $this->statsd->count('deployments.{scope}.storage.size' . $functionTags, $deploymentSize); + } if ($functionExecution >= 1) { - $this->statsd->increment('executions.{scope}.compute' . $tags . ',functionId=' . $functionId . ',functionStatus=' . $functionExecutionStatus); + $this->statsd->increment('executions.{scope}.compute' . $functionTags . ',functionStatus=' . $functionExecutionStatus); if ($functionExecutionTime > 0) { - $this->statsd->count('executions.{scope}.compute.time' . $tags . ',functionId=' . $functionId, $functionExecutionTime); + $this->statsd->count('executions.{scope}.compute.time' . $functionTags, $functionExecutionTime); } } if ($functionBuild >= 1) { - $this->statsd->increment('builds.{scope}.compute' . $tags . ',functionId=' . $functionId . ',functionBuildStatus=' . $functionBuildStatus); - $this->statsd->count('builds.{scope}.compute.time' . $tags . ',functionId=' . $functionId, $functionBuildTime); + $this->statsd->increment('builds.{scope}.compute' . $functionTags . ',functionBuildStatus=' . $functionBuildStatus); + $this->statsd->count('builds.{scope}.compute.time' . $functionTags, $functionBuildTime); } if ($functionBuild + $functionExecution >= 1) { - $this->statsd->count('project.{scope}.compute.time' . $tags . ',functionId=' . $functionId, $functionCompute); + $this->statsd->count('project.{scope}.compute.time' . $functionTags, $functionCompute); } $this->reset(); diff --git a/src/Appwrite/Utopia/Response/Model/Provider.php b/src/Appwrite/Utopia/Response/Model/Provider.php index 41fe58868d..0f14993508 100644 --- a/src/Appwrite/Utopia/Response/Model/Provider.php +++ b/src/Appwrite/Utopia/Response/Model/Provider.php @@ -7,6 +7,9 @@ use Appwrite\Utopia\Response\Model; class Provider extends Model { + /** + * @var bool + */ protected bool $public = false; public function __construct() diff --git a/tests/e2e/General/HTTPTest.php b/tests/e2e/General/HTTPTest.php index 14e6ada761..0cb7625ba4 100644 --- a/tests/e2e/General/HTTPTest.php +++ b/tests/e2e/General/HTTPTest.php @@ -163,7 +163,8 @@ class HTTPTest extends Scope $response['body'] = json_decode($response['body'], true); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEmpty($response['body']['schemaValidationMessages']); + // looks like recent change in the validator + $this->assertTrue(empty($response['body']['schemaValidationMessages'])); } } diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 6f17d940ae..572bc4abf8 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -85,7 +85,7 @@ class UsageTest extends Scope #[Retry(count: 1)] public function testUsersStats(array $data): array { - sleep(35); + sleep(10); $projectId = $data['projectId']; $headers = $data['headers']; @@ -114,6 +114,7 @@ class UsageTest extends Scope 'x-appwrite-project' => $projectId, 'x-appwrite-mode' => 'admin' ])); + $requestsCount++; $res = $res['body']; $this->assertEquals(10, $res['usersCreate'][array_key_last($res['usersCreate'])]['value']); $this->validateDates($res['usersCreate']); @@ -255,7 +256,7 @@ class UsageTest extends Scope $filesCreate = $data['filesCreate']; $filesDelete = $data['filesDelete']; - sleep(35); + sleep(10); // console request $headers = [ @@ -279,6 +280,7 @@ class UsageTest extends Scope 'x-appwrite-project' => $projectId, 'x-appwrite-mode' => 'admin' ])); + $requestsCount++; $res = $res['body']; $this->assertEquals($storageTotal, $res['storage'][array_key_last($res['storage'])]['value']); $this->validateDates($res['storage']); @@ -303,6 +305,7 @@ class UsageTest extends Scope 'x-appwrite-project' => $projectId, 'x-appwrite-mode' => 'admin' ])); + $requestsCount++; $res = $res['body']; $this->assertEquals($storageTotal, $res['filesStorage'][array_key_last($res['filesStorage'])]['value']); $this->assertEquals($filesCount, $res['filesCount'][array_key_last($res['filesCount'])]['value']); @@ -493,7 +496,7 @@ class UsageTest extends Scope $documentsRead = $data['documentsRead']; $documentsDelete = $data['documentsDelete']; - sleep(35); + sleep(10); // check datbase stats $headers = [ @@ -701,7 +704,7 @@ class UsageTest extends Scope $executions = $data['executions']; $failures = $data['failures']; - sleep(25); + sleep(10); $response = $this->client->call(Client::METHOD_GET, '/functions/' . $functionId . '/usage', $headers, [ 'range' => '30d' diff --git a/tests/unit/Usage/StatsTest.php b/tests/unit/Usage/StatsTest.php index 0b39dfdaa3..f021841396 100644 --- a/tests/unit/Usage/StatsTest.php +++ b/tests/unit/Usage/StatsTest.php @@ -38,10 +38,12 @@ class StatsTest extends TestCase { $this->object ->setParam('projectId', 'appwrite_test') + ->setParam('projectInternalId', 1) ->setParam('networkRequestSize', 100) ; $this->assertEquals('appwrite_test', $this->object->getParam('projectId')); + $this->assertEquals(1, $this->object->getParam('projectInternalId')); $this->assertEquals(100, $this->object->getParam('networkRequestSize')); $this->object->submit();