diff --git a/.env b/.env index feeb74d849..9fbc425977 100644 --- a/.env +++ b/.env @@ -50,10 +50,6 @@ _APP_STORAGE_WASABI_BUCKET= _APP_STORAGE_ANTIVIRUS=disabled _APP_STORAGE_ANTIVIRUS_HOST=clamav _APP_STORAGE_ANTIVIRUS_PORT=3310 -_APP_INFLUXDB_HOST=influxdb -_APP_INFLUXDB_PORT=8086 -_APP_STATSD_HOST=telegraf -_APP_STATSD_PORT=8125 _APP_SMTP_HOST=maildev _APP_SMTP_PORT=1025 _APP_SMTP_SECURE= diff --git a/.gitmodules b/.gitmodules index 754fa9f80e..effe15f68a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "app/console"] path = app/console url = https://github.com/appwrite/console - branch = 3.2.3 + branch = feat-usage-sh-1.4 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ccd61e742b..0522e33da5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -210,7 +210,6 @@ Appwrite's current structure is a combination of both [Monolithic](https://en.wi │ ├── Task │ ├── Template │ ├── URL -│ ├── Usage │ └── Utopia └── tests # End to end & unit tests ├── e2e diff --git a/Dockerfile b/Dockerfile index 8cea481543..eb5d41c8f6 100755 --- a/Dockerfile +++ b/Dockerfile @@ -74,7 +74,6 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/maintenance && \ - chmod +x /usr/local/bin/usage && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/upgrade && \ chmod +x /usr/local/bin/migrate && \ @@ -94,6 +93,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-mails && \ chmod +x /usr/local/bin/worker-messaging && \ chmod +x /usr/local/bin/worker-webhooks && \ + chmod +x /usr/local/bin/worker-usage && \ chmod +x /usr/local/bin/worker-migrations # Cloud Executabless diff --git a/app/cli.php b/app/cli.php index 5b5ae47bf6..992c85eca2 100644 --- a/app/cli.php +++ b/app/cli.php @@ -117,30 +117,6 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, }; }, ['pools', 'dbForConsole', 'cache']); -CLI::setResource('influxdb', function (Registry $register) { - $client = $register->get('influxdb'); /** @var InfluxDB\Client $client */ - $attempts = 0; - $max = 10; - $sleep = 1; - - do { // check if telegraf database is ready - try { - $attempts++; - $database = $client->selectDB('telegraf'); - if (in_array('telegraf', $client->listDatabases())) { - break; // leave the do-while if successful - } - } catch (\Throwable $th) { - Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('InfluxDB database not ready yet'); - } - sleep($sleep); - } - } while ($attempts < $max); - return $database; -}, ['register']); - CLI::setResource('queue', function (Group $pools) { return $pools->get('queue')->pop()->getResource(); }, ['pools']); diff --git a/app/config/collections.php b/app/config/collections.php index db229ce87a..8c2cb77191 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -1302,7 +1302,7 @@ $commonCollections = [ 'type' => Database::VAR_INTEGER, 'format' => '', 'size' => 8, - 'signed' => false, + 'signed' => true, 'required' => true, 'default' => null, 'array' => false, @@ -1330,17 +1330,6 @@ $commonCollections = [ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('type'), - 'type' => Database::VAR_INTEGER, - 'format' => '', - 'size' => 1, - 'signed' => false, - 'required' => true, - 'default' => 0, // 0 -> count, 1 -> sum - 'array' => false, - 'filters' => [], - ], ], 'indexes' => [ [ @@ -1359,7 +1348,7 @@ $commonCollections = [ ], [ '$id' => ID::custom('_key_metric_period_time'), - 'type' => Database::INDEX_KEY, + 'type' => Database::INDEX_UNIQUE, 'attributes' => ['metric', 'period', 'time'], 'lengths' => [], 'orders' => [Database::ORDER_DESC], diff --git a/app/config/variables.php b/app/config/variables.php index 9d555bf013..793fa5c74c 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -335,54 +335,6 @@ return [ ], ], ], - [ - 'category' => 'InfluxDB', - 'description' => 'Appwrite uses an InfluxDB server for managing time-series data and server stats. The InfluxDB env vars are used to allow Appwrite server to connect to the InfluxDB container.', - 'variables' => [ - [ - 'name' => '_APP_INFLUXDB_HOST', - 'description' => 'InfluxDB server host name address. Default value is: \'influxdb\'.', - 'introduction' => '', - 'default' => 'influxdb', - 'required' => false, - 'question' => '', - 'filter' => '' - ], - [ - 'name' => '_APP_INFLUXDB_PORT', - 'description' => 'InfluxDB server TCP port. Default value is: \'8086\'.', - 'introduction' => '', - 'default' => '8086', - 'required' => false, - 'question' => '', - 'filter' => '' - ], - ], - ], - [ - 'category' => 'StatsD', - 'description' => 'Appwrite uses a StatsD server for aggregating and sending stats data over a fast UDP connection. The StatsD env vars are used to allow Appwrite server to connect to the StatsD container.', - 'variables' => [ - [ - 'name' => '_APP_STATSD_HOST', - 'description' => 'StatsD server host name address. Default value is: \'telegraf\'.', - 'introduction' => '', - 'default' => 'telegraf', - 'required' => false, - 'question' => '', - 'filter' => '' - ], - [ - 'name' => '_APP_STATSD_PORT', - 'description' => 'StatsD server TCP port. Default value is: \'8125\'.', - 'introduction' => '', - 'default' => '8125', - 'required' => false, - 'question' => '', - 'filter' => '' - ], - ], - ], [ 'category' => 'SMTP', 'description' => "Appwrite is using an SMTP server for emailing your projects users and server admins. The SMTP env vars are used to allow Appwrite server to connect to the SMTP container.\n\nIf running in production, it might be easier to use a 3rd party SMTP server as it might be a little more difficult to set up a production SMTP server that will not send all your emails into your user\'s SPAM folder.", diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 2bc7759620..1949a60d22 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -174,8 +174,6 @@ App::post('/v1/account/sessions/email') ->label('audits.event', 'session.create') ->label('audits.resource', 'user/{response.userId}') ->label('audits.userId', '{response.userId}') - ->label('usage.metric', 'sessions.{scope}.requests.create') - ->label('usage.params', ['provider:email']) ->label('sdk.auth', []) ->label('sdk.namespace', 'account') ->label('sdk.method', 'createEmailSession') @@ -428,8 +426,6 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') ->label('abuse-limit', 50) ->label('abuse-key', 'ip:{ip}') ->label('docs', false) - ->label('usage.metric', 'sessions.{scope}.requests.create') - ->label('usage.params', ['provider:{request.provider}']) ->param('provider', '', new WhiteList(\array_keys(Config::getParam('providers')), true), 'OAuth2 provider.') ->param('code', '', new Text(2048, 0), 'OAuth2 code. This is a temporary code that the will be later exchanged for an access token.', true) ->param('state', '', new Text(2048), 'OAuth2 state params.', true) @@ -794,7 +790,6 @@ App::get('/v1/account/identities') ->desc('List Identities') ->groups(['api', 'account']) ->label('scope', 'account') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'listIdentities') @@ -849,7 +844,6 @@ App::delete('/v1/account/identities/:identityId') ->label('audits.event', 'identity.delete') ->label('audits.resource', 'identity/{request.$identityId}') ->label('audits.userId', '{user.$id}') - ->label('usage.metric', 'identities.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'deleteIdentity') @@ -1098,8 +1092,6 @@ App::put('/v1/account/sessions/magic-url') ->label('audits.event', 'session.update') ->label('audits.resource', 'user/{response.userId}') ->label('audits.userId', '{response.userId}') - ->label('usage.metric', 'sessions.{scope}.requests.create') - ->label('usage.params', ['provider:magic-url']) ->label('sdk.auth', []) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updateMagicURLSession') @@ -1119,7 +1111,7 @@ App::put('/v1/account/sessions/magic-url') ->inject('locale') ->inject('geodb') ->inject('queueForEvents') - ->action(function (string $userId, string $secret, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents) { + ->action(function (string $userId, string $secret, Request $request, Response $response,Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents) { /** @var Utopia\Database\Document $user */ @@ -1473,8 +1465,6 @@ App::post('/v1/account/sessions/anonymous') ->label('audits.event', 'session.create') ->label('audits.resource', 'user/{response.userId}') ->label('audits.userId', '{response.userId}') - ->label('usage.metric', 'sessions.{scope}.requests.create') - ->label('usage.params', ['provider:anonymous']) ->label('sdk.auth', []) ->label('sdk.namespace', 'account') ->label('sdk.method', 'createAnonymousSession') @@ -1673,7 +1663,6 @@ App::get('/v1/account/prefs') ->desc('Get account preferences') ->groups(['api', 'account']) ->label('scope', 'account') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'getPrefs') @@ -1735,7 +1724,6 @@ App::get('/v1/account/logs') ->desc('List logs') ->groups(['api', 'account']) ->label('scope', 'account') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'listLogs') @@ -1796,7 +1784,6 @@ App::get('/v1/account/sessions/:sessionId') ->desc('Get session') ->groups(['api', 'account']) ->label('scope', 'account') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'getSession') @@ -1844,7 +1831,6 @@ App::patch('/v1/account/name') ->label('scope', 'account') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updateName') @@ -1879,7 +1865,6 @@ App::patch('/v1/account/password') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updatePassword') @@ -1945,7 +1930,6 @@ App::patch('/v1/account/email') ->label('scope', 'account') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updateEmail') @@ -2015,7 +1999,6 @@ App::patch('/v1/account/phone') ->label('scope', 'account') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updatePhone') @@ -2074,7 +2057,6 @@ App::patch('/v1/account/prefs') ->label('scope', 'account') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updatePrefs') @@ -2108,7 +2090,6 @@ App::patch('/v1/account/status') ->label('scope', 'account') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updateStatus') @@ -2152,7 +2133,6 @@ App::delete('/v1/account/sessions/:sessionId') ->label('event', 'users.[userId].sessions.[sessionId].delete') ->label('audits.event', 'session.delete') ->label('audits.resource', 'user/{user.$id}') - ->label('usage.metric', 'sessions.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'deleteSession') @@ -2229,7 +2209,6 @@ App::patch('/v1/account/sessions/:sessionId') ->label('audits.event', 'session.update') ->label('audits.resource', 'user/{response.userId}') ->label('audits.userId', '{response.userId}') - ->label('usage.metric', 'sessions.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updateSession') @@ -2314,7 +2293,6 @@ App::delete('/v1/account/sessions') ->label('event', 'users.[userId].sessions.[sessionId].delete') ->label('audits.event', 'session.delete') ->label('audits.resource', 'user/{user.$id}') - ->label('usage.metric', 'sessions.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'deleteSessions') @@ -2376,7 +2354,6 @@ App::post('/v1/account/recovery') ->label('audits.event', 'recovery.create') ->label('audits.resource', 'user/{response.userId}') ->label('audits.userId', '{response.userId}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'createRecovery') @@ -2559,7 +2536,6 @@ App::put('/v1/account/recovery') ->label('audits.event', 'recovery.update') ->label('audits.resource', 'user/{response.userId}') ->label('audits.userId', '{response.userId}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updateRecovery') @@ -2646,7 +2622,6 @@ App::post('/v1/account/verification') ->label('event', 'users.[userId].verification.[tokenId].create') ->label('audits.event', 'verification.create') ->label('audits.resource', 'user/{response.userId}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'createVerification') @@ -2807,7 +2782,6 @@ App::put('/v1/account/verification') ->label('event', 'users.[userId].verification.[tokenId].update') ->label('audits.event', 'verification.update') ->label('audits.resource', 'user/{response.userId}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updateVerification') @@ -2868,7 +2842,6 @@ App::post('/v1/account/verification/phone') ->label('event', 'users.[userId].verification.[tokenId].create') ->label('audits.event', 'verification.create') ->label('audits.resource', 'user/{response.userId}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'createPhoneVerification') @@ -2964,7 +2937,6 @@ App::put('/v1/account/verification/phone') ->label('event', 'users.[userId].verification.[tokenId].update') ->label('audits.event', 'verification.update') ->label('audits.resource', 'user/{response.userId}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'account') ->label('sdk.method', 'updatePhoneVerification') diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index aaf30e00c3..c152250ecb 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -402,7 +402,6 @@ App::post('/v1/databases') ->label('scope', 'databases.write') ->label('audits.event', 'database.create') ->label('audits.resource', 'database/{response.$id}') - ->label('usage.metric', 'databases.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'create') @@ -476,7 +475,6 @@ App::get('/v1/databases') ->desc('List databases') ->groups(['api', 'database']) ->label('scope', 'databases.read') - ->label('usage.metric', 'databases.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'list') @@ -524,7 +522,6 @@ App::get('/v1/databases/:databaseId') ->desc('Get database') ->groups(['api', 'database']) ->label('scope', 'databases.read') - ->label('usage.metric', 'databases.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'get') @@ -639,7 +636,6 @@ App::put('/v1/databases/:databaseId') ->label('event', 'databases.[databaseId].update') ->label('audits.event', 'database.update') ->label('audits.resource', 'database/{response.$id}') - ->label('usage.metric', 'databases.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'update') @@ -684,7 +680,6 @@ App::delete('/v1/databases/:databaseId') ->label('event', 'databases.[databaseId].delete') ->label('audits.event', 'database.delete') ->label('audits.resource', 'database/{request.databaseId}') - ->label('usage.metric', 'databases.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'delete') @@ -729,8 +724,6 @@ App::post('/v1/databases/:databaseId/collections') ->label('scope', 'collections.write') ->label('audits.event', 'collection.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{response.$id}') - ->label('usage.metric', 'collections.{scope}.requests.create') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'createCollection') @@ -796,8 +789,6 @@ App::get('/v1/databases/:databaseId/collections') ->desc('List collections') ->groups(['api', 'database']) ->label('scope', 'collections.read') - ->label('usage.metric', 'collections.{scope}.requests.read') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'listCollections') @@ -855,8 +846,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId') ->desc('Get collection') ->groups(['api', 'database']) ->label('scope', 'collections.read') - ->label('usage.metric', 'collections.{scope}.requests.read') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'getCollection') @@ -891,8 +880,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs') ->desc('List collection logs') ->groups(['api', 'database']) ->label('scope', 'collections.read') - ->label('usage.metric', 'collections.{scope}.requests.read') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'listCollectionLogs') @@ -990,8 +977,6 @@ App::put('/v1/databases/:databaseId/collections/:collectionId') ->label('event', 'databases.[databaseId].collections.[collectionId].update') ->label('audits.event', 'collection.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'updateCollection') @@ -1060,8 +1045,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId') ->label('event', 'databases.[databaseId].collections.[collectionId].delete') ->label('audits.event', 'collection.delete') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.delete') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'deleteCollection') @@ -1117,8 +1100,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/string ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'createStringAttribute') @@ -1175,8 +1156,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/email' ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createEmailAttribute') @@ -1219,8 +1198,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum') ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createEnumAttribute') @@ -1278,8 +1255,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/ip') ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createIpAttribute') @@ -1322,8 +1297,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/url') ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createUrlAttribute') @@ -1366,8 +1339,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/intege ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createIntegerAttribute') @@ -1439,8 +1410,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/float' ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createFloatAttribute') @@ -1515,8 +1484,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/boolea ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createBooleanAttribute') @@ -1558,8 +1525,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/dateti ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createDatetimeAttribute') @@ -1604,8 +1569,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/relati ->label('scope', 'collections.write') ->label('audits.event', 'attribute.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.namespace', 'databases') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.method', 'createRelationshipAttribute') @@ -1683,8 +1646,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes') ->desc('List attributes') ->groups(['api', 'database']) ->label('scope', 'collections.read') - ->label('usage.metric', 'collections.{scope}.requests.read') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'listAttributes') @@ -1758,8 +1719,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes/:key') ->desc('Get attribute') ->groups(['api', 'database']) ->label('scope', 'collections.read') - ->label('usage.metric', 'collections.{scope}.requests.read') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'getAttribute') @@ -1837,8 +1796,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/strin ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') ->label('audits.event', 'attribute.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'updateStringAttribute') @@ -2282,8 +2239,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update') ->label('audits.event', 'attribute.delete') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'deleteAttribute') @@ -2393,8 +2348,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') ->label('scope', 'collections.write') ->label('audits.event', 'index.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'createIndex') @@ -2553,8 +2506,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes') ->desc('List indexes') ->groups(['api', 'database']) ->label('scope', 'collections.read') - ->label('usage.metric', 'collections.{scope}.requests.read') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'listIndexes') @@ -2618,8 +2569,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes/:key') ->desc('Get index') ->groups(['api', 'database']) ->label('scope', 'collections.read') - ->label('usage.metric', 'collections.{scope}.requests.read') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'getIndex') @@ -2662,8 +2611,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/indexes/:key') ->label('event', 'databases.[databaseId].collections.[collectionId].indexes.[indexId].update') ->label('audits.event', 'index.delete') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'collections.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}']) ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'deleteIndex') @@ -2728,8 +2675,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->label('scope', 'documents.write') ->label('audits.event', 'document.create') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') - ->label('usage.metric', 'documents.{scope}.requests.create') - ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}']) ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) @@ -2966,8 +2911,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->desc('List documents') ->groups(['api', 'database']) ->label('scope', 'documents.read') - ->label('usage.metric', 'documents.{scope}.requests.read') - ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}']) ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'listDocuments') @@ -3093,8 +3036,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->desc('Get document') ->groups(['api', 'database']) ->label('scope', 'documents.read') - ->label('usage.metric', 'documents.{scope}.requests.read') - ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}']) ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'getDocument') @@ -3188,8 +3129,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->desc('List document logs') ->groups(['api', 'database']) ->label('scope', 'documents.read') - ->label('usage.metric', 'documents.{scope}.requests.read') - ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}']) ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'databases') ->label('sdk.method', 'listDocumentLogs') @@ -3292,8 +3231,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->label('scope', 'documents.write') ->label('audits.event', 'document.update') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}') - ->label('usage.metric', 'documents.{scope}.requests.update') - ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}']) ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) @@ -3522,8 +3459,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].delete') ->label('audits.event', 'document.delete') ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{request.documentId}') - ->label('usage.metric', 'documents.{scope}.requests.delete') - ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}']) ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) @@ -3645,107 +3580,57 @@ App::get('/v1/databases/usage') ->inject('dbForProject') ->action(function (string $range, Response $response, Database $dbForProject) { - $usage = []; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $periods = [ - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - '7d' => [ - 'period' => '1d', - 'limit' => 7, - ], - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ], - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + METRIC_DATABASES, + METRIC_COLLECTIONS, + METRIC_DOCUMENTS, + ]; - $metrics = [ - 'databases.$all.count.total', - 'documents.$all.count.total', - 'collections.$all.count.total', - 'databases.$all.requests.create', - 'databases.$all.requests.read', - 'databases.$all.requests.update', - 'databases.$all.requests.delete', - 'collections.$all.requests.create', - 'collections.$all.requests.read', - 'collections.$all.requests.update', - 'collections.$all.requests.delete', - 'documents.$all.requests.create', - 'documents.$all.requests.read', - 'documents.$all.requests.update', - 'documents.$all.requests.delete' - ]; - - $stats = []; - - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - // backfill metrics with empty values for graphs - $backfill = $limit - \count($requestDocs); - while ($backfill > 0) { - $last = $limit - $backfill - 1; // array index of last added metric - $diff = match ($period) { // convert period to seconds for unix timestamp math - '1h' => 3600, - '1d' => 86400, - }; - $stats[$metric][] = [ - 'value' => 0, - 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), - ]; - $backfill--; - } - // Added 3'rd level to Index [period, metric, time] because of order by. - $stats[$metric] = array_reverse($stats[$metric]); + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric] = []; + foreach ($results as $result) { + $stats[$metric][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; } - }); + } + }); - $usage = new Document([ - 'range' => $range, - 'databasesCount' => $stats['databases.$all.count.total'] ?? [], - 'documentsCount' => $stats['documents.$all.count.total'] ?? [], - 'collectionsCount' => $stats['collections.$all.count.total'] ?? [], - 'documentsCreate' => $stats['documents.$all.requests.create'] ?? [], - 'documentsRead' => $stats['documents.$all.requests.read'] ?? [], - 'documentsUpdate' => $stats['documents.$all.requests.update'] ?? [], - 'documentsDelete' => $stats['documents.$all.requests.delete'] ?? [], - 'collectionsCreate' => $stats['collections.$all.requests.create'] ?? [], - 'collectionsRead' => $stats['collections.$all.requests.read'] ?? [], - 'collectionsUpdate' => $stats['collections.$all.requests.update'] ?? [], - 'collectionsDelete' => $stats['collections.$all.requests.delete'] ?? [], - 'databasesCreate' => $stats['databases.$all.requests.create'] ?? [], - 'databasesRead' => $stats['databases.$all.requests.read'] ?? [], - 'databasesUpdate' => $stats['databases.$all.requests.update'] ?? [], - 'databasesDelete' => $stats['databases.$all.requests.delete'] ?? [], - ]); + $format = match ($days['period']) { + '1h' => 'Y-m-d\TH:00:00.000P', + '1d' => 'Y-m-d\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric][] = [ + 'value' => $stats[$metric][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; } - - $response->dynamic($usage, Response::MODEL_USAGE_DATABASES); + } + $response->dynamic(new Document([ + 'range' => $range, + 'databasesTotal' => $usage[$metrics[0]], + 'collectionsTotal' => $usage[$metrics[1]], + 'documentsTotal' => $usage[$metrics[2]], + ]), Response::MODEL_USAGE_DATABASES); }); App::get('/v1/databases/:databaseId/usage') @@ -3764,97 +3649,62 @@ App::get('/v1/databases/:databaseId/usage') ->inject('dbForProject') ->action(function (string $databaseId, string $range, Response $response, Database $dbForProject) { - $usage = []; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $periods = [ - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - '7d' => [ - 'period' => '1d', - 'limit' => 7, - ], - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ], - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; + $database = $dbForProject->getDocument('databases', $databaseId); - $metrics = [ - 'collections.' . $databaseId . '.count.total', - 'collections.' . $databaseId . '.requests.create', - 'collections.' . $databaseId . '.requests.read', - 'collections.' . $databaseId . '.requests.update', - 'collections.' . $databaseId . '.requests.delete', - 'documents.' . $databaseId . '.count.total', - 'documents.' . $databaseId . '.requests.create', - 'documents.' . $databaseId . '.requests.read', - 'documents.' . $databaseId . '.requests.update', - 'documents.' . $databaseId . '.requests.delete' - ]; - - $stats = []; - - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - // backfill metrics with empty values for graphs - $backfill = $limit - \count($requestDocs); - while ($backfill > 0) { - $last = $limit - $backfill - 1; // array index of last added metric - $diff = match ($period) { // convert period to seconds for unix timestamp math - '1h' => 3600, - '1d' => 86400, - }; - $stats[$metric][] = [ - 'value' => 0, - 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), - ]; - $backfill--; - } - // TODO@kodumbeats explore performance if query is ordered by time ASC - $stats[$metric] = array_reverse($stats[$metric]); - } - }); - - $usage = new Document([ - 'range' => $range, - 'collectionsCount' => $stats["collections.{$databaseId}.count.total"] ?? [], - 'collectionsCreate' => $stats["collections.{$databaseId}.requests.create"] ?? [], - 'collectionsRead' => $stats["collections.{$databaseId}.requests.read"] ?? [], - 'collectionsUpdate' => $stats["collections.{$databaseId}.requests.update"] ?? [], - 'collectionsDelete' => $stats["collections.{$databaseId}.requests.delete"] ?? [], - 'documentsCount' => $stats["documents.{$databaseId}.count.total"] ?? [], - 'documentsCreate' => $stats["documents.{$databaseId}.requests.create"] ?? [], - 'documentsRead' => $stats["documents.{$databaseId}.requests.read"] ?? [], - 'documentsUpdate' => $stats["documents.{$databaseId}.requests.update"] ?? [], - 'documentsDelete' => $stats["documents.{$databaseId}.requests.delete"] ?? [], - ]); + if ($database->isEmpty()) { + throw new Exception(Exception::DATABASE_NOT_FOUND); } - $response->dynamic($usage, Response::MODEL_USAGE_DATABASE); + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS), + str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS), + ]; + + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric] = []; + foreach ($results as $result) { + $stats[$metric][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; + } + } + }); + + $format = match ($days['period']) { + '1h' => 'Y-m-d\TH:00:00.000P', + '1d' => 'Y-m-d\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric][] = [ + 'value' => $stats[$metric][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; + } + } + + $response->dynamic(new Document([ + 'range' => $range, + 'collectionsTotal' => $usage[$metrics[0]], + 'documentsTotal' => $usage[$metrics[1]], + ]), Response::MODEL_USAGE_DATABASE); }); App::get('/v1/databases/:databaseId/collections/:collectionId/usage') @@ -3883,84 +3733,52 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage') throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $usage = []; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $periods = [ - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - '7d' => [ - 'period' => '1d', - 'limit' => 7, - ], - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ], - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collectionDocument->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), + ]; - $metrics = [ - "documents.{$databaseId}/{$collectionId}.count.total", - "documents.{$databaseId}/{$collectionId}.requests.create", - "documents.{$databaseId}/{$collectionId}.requests.read", - "documents.{$databaseId}/{$collectionId}.requests.update", - "documents.{$databaseId}/{$collectionId}.requests.delete", - ]; - - $stats = []; - - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - // backfill metrics with empty values for graphs - $backfill = $limit - \count($requestDocs); - while ($backfill > 0) { - $last = $limit - $backfill - 1; // array index of last added metric - $diff = match ($period) { // convert period to seconds for unix timestamp math - '1h' => 3600, - '1d' => 86400, - }; - $stats[$metric][] = [ - 'value' => 0, - 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), - ]; - $backfill--; - } - $stats[$metric] = array_reverse($stats[$metric]); + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric] = []; + foreach ($results as $result) { + $stats[$metric][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; } - }); + } + }); - $usage = new Document([ - 'range' => $range, - 'documentsCount' => $stats["documents.{$databaseId}/{$collectionId}.count.total"] ?? [], - 'documentsCreate' => $stats["documents.{$databaseId}/{$collectionId}.requests.create"] ?? [], - 'documentsRead' => $stats["documents.{$databaseId}/{$collectionId}.requests.read"] ?? [], - 'documentsUpdate' => $stats["documents.{$databaseId}/{$collectionId}.requests.update"] ?? [], - 'documentsDelete' => $stats["documents.{$databaseId}/{$collectionId}.requests.delete" ?? []] - ]); + $format = match ($days['period']) { + '1h' => 'Y-m-d\TH:00:00.000P', + '1d' => 'Y-m-d\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric][] = [ + 'value' => $stats[$metric][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; } + } - $response->dynamic($usage, Response::MODEL_USAGE_COLLECTION); + $response->dynamic(new Document([ + 'range' => $range, + 'documentsTotal' => $usage[$metrics[0]], + ]), Response::MODEL_USAGE_COLLECTION); }); diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 4a96308492..da36619c81 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -12,7 +12,6 @@ use Appwrite\Extend\Exception; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Messaging\Adapter\Realtime; use Utopia\Validator\Assoc; -use Appwrite\Usage\Stats; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; @@ -458,97 +457,71 @@ App::get('/v1/functions/:functionId/usage') throw new Exception(Exception::FUNCTION_NOT_FOUND); } - $usage = []; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $periods = [ - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - '7d' => [ - 'period' => '1d', - 'limit' => 7, - ], - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ], - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS), + str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), + str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), + str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), + str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), + str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), + str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), + ]; - $metrics = [ - "executions.$functionId.compute.total", - "executions.$functionId.compute.success", - "executions.$functionId.compute.failure", - "executions.$functionId.compute.time", - "builds.$functionId.compute.total", - "builds.$functionId.compute.success", - "builds.$functionId.compute.failure", - "builds.$functionId.compute.time", - ]; - - $stats = []; - - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - // backfill metrics with empty values for graphs - $backfill = $limit - \count($requestDocs); - while ($backfill > 0) { - $last = $limit - $backfill - 1; // array index of last added metric - $diff = match ($period) { // convert period to seconds for unix timestamp math - '1h' => 3600, - '1d' => 86400, - }; - $stats[$metric][] = [ - 'value' => 0, - 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), - ]; - $backfill--; - } - $stats[$metric] = array_reverse($stats[$metric]); + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric] = []; + foreach ($results as $result) { + $stats[$metric][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; } - }); + } + }); - $usage = new Document([ - 'range' => $range, - 'executionsTotal' => $stats["executions.$functionId.compute.total"] ?? [], - 'executionsFailure' => $stats["executions.$functionId.compute.failure"] ?? [], - 'executionsSuccess' => $stats["executions.$functionId.compute.success"] ?? [], - 'executionsTime' => $stats["executions.$functionId.compute.time"] ?? [], - 'buildsTotal' => $stats["builds.$functionId.compute.total"] ?? [], - 'buildsFailure' => $stats["builds.$functionId.compute.failure"] ?? [], - 'buildsSuccess' => $stats["builds.$functionId.compute.success"] ?? [], - 'buildsTime' => $stats["builds.$functionId.compute.time" ?? []] - ]); + $format = match ($days['period']) { + '1h' => 'Y-m-d\TH:00:00.000P', + '1d' => 'Y-m-d\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric][] = [ + 'value' => $stats[$metric][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; } + } - $response->dynamic($usage, Response::MODEL_USAGE_FUNCTION); + $response->dynamic(new Document([ + 'range' => $range, + 'deploymentsTotal' => $usage[$metrics[0]], + 'deploymentsStorage' => $usage[$metrics[1]], + 'buildsTotal' => $usage[$metrics[2]], + 'buildsStorage' => $usage[$metrics[3]], + 'buildsTime' => $usage[$metrics[4]], + 'executionsTotal' => $usage[$metrics[5]], + 'executionsTime' => $usage[$metrics[6]], + ]), Response::MODEL_USAGE_FUNCTION); }); App::get('/v1/functions/usage') ->desc('Get functions usage') - ->groups(['api', 'functions', 'usage']) + ->groups(['api', 'functions']) ->label('scope', 'functions.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'functions') @@ -561,92 +534,67 @@ App::get('/v1/functions/usage') ->inject('dbForProject') ->action(function (string $range, Response $response, Database $dbForProject) { - $usage = []; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $periods = [ - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - '7d' => [ - 'period' => '1d', - 'limit' => 7, - ], - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ], - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + METRIC_FUNCTIONS, + METRIC_DEPLOYMENTS, + METRIC_DEPLOYMENTS_STORAGE, + METRIC_BUILDS, + METRIC_BUILDS_STORAGE, + METRIC_BUILDS_COMPUTE, + METRIC_EXECUTIONS, + METRIC_EXECUTIONS_COMPUTE, + ]; - $metrics = [ - 'executions.$all.compute.total', - 'executions.$all.compute.failure', - 'executions.$all.compute.success', - 'executions.$all.compute.time', - 'builds.$all.compute.total', - 'builds.$all.compute.failure', - 'builds.$all.compute.success', - 'builds.$all.compute.time', - ]; - - $stats = []; - - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - // backfill metrics with empty values for graphs - $backfill = $limit - \count($requestDocs); - while ($backfill > 0) { - $last = $limit - $backfill - 1; // array index of last added metric - $diff = match ($period) { // convert period to seconds for unix timestamp math - '1h' => 3600, - '1d' => 86400, - }; - $stats[$metric][] = [ - 'value' => 0, - 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), - ]; - $backfill--; - } - $stats[$metric] = array_reverse($stats[$metric]); + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric] = []; + foreach ($results as $result) { + $stats[$metric][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; } - }); + } + }); - $usage = new Document([ - 'range' => $range, - 'executionsTotal' => $stats[$metrics[0]] ?? [], - 'executionsFailure' => $stats[$metrics[1]] ?? [], - 'executionsSuccess' => $stats[$metrics[2]] ?? [], - 'executionsTime' => $stats[$metrics[3]] ?? [], - 'buildsTotal' => $stats[$metrics[4]] ?? [], - 'buildsFailure' => $stats[$metrics[5]] ?? [], - 'buildsSuccess' => $stats[$metrics[6]] ?? [], - 'buildsTime' => $stats[$metrics[7]] ?? [], - ]); + $format = match ($days['period']) { + '1h' => 'Y-m-d\TH:00:00.000P', + '1d' => 'Y-m-d\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric][] = [ + 'value' => $stats[$metric][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; } - - $response->dynamic($usage, Response::MODEL_USAGE_FUNCTIONS); + } + $response->dynamic(new Document([ + 'range' => $range, + 'functionsTotal' => $usage[$metrics[0]], + 'deploymentsTotal' => $usage[$metrics[1]], + 'deploymentsStorage' => $usage[$metrics[2]], + 'buildsTotal' => $usage[$metrics[3]], + 'buildsStorage' => $usage[$metrics[4]], + 'buildsTime' => $usage[$metrics[5]], + 'executionsTotal' => $usage[$metrics[6]], + 'executionsTime' => $usage[$metrics[7]], + ]), Response::MODEL_USAGE_FUNCTIONS); }); App::put('/v1/functions/:functionId') @@ -1511,11 +1459,11 @@ App::post('/v1/functions/:functionId/executions') ->inject('dbForProject') ->inject('user') ->inject('queueForEvents') - ->inject('usage') + ->inject('queueForUsage') ->inject('mode') ->inject('queueForFunctions') ->inject('geodb') - ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $queueForEvents, Stats $usage, string $mode, Func $queueForFunctions, Reader $geodb) { + ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $queueForEvents, Usage $queueForUsage, string $mode, Func $queueForFunctions, Reader $geodb) { $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); @@ -1748,12 +1696,13 @@ App::post('/v1/functions/:functionId/executions') $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution)); } - // TODO revise this later using route label - $usage - ->setParam('functionId', $function->getId()) - ->setParam('executions.{scope}.compute', 1) - ->setParam('executionStatus', $execution->getAttribute('status', '')) - ->setParam('executionTime', $execution->getAttribute('duration')); // ms + $queueForUsage + ->addMetric(METRIC_EXECUTIONS, 1) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1) + ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($executionResponse['duration'] * 1000))// per project + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($executionResponse['duration'] * 1000))// per function + ; + $roles = Authorization::getRoles(); diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index bd8d9d5f73..bf0c6fc24b 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -18,7 +18,7 @@ use Utopia\Database\DateTime; App::get('/v1/project/usage') ->desc('Get usage stats for a project') - ->groups(['api']) + ->groups(['api', 'usage']) ->label('scope', 'projects.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'project') @@ -30,94 +30,76 @@ App::get('/v1/project/usage') ->inject('response') ->inject('dbForProject') ->action(function (string $range, Response $response, Database $dbForProject) { - $usage = []; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $periods = [ - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - '7d' => [ - 'period' => '1d', - 'limit' => 7, - ], - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ], - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; - $metrics = [ - 'project.$all.network.requests', - 'project.$all.network.bandwidth', - 'project.$all.storage.size', - 'users.$all.count.total', - 'databases.$all.count.total', - 'documents.$all.count.total', - 'executions.$all.compute.total', - 'buckets.$all.count.total' - ]; + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + METRIC_NETWORK_REQUESTS, + METRIC_NETWORK_INBOUND, + METRIC_NETWORK_OUTBOUND, + METRIC_EXECUTIONS, + METRIC_DOCUMENTS, + METRIC_DATABASES, + METRIC_USERS, + METRIC_BUCKETS, + METRIC_FILES_STORAGE + ]; - $stats = []; + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - // backfill metrics with empty values for graphs - $backfill = $limit - \count($requestDocs); - while ($backfill > 0) { - $last = $limit - $backfill - 1; // array index of last added metric - $diff = match ($period) { // convert period to seconds for unix timestamp math - '1h' => 3600, - '1d' => 86400, - }; - $stats[$metric][] = [ - 'value' => 0, - 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), - ]; - $backfill--; - } - $stats[$metric] = array_reverse($stats[$metric]); + $stats[$metric] = []; + foreach ($results as $result) { + $stats[$metric][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; } - }); + } + }); - $usage = new Document([ - 'range' => $range, - 'requests' => $stats[$metrics[0]] ?? [], - 'network' => $stats[$metrics[1]] ?? [], - 'storage' => $stats[$metrics[2]] ?? [], - 'users' => $stats[$metrics[3]] ?? [], - 'databases' => $stats[$metrics[4]] ?? [], - 'documents' => $stats[$metrics[5]] ?? [], - 'executions' => $stats[$metrics[6]] ?? [], - 'buckets' => $stats[$metrics[7]] ?? [], - ]); + + $format = match ($days['period']) { + '1h' => 'Y-m-d\TH:00:00.000P', + '1d' => 'Y-m-d\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric][] = [ + 'value' => $stats[$metric][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; } + } - $response->dynamic($usage, Response::MODEL_USAGE_PROJECT); + + $response->dynamic(new Document([ + 'range' => $range, + 'requestsTotal' => ($usage[$metrics[0]]), + 'network' => ($usage[$metrics[1]] + $usage[$metrics[2]]), + 'executionsTotal' => $usage[$metrics[3]], + 'documentsTotal' => $usage[$metrics[4]], + 'databasesTotal' => $usage[$metrics[5]], + 'usersTotal' => $usage[$metrics[6]], + 'bucketsTotal' => $usage[$metrics[7]], + 'filesStorage' => $usage[$metrics[8]], + ]), Response::MODEL_USAGE_PROJECT); }); + // Variables App::post('/v1/project/variables') diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index fe7759a6df..200c7372f2 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -308,120 +308,6 @@ App::get('/v1/projects/:projectId') $response->dynamic($project, Response::MODEL_PROJECT); }); -App::get('/v1/projects/:projectId/usage') - ->desc('Get usage stats for a project') - ->groups(['api', 'projects', 'usage']) - ->label('scope', 'projects.read') - ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) - ->label('sdk.namespace', 'projects') - ->label('sdk.method', 'getUsage') - ->label('sdk.response.code', Response::STATUS_CODE_OK) - ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_USAGE_PROJECT) - ->param('projectId', '', new UID(), 'Project unique ID.') - ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) - ->inject('response') - ->inject('dbForConsole') - ->inject('dbForProject') - ->inject('register') - ->action(function (string $projectId, string $range, Response $response, Database $dbForConsole, Database $dbForProject, Registry $register) { - - $project = $dbForConsole->getDocument('projects', $projectId); - - if ($project->isEmpty()) { - throw new Exception(Exception::PROJECT_NOT_FOUND); - } - - $usage = []; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $periods = [ - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - '7d' => [ - 'period' => '1d', - 'limit' => 7, - ], - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ], - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; - - $dbForProject->setNamespace("_{$project->getInternalId()}"); - - $metrics = [ - 'project.$all.network.requests', - 'project.$all.network.bandwidth', - 'project.$all.storage.size', - 'users.$all.count.total', - 'databases.$all.count.total', - 'documents.$all.count.total', - 'executions.$all.compute.total', - 'buckets.$all.count.total' - ]; - - $stats = []; - - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - // backfill metrics with empty values for graphs - $backfill = $limit - \count($requestDocs); - while ($backfill > 0) { - $last = $limit - $backfill - 1; // array index of last added metric - $diff = match ($period) { // convert period to seconds for unix timestamp math - '1h' => 3600, - '1d' => 86400, - }; - $stats[$metric][] = [ - 'value' => 0, - 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), - ]; - $backfill--; - } - $stats[$metric] = array_reverse($stats[$metric]); - } - }); - - $usage = new Document([ - 'range' => $range, - 'requests' => $stats[$metrics[0]] ?? [], - 'network' => $stats[$metrics[1]] ?? [], - 'storage' => $stats[$metrics[2]] ?? [], - 'users' => $stats[$metrics[3]] ?? [], - 'databases' => $stats[$metrics[4]] ?? [], - 'documents' => $stats[$metrics[5]] ?? [], - 'executions' => $stats[$metrics[6]] ?? [], - 'buckets' => $stats[$metrics[7]] ?? [], - ]); - } - - $response->dynamic($usage, Response::MODEL_USAGE_PROJECT); - }); - App::patch('/v1/projects/:projectId') ->desc('Update project') ->groups(['api', 'projects']) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 04c3f300e9..0c7f10a0e0 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -52,7 +52,6 @@ App::post('/v1/storage/buckets') ->label('event', 'buckets.[bucketId].create') ->label('audits.event', 'bucket.create') ->label('audits.resource', 'bucket/{response.$id}') - ->label('usage.metric', 'buckets.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'createBucket') @@ -148,7 +147,6 @@ App::get('/v1/storage/buckets') ->desc('List buckets') ->groups(['api', 'storage']) ->label('scope', 'buckets.read') - ->label('usage.metric', 'buckets.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'listBuckets') @@ -197,7 +195,6 @@ App::get('/v1/storage/buckets/:bucketId') ->desc('Get bucket') ->groups(['api', 'storage']) ->label('scope', 'buckets.read') - ->label('usage.metric', 'buckets.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getBucket') @@ -226,7 +223,6 @@ App::put('/v1/storage/buckets/:bucketId') ->label('event', 'buckets.[bucketId].update') ->label('audits.event', 'bucket.update') ->label('audits.resource', 'bucket/{response.$id}') - ->label('usage.metric', 'buckets.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'updateBucket') @@ -294,7 +290,6 @@ App::delete('/v1/storage/buckets/:bucketId') ->label('audits.event', 'bucket.delete') ->label('event', 'buckets.[bucketId].delete') ->label('audits.resource', 'bucket/{request.bucketId}') - ->label('usage.metric', 'buckets.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'deleteBucket') @@ -337,8 +332,6 @@ App::post('/v1/storage/buckets/:bucketId/files') ->label('audits.event', 'file.create') ->label('event', 'buckets.[bucketId].files.[fileId].create') ->label('audits.resource', 'file/{response.$id}') - ->label('usage.metric', 'files.{scope}.requests.create') - ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) @@ -689,8 +682,6 @@ App::get('/v1/storage/buckets/:bucketId/files') ->groups(['api', 'storage']) ->label('scope', 'files.read') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) - ->label('usage.metric', 'files.{scope}.requests.read') - ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'listFiles') ->label('sdk.description', '/docs/references/storage/list-files.md') @@ -770,8 +761,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId') ->groups(['api', 'storage']) ->label('scope', 'files.read') ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) - ->label('usage.metric', 'files.{scope}.requests.read') - ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getFile') ->label('sdk.description', '/docs/references/storage/get-file.md') @@ -821,8 +810,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->label('cache', true) ->label('cache.resourceType', 'bucket/{request.bucketId}') ->label('cache.resource', 'file/{request.fileId}') - ->label('usage.metric', 'files.{scope}.requests.read') - ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getFilePreview') @@ -987,8 +974,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download') ->desc('Get file for download') ->groups(['api', 'storage']) ->label('scope', 'files.read') - ->label('usage.metric', 'files.{scope}.requests.read') - ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getFileDownload') @@ -1130,8 +1115,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view') ->desc('Get file for view') ->groups(['api', 'storage']) ->label('scope', 'files.read') - ->label('usage.metric', 'files.{scope}.requests.read') - ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getFileView') @@ -1286,8 +1269,6 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId') ->label('event', 'buckets.[bucketId].files.[fileId].update') ->label('audits.event', 'file.update') ->label('audits.resource', 'file/{response.$id}') - ->label('usage.metric', 'files.{scope}.requests.update') - ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) @@ -1396,8 +1377,6 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') ->label('event', 'buckets.[bucketId].files.[fileId].delete') ->label('audits.event', 'file.delete') ->label('audits.resource', 'file/{request.fileId}') - ->label('usage.metric', 'files.{scope}.requests.delete') - ->label('usage.params', ['bucketId:{request.bucketId}']) ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT) ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) @@ -1489,7 +1468,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId') App::get('/v1/storage/usage') ->desc('Get usage stats for storage') - ->groups(['api', 'storage', 'usage']) + ->groups(['api', 'storage']) ->label('scope', 'files.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'storage') @@ -1502,104 +1481,63 @@ App::get('/v1/storage/usage') ->inject('dbForProject') ->action(function (string $range, Response $response, Database $dbForProject) { - $usage = []; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { - $periods = [ - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - '7d' => [ - 'period' => '1d', - 'limit' => 7, - ], - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ], - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + METRIC_BUCKETS, + METRIC_FILES, + METRIC_FILES_STORAGE, + ]; - $metrics = [ - 'project.$all.storage.size', - 'buckets.$all.count.total', - 'buckets.$all.requests.create', - 'buckets.$all.requests.read', - 'buckets.$all.requests.update', - 'buckets.$all.requests.delete', - 'files.$all.storage.size', - 'files.$all.count.total', - 'files.$all.requests.create', - 'files.$all.requests.read', - 'files.$all.requests.update', - 'files.$all.requests.delete', - ]; - - $stats = []; - - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - // backfill metrics with empty values for graphs - $backfill = $limit - \count($requestDocs); - while ($backfill > 0) { - $last = $limit - $backfill - 1; // array index of last added metric - $diff = match ($period) { // convert period to seconds for unix timestamp math - '1h' => 3600, - '1d' => 86400, - }; - $stats[$metric][] = [ - 'value' => 0, - 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), - ]; - $backfill--; - } - $stats[$metric] = array_reverse($stats[$metric]); + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric] = []; + foreach ($results as $result) { + $stats[$metric][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; } - }); + } + }); - $usage = new Document([ - 'range' => $range, - 'bucketsCount' => $stats['buckets.$all.count.total'], - 'bucketsCreate' => $stats['buckets.$all.requests.create'], - 'bucketsRead' => $stats['buckets.$all.requests.read'], - 'bucketsUpdate' => $stats['buckets.$all.requests.update'], - 'bucketsDelete' => $stats['buckets.$all.requests.delete'], - 'storage' => $stats['project.$all.storage.size'], - 'filesCount' => $stats['files.$all.count.total'], - 'filesCreate' => $stats['files.$all.requests.create'], - 'filesRead' => $stats['files.$all.requests.read'], - 'filesUpdate' => $stats['files.$all.requests.update'], - 'filesDelete' => $stats['files.$all.requests.delete'], - ]); + $format = match ($days['period']) { + '1h' => 'Y-m-d\TH:00:00.000P', + '1d' => 'Y-m-d\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric][] = [ + 'value' => $stats[$metric][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; } + } - $response->dynamic($usage, Response::MODEL_USAGE_STORAGE); + $response->dynamic(new Document([ + 'range' => $range, + 'bucketsTotal' => $usage[$metrics[0]], + 'filesTotal' => $usage[$metrics[1]], + 'filesStorage' => $usage[$metrics[2]], + ]), Response::MODEL_USAGE_STORAGE); }); App::get('/v1/storage/:bucketId/usage') - ->desc('Get usage stats for a storage bucket') - ->groups(['api', 'storage', 'usage']) + ->desc('Get usage stats for storage bucket') + ->groups(['api', 'storage']) ->label('scope', 'files.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'storage') @@ -1619,86 +1557,55 @@ App::get('/v1/storage/:bucketId/usage') throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } - $usage = []; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { - $periods = [ - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - '7d' => [ - 'period' => '1d', - 'limit' => 7, - ], - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ], - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES), + str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE), + ]; - $metrics = [ - "files.{$bucketId}.count.total", - "files.{$bucketId}.storage.size", - "files.{$bucketId}.requests.create", - "files.{$bucketId}.requests.read", - "files.{$bucketId}.requests.update", - "files.{$bucketId}.requests.delete", - ]; - - $stats = []; - - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - // backfill metrics with empty values for graphs - $backfill = $limit - \count($requestDocs); - while ($backfill > 0) { - $last = $limit - $backfill - 1; // array index of last added metric - $diff = match ($period) { // convert period to seconds for unix timestamp math - '1h' => 3600, - '1d' => 86400, - }; - $stats[$metric][] = [ - 'value' => 0, - 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), - ]; - $backfill--; - } - $stats[$metric] = array_reverse($stats[$metric]); + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric] = []; + foreach ($results as $result) { + $stats[$metric][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; } - }); + } + }); - $usage = new Document([ - 'range' => $range, - 'filesCount' => $stats[$metrics[0]], - 'filesStorage' => $stats[$metrics[1]], - 'filesCreate' => $stats[$metrics[2]], - 'filesRead' => $stats[$metrics[3]], - 'filesUpdate' => $stats[$metrics[4]], - 'filesDelete' => $stats[$metrics[5]], - ]); + $format = match ($days['period']) { + '1h' => 'Y-m-d\TH:00:00.000P', + '1d' => 'Y-m-d\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric] = []; + $leap = time() - ($days['limit'] * $days['factor']); + + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric][] = [ + 'value' => $stats[$metric][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; } + } - $response->dynamic($usage, Response::MODEL_USAGE_BUCKETS); + $response->dynamic(new Document([ + 'range' => $range, + 'filesTotal' => $usage[$metrics[0]], + 'filesStorage' => $usage[$metrics[1]], + ]), Response::MODEL_USAGE_BUCKETS); }); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index fcaa2ff641..a992d51965 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -114,7 +114,6 @@ App::post('/v1/users') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'create') @@ -146,7 +145,6 @@ App::post('/v1/users/bcrypt') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createBcryptUser') @@ -177,7 +175,6 @@ App::post('/v1/users/md5') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createMD5User') @@ -208,7 +205,6 @@ App::post('/v1/users/argon2') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createArgon2User') @@ -239,7 +235,6 @@ App::post('/v1/users/sha') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createSHAUser') @@ -277,7 +272,6 @@ App::post('/v1/users/phpass') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createPHPassUser') @@ -308,7 +302,6 @@ App::post('/v1/users/scrypt') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createScryptUser') @@ -352,7 +345,6 @@ App::post('/v1/users/scrypt-modified') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createScryptModifiedUser') @@ -383,7 +375,6 @@ App::get('/v1/users') ->desc('List users') ->groups(['api', 'users']) ->label('scope', 'users.read') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'list') @@ -432,7 +423,6 @@ App::get('/v1/users/:userId') ->desc('Get user') ->groups(['api', 'users']) ->label('scope', 'users.read') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'get') @@ -458,7 +448,6 @@ App::get('/v1/users/:userId/prefs') ->desc('Get user preferences') ->groups(['api', 'users']) ->label('scope', 'users.read') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'getPrefs') @@ -486,7 +475,6 @@ App::get('/v1/users/:userId/sessions') ->desc('List user sessions') ->groups(['api', 'users']) ->label('scope', 'users.read') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'listSessions') @@ -528,7 +516,6 @@ App::get('/v1/users/:userId/memberships') ->desc('List user memberships') ->groups(['api', 'users']) ->label('scope', 'users.read') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'listMemberships') @@ -568,7 +555,6 @@ App::get('/v1/users/:userId/logs') ->desc('List user logs') ->groups(['api', 'users']) ->label('scope', 'users.read') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'listLogs') @@ -650,7 +636,6 @@ App::get('/v1/users/identities') ->desc('List Identities') ->groups(['api', 'users']) ->label('scope', 'users.read') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'listIdentities') @@ -703,7 +688,6 @@ App::patch('/v1/users/:userId/status') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updateStatus') @@ -739,7 +723,6 @@ App::put('/v1/users/:userId/labels') ->label('scope', 'users.write') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updateLabels') @@ -777,7 +760,6 @@ App::patch('/v1/users/:userId/verification/phone') ->label('scope', 'users.write') ->label('audits.event', 'verification.update') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updatePhoneVerification') @@ -814,7 +796,6 @@ App::patch('/v1/users/:userId/name') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updateName') @@ -852,7 +833,6 @@ App::patch('/v1/users/:userId/password') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updatePassword') @@ -917,7 +897,6 @@ App::patch('/v1/users/:userId/email') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updateEmail') @@ -973,7 +952,6 @@ App::patch('/v1/users/:userId/phone') ->label('scope', 'users.write') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updatePhone') @@ -1018,7 +996,6 @@ App::patch('/v1/users/:userId/verification') ->label('audits.event', 'verification.update') ->label('audits.resource', 'user/{request.userId}') ->label('audits.userId', '{request.userId}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updateEmailVerification') @@ -1051,7 +1028,6 @@ App::patch('/v1/users/:userId/prefs') ->groups(['api', 'users']) ->label('event', 'users.[userId].update.prefs') ->label('scope', 'users.write') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updatePrefs') @@ -1087,7 +1063,6 @@ App::delete('/v1/users/:userId/sessions/:sessionId') ->label('scope', 'users.write') ->label('audits.event', 'session.delete') ->label('audits.resource', 'user/{request.userId}') - ->label('usage.metric', 'sessions.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'deleteSession') @@ -1131,7 +1106,6 @@ App::delete('/v1/users/:userId/sessions') ->label('scope', 'users.write') ->label('audits.event', 'session.delete') ->label('audits.resource', 'user/{user.$id}') - ->label('usage.metric', 'sessions.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'deleteSessions') @@ -1174,7 +1148,6 @@ App::delete('/v1/users/:userId') ->label('scope', 'users.write') ->label('audits.event', 'user.delete') ->label('audits.resource', 'user/{request.userId}') - ->label('usage.metric', 'users.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'delete') @@ -1217,7 +1190,6 @@ App::delete('/v1/users/identities/:identityId') ->label('scope', 'users.write') ->label('audits.event', 'identity.delete') ->label('audits.resource', 'identity/{request.$identityId}') - ->label('usage.metric', 'users.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'deleteIdentity') @@ -1242,7 +1214,7 @@ App::delete('/v1/users/identities/:identityId') App::get('/v1/users/usage') ->desc('Get usage stats for the users API') - ->groups(['api', 'users', 'usage']) + ->groups(['api', 'users']) ->label('scope', 'users.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'users') @@ -1250,97 +1222,60 @@ App::get('/v1/users/usage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USAGE_USERS) - ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) - ->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(fn ($value) => "oauth-" . $value, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true) + ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForProject') ->inject('register') - ->action(function (string $range, string $provider, Response $response, Database $dbForProject) { + ->action(function (string $range, Response $response, Database $dbForProject) { - $usage = []; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $periods = [ - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - '7d' => [ - 'period' => '1d', - 'limit' => 7, - ], - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ], - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + METRIC_USERS, + METRIC_SESSIONS, + ]; - $metrics = [ - 'users.$all.count.total', - 'users.$all.requests.create', - 'users.$all.requests.read', - 'users.$all.requests.update', - 'users.$all.requests.delete', - 'sessions.$all.requests.create', - 'sessions.$all.requests.delete', - "sessions.$provider.requests.create", - ]; - - $stats = []; - - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - // backfill metrics with empty values for graphs - $backfill = $limit - \count($requestDocs); - while ($backfill > 0) { - $last = $limit - $backfill - 1; // array index of last added metric - $diff = match ($period) { // convert period to seconds for unix timestamp math - '1h' => 3600, - '1d' => 86400, - }; - $stats[$metric][] = [ - 'value' => 0, - 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), - ]; - $backfill--; - } - $stats[$metric] = array_reverse($stats[$metric]); + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric] = []; + foreach ($results as $result) { + $stats[$metric][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; } - }); + } + }); - $usage = new Document([ - 'range' => $range, - 'usersCount' => $stats['users.$all.count.total'] ?? [], - 'usersCreate' => $stats['users.$all.requests.create'] ?? [], - 'usersRead' => $stats['users.$all.requests.read'] ?? [], - 'usersUpdate' => $stats['users.$all.requests.update'] ?? [], - 'usersDelete' => $stats['users.$all.requests.delete'] ?? [], - 'sessionsCreate' => $stats['sessions.$all.requests.create'] ?? [], - 'sessionsProviderCreate' => $stats["sessions.$provider.requests.create"] ?? [], - 'sessionsDelete' => $stats['sessions.$all.requests.delete' ?? []] - ]); + $format = match ($days['period']) { + '1h' => 'Y-m-d\TH:00:00.000P', + '1d' => 'Y-m-d\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric][] = [ + 'value' => $stats[$metric][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; } + } - $response->dynamic($usage, Response::MODEL_USAGE_USERS); + $response->dynamic(new Document([ + 'range' => $range, + 'usersTotal' => $usage[$metrics[0]], + 'sessionsTotal' => $usage[$metrics[1]], + ]), Response::MODEL_USAGE_USERS); }); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 388851faef..872f2902d1 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -8,8 +8,8 @@ use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Extend\Exception; +use Appwrite\Event\Usage; use Appwrite\Messaging\Adapter\Realtime; -use Appwrite\Usage\Stats; use Appwrite\Utopia\Response; use Appwrite\Utopia\Request; use Utopia\App; @@ -48,43 +48,95 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar return $label; }; -$databaseListener = function (string $event, Document $document, Stats $usage) { - $multiplier = 1; +$databaseListener = function (string $event, Document $document, Document $project, Usage $queueForUsage, Database $dbForProject) { + + $value = 1; if ($event === Database::EVENT_DOCUMENT_DELETE) { - $multiplier = -1; + $value = -1; } - $collection = $document->getCollection(); - switch ($collection) { - case 'users': - $usage->setParam('users.{scope}.count.total', 1 * $multiplier); + switch (true) { + case $document->getCollection() === 'teams': + $queueForUsage + ->addMetric(METRIC_TEAMS, $value); // per project break; - case 'databases': - $usage->setParam('databases.{scope}.count.total', 1 * $multiplier); + case $document->getCollection() === 'users': + $queueForUsage + ->addMetric(METRIC_USERS, $value); // per project + if ($event === Database::EVENT_DOCUMENT_DELETE) { + $queueForUsage + ->addReduce($document); + } break; - case 'buckets': - $usage->setParam('buckets.{scope}.count.total', 1 * $multiplier); + case $document->getCollection() === 'sessions': // sessions + $queueForUsage + ->addMetric(METRIC_SESSIONS, $value); //per project break; - case 'deployments': - $usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size') * $multiplier); + case $document->getCollection() === 'databases': // databases + $queueForUsage + ->addMetric(METRIC_DATABASES, $value); // per project + + if ($event === Database::EVENT_DOCUMENT_DELETE) { + $queueForUsage + ->addReduce($document); + } + break; + case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections + $parts = explode('_', $document->getCollection()); + $databaseInternalId = $parts[1] ?? 0; + $queueForUsage + ->addMetric(METRIC_COLLECTIONS, $value) // per project + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database + ; + + if ($event === Database::EVENT_DOCUMENT_DELETE) { + $queueForUsage + ->addReduce($document); + } + break; + case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents + $parts = explode('_', $document->getCollection()); + $databaseInternalId = $parts[1] ?? 0; + $collectionInternalId = $parts[3] ?? 0; + $queueForUsage + ->addMetric(METRIC_DOCUMENTS, $value) // per project + ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection + break; + case $document->getCollection() === 'buckets': //buckets + $queueForUsage + ->addMetric(METRIC_BUCKETS, $value); // per project + if ($event === Database::EVENT_DOCUMENT_DELETE) { + $queueForUsage + ->addReduce($document); + } + break; + case str_starts_with($document->getCollection(), 'bucket_'): // files + $parts = explode('_', $document->getCollection()); + $bucketInternalId = $parts[1]; + $queueForUsage + ->addMetric(METRIC_FILES, $value) // per project + ->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project + ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES), $value) // per bucket + ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket + break; + case $document->getCollection() === 'functions': + $queueForUsage + ->addMetric(METRIC_FUNCTIONS, $value); // per project + + if ($event === Database::EVENT_DOCUMENT_DELETE) { + $queueForUsage + ->addReduce($document); + } + break; + case $document->getCollection() === 'deployments': + $queueForUsage + ->addMetric(METRIC_DEPLOYMENTS, $value) // per project + ->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value)// per function + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), $document->getAttribute('size') * $value); break; default: - if (strpos($collection, 'bucket_') === 0) { - $usage - ->setParam('bucketId', $document->getAttribute('bucketId')) - ->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal') * $multiplier) - ->setParam('files.{scope}.count.total', 1 * $multiplier); - } elseif (strpos($collection, 'database_') === 0) { - $usage - ->setParam('databaseId', $document->getAttribute('databaseId')); - if (strpos($collection, '_collection_') !== false) { - $usage - ->setParam('collectionId', $document->getAttribute('$collectionId')) - ->setParam('documents.{scope}.count.total', 1 * $multiplier); - } else { - $usage->setParam('collections.{scope}.count.total', 1 * $multiplier); - } - } break; } }; @@ -103,8 +155,8 @@ App::init() ->inject('dbForProject') ->inject('mode') ->inject('queueForMails') - ->inject('usage') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, string $mode, Mail $queueForMails, Stats $usage) use ($databaseListener) { + ->inject('queueForUsage') + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, string $mode, Mail $queueForMails, Usage $queueForUsage) use ($databaseListener) { $route = $utopia->getRoute(); @@ -186,19 +238,14 @@ App::init() ->setProject($project) ->setUser($user); - $usage - ->setParam('projectInternalId', $project->getInternalId()) - ->setParam('projectId', $project->getId()) - ->setParam('project.{scope}.network.requests', 1) - ->setParam('httpMethod', $request->getMethod()) - ->setParam('project.{scope}.network.inbound', 0) - ->setParam('project.{scope}.network.outbound', 0); $queueForDeletes->setProject($project); $queueForDatabase->setProject($project); - $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage)); - $dbForProject->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage)); + $dbForProject + ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject)) + ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject)) + ; $useCache = $route->getLabel('cache', false); @@ -362,14 +409,14 @@ App::shutdown() ->inject('user') ->inject('queueForEvents') ->inject('queueForAudits') - ->inject('usage') + ->inject('queueForUsage') ->inject('queueForDeletes') ->inject('queueForDatabase') ->inject('dbForProject') ->inject('queueForFunctions') ->inject('mode') ->inject('dbForConsole') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Stats $usage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) { $responsePayload = $response->getPayload(); @@ -522,35 +569,24 @@ App::shutdown() } } - if ( - App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled' - && $project->getId() - && !empty($route->getLabel('sdk.namespace', null)) - ) { // Don't calculate console usage on admin mode - $metric = $route->getLabel('usage.metric', ''); - $usageParams = $route->getLabel('usage.params', []); - if (!empty($metric)) { - $usage->setParam($metric, 1); - foreach ($usageParams as $param) { - $param = $parseLabel($param, $responsePayload, $requestParams, $user); - $parts = explode(':', $param); - if (count($parts) != 2) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Usage params not properly set'); - } - $usage->setParam($parts[0], $parts[1]); + + if ($project->getId() !== 'console') { + if ($mode !== APP_MODE_ADMIN) { + $fileSize = 0; + $file = $request->getFiles('file'); + if (!empty($file)) { + $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; } + + $queueForUsage + ->addMetric(METRIC_NETWORK_REQUESTS, 1) + ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) + ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); } - $fileSize = 0; - $file = $request->getFiles('file'); - if (!empty($file)) { - $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; - } - - $usage - ->setParam('project.{scope}.network.inbound', $request->getSize() + $fileSize) - ->setParam('project.{scope}.network.outbound', $response->getSize()) - ->submit(); + $queueForUsage + ->setProject($project) + ->trigger(); } }); diff --git a/app/init.php b/app/init.php index 86e4790a30..0d12c57be4 100644 --- a/app/init.php +++ b/app/init.php @@ -19,6 +19,7 @@ ini_set('default_socket_timeout', -1); error_reporting(E_ALL); use Appwrite\Event\Migration; +use Appwrite\Event\Usage; use Appwrite\Extend\Exception; use Appwrite\Auth\Auth; use Appwrite\Event\Audit; @@ -32,7 +33,6 @@ use Appwrite\Network\Validator\Email; use Appwrite\Network\Validator\Origin; use Appwrite\OpenSSL\OpenSSL; use Appwrite\URL\URL as AppwriteURL; -use Appwrite\Usage\Stats; use Utopia\App; use Utopia\Logger\Logger; use Utopia\Cache\Adapter\Redis as RedisCache; @@ -242,6 +242,7 @@ Config::load('platforms', __DIR__ . '/config/platforms.php'); Config::load('collections', __DIR__ . '/config/collections.php'); Config::load('runtimes', __DIR__ . '/config/runtimes.php'); Config::load('runtimes-v2', __DIR__ . '/config/runtimes-v2.php'); +Config::load('usage', __DIR__ . '/config/usage.php'); Config::load('roles', __DIR__ . '/config/roles.php'); // User roles and scopes Config::load('scopes', __DIR__ . '/config/scopes.php'); // User roles and scopes Config::load('services', __DIR__ . '/config/services.php'); // List of services @@ -767,31 +768,6 @@ $register->set('pools', function () { return $group; }); -$register->set('influxdb', function () { - - // Register DB connection - $host = App::getEnv('_APP_INFLUXDB_HOST', ''); - $port = App::getEnv('_APP_INFLUXDB_PORT', ''); - - if (empty($host) || empty($port)) { - return; - } - $driver = new InfluxDB\Driver\Curl("http://{$host}:{$port}"); - $client = new InfluxDB\Client($host, $port, '', '', false, false, 5); - $client->setDriver($driver); - - return $client; -}); -$register->set('statsd', function () { - // Register DB connection - $host = App::getEnv('_APP_STATSD_HOST', 'telegraf'); - $port = App::getEnv('_APP_STATSD_PORT', 8125); - - $connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port); - $statsd = new \Domnikl\Statsd\Client($connection); - - return $statsd; -}); $register->set('smtp', function () { $mail = new PHPMailer(true); @@ -910,15 +886,15 @@ App::setResource('queueForAudits', function (Connection $queue) { App::setResource('queueForFunctions', function (Connection $queue) { return new Func($queue); }, ['queue']); +App::setResource('queueForUsage', function (Connection $queue) { + return new Usage($queue); +}, ['queue']); App::setResource('queueForCertificates', function (Connection $queue) { return new Certificate($queue); }, ['queue']); App::setResource('queueForMigrations', function (Connection $queue) { return new Migration($queue); }, ['queue']); -App::setResource('usage', function ($register) { - return new Stats($register->get('statsd')); -}, ['register']); App::setResource('clients', function ($request, $console, $project) { $console->setAttribute('platforms', [ // Always allow current host '$collection' => ID::custom('platforms'), diff --git a/app/worker.php b/app/worker.php index 8aa52ab931..879c025849 100644 --- a/app/worker.php +++ b/app/worker.php @@ -10,11 +10,10 @@ use Appwrite\Event\Database as EventDatabase; use Appwrite\Event\Delete; use Appwrite\Event\Func; use Appwrite\Event\Mail; -use Appwrite\Event\Messaging; use Appwrite\Event\Migration; use Appwrite\Event\Phone; +use Appwrite\Event\Usage; use Appwrite\Platform\Appwrite; -use Appwrite\Usage\Stats; use Swoole\Runtime; use Utopia\App; use Utopia\Cache\Adapter\Sharding; @@ -120,9 +119,9 @@ Server::setResource('cache', function (Registry $register) { return new Cache(new Sharding($adapters)); }, ['register']); Server::setResource('log', fn() => new Log()); -Server::setResource('usage', function ($register) { - return new Stats($register->get('statsd')); -}, ['register']); +Server::setResource('queueForUsage', function (Connection $queue) { + return new Usage($queue); +}, ['queue']); Server::setResource('queue', function (Group $pools) { return $pools->get('queue')->pop()->getResource(); }, ['pools']); @@ -273,9 +272,13 @@ $worker Console::error('[Error] Line: ' . $error->getLine()); }); - $worker->workerStart() - ->action(function () use ($workerName) { - Console::info("Worker $workerName started"); - }); +try { + $workerStart = $worker->getWorkerStart(); +} catch (\Throwable $error) { + $worker->workerStart() + ->action(function () use ($workerName) { + Console::info("Worker $workerName started"); + }); +} - $worker->start(); +$worker->start(); diff --git a/composer.json b/composer.json index ef10f99a17..64bcd52ee7 100644 --- a/composer.json +++ b/composer.json @@ -62,7 +62,7 @@ "utopia-php/platform": "0.5.*", "utopia-php/pools": "0.4.*", "utopia-php/preloader": "0.2.*", - "utopia-php/queue": "0.5.*", + "utopia-php/queue": "dev-feat-get-worker-start as 0.5.3", "utopia-php/registry": "0.5.*", "utopia-php/storage": "0.18.*", "utopia-php/swoole": "0.5.*", diff --git a/docker-compose.yml b/docker-compose.yml index 42091e5e46..6b1cc2d354 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -127,8 +127,6 @@ services: - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD - _APP_USAGE_STATS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - _APP_STORAGE_LIMIT - _APP_STORAGE_PREVIEW_LIMIT - _APP_STORAGE_ANTIVIRUS @@ -165,8 +163,6 @@ services: - _APP_EXECUTOR_HOST - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - - _APP_STATSD_HOST - - _APP_STATSD_PORT - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE @@ -648,19 +644,21 @@ services: - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY - _APP_MAINTENANCE_RETENTION_SCHEDULES - appwrite-usage: - entrypoint: usage + appwrite-worker-usage: + entrypoint: worker-usage <<: *x-logging - container_name: appwrite-usage + container_name: appwrite-worker-usage image: appwrite-dev networks: - appwrite volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - - ./dev:/usr/local/dev + - ./vendor/utopia-php/platform:/usr/src/code/vendor/utopia-php/platform + - ./vendor/utopia-php/framework:/usr/src/code/vendor/utopia-php/framework + - ./vendor/utopia-php/queue:/usr/src/code/vendor/utopia-php/queue depends_on: - - influxdb + - redis - mariadb environment: - _APP_ENV @@ -671,9 +669,6 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_AGGREGATION_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER @@ -843,26 +838,6 @@ services: # - appwrite # volumes: # - appwrite-uploads:/storage/uploads - - influxdb: - image: appwrite/influxdb:1.5.0 - container_name: appwrite-influxdb - <<: *x-logging - networks: - - appwrite - volumes: - - appwrite-influxdb:/var/lib/influxdb:rw - - telegraf: - image: appwrite/telegraf:1.4.0 - container_name: appwrite-telegraf - <<: *x-logging - networks: - - appwrite - environment: - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - # Dev Tools Start ------------------------------------------------------------------------------------------ # # The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack @@ -872,7 +847,6 @@ services: # MailCatcher - An SMTP server. Catches all system emails and displays them in a nice UI. # RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks # RedisCommander - A nice UI for exploring Redis data - # Chronograf - A nice UI for exploring InfluxDB data # Webgrind - A nice UI for exploring and debugging code-level stuff maildev: # used mainly for dev tests @@ -912,26 +886,6 @@ services: # - REDIS_HOSTS=redis # ports: # - "8081:8081" - - # chronograf: - # image: chronograf:1.6 - # container_name: appwrite-chronograf - # restart: unless-stopped - # networks: - # - appwrite - # volumes: - # - appwrite-chronograf:/var/lib/chronograf - # ports: - # - "8888:8888" - # environment: - # - INFLUXDB_URL=http://influxdb:8086 - # - KAPACITOR_URL=http://kapacitor:9092 - # - AUTH_DURATION=48h - # - TOKEN_SECRET=duperduper5674829!jwt - # - GH_CLIENT_ID=d86f7145a41eacfc52cc - # - GH_CLIENT_SECRET=9e0081062367a2134e7f2ea95ba1a32d08b6c8ab - # - GH_ORGS=appwrite - # webgrind: # image: 'jokkedk/webgrind:latest' # volumes: @@ -969,6 +923,4 @@ volumes: appwrite-certificates: appwrite-functions: appwrite-builds: - appwrite-influxdb: appwrite-config: - # appwrite-chronograf: diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 00779084d4..f597f36028 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -12,7 +12,6 @@ use Appwrite\Platform\Tasks\SDKs; use Appwrite\Platform\Tasks\Specs; use Appwrite\Platform\Tasks\SSL; use Appwrite\Platform\Tasks\Hamster; -use Appwrite\Platform\Tasks\Usage; use Appwrite\Platform\Tasks\Vars; use Appwrite\Platform\Tasks\Version; use Appwrite\Platform\Tasks\VolumeSync; @@ -26,7 +25,6 @@ class Tasks extends Service $this->type = self::TYPE_CLI; $this ->addAction(Version::getName(), new Version()) - ->addAction(Usage::getName(), new Usage()) ->addAction(Vars::getName(), new Vars()) ->addAction(SSL::getName(), new SSL()) ->addAction(Hamster::getName(), new Hamster()) diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index 07fc25434e..67ecf3dd55 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -12,6 +12,8 @@ use Appwrite\Platform\Workers\Databases; use Appwrite\Platform\Workers\Functions; use Appwrite\Platform\Workers\Builds; use Appwrite\Platform\Workers\Deletes; +use Appwrite\Platform\Workers\Usage; +use Appwrite\Platform\Workers\UsageHook; use Appwrite\Platform\Workers\Migrations; class Workers extends Service @@ -29,6 +31,8 @@ class Workers extends Service ->addAction(Functions::getName(), new Functions()) ->addAction(Builds::getName(), new Builds()) ->addAction(Deletes::getName(), new Deletes()) + ->addAction(UsageHook::getName(), new UsageHook()) + ->addAction(Usage::getName(), new Usage()) ->addAction(Migrations::getName(), new Migrations()) ; diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index f1ef0df602..120a514fc4 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -6,7 +6,6 @@ use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Usage; use Appwrite\Messaging\Adapter\Realtime; -use Appwrite\Usage\Stats; use Appwrite\Utopia\Response\Model\Deployment; use Appwrite\Vcs\Comment; use Exception; @@ -48,11 +47,11 @@ class Builds extends Action ->inject('dbForConsole') ->inject('queueForEvents') ->inject('queueForFunctions') - ->inject('usage') + ->inject('queueForUsage') ->inject('cache') ->inject('dbForProject') ->inject('getFunctionsDevice') - ->callback(fn($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Stats $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $getFunctionsDevice)); + ->callback(fn($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $queueForUsage, $cache, $dbForProject, $getFunctionsDevice)); } /** @@ -60,14 +59,14 @@ class Builds extends Action * @param Database $dbForConsole * @param Event $queueForEvents * @param Func $queueForFunctions - * @param Stats $usage + * @param Usage $queueForUsage * @param Cache $cache * @param Database $dbForProject * @param callable $getFunctionsDevice * @return void * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Stats $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice): void + public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice): void { $payload = $message->getPayload() ?? []; @@ -86,7 +85,7 @@ class Builds extends Action case BUILD_TYPE_RETRY: Console::info('Creating build for deployment: ' . $deployment->getId()); $github = new GitHub($cache); - $this->buildDeployment($getFunctionsDevice, $queueForFunctions, $queueForEvents, $usage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template); + $this->buildDeployment($getFunctionsDevice, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template); break; default: @@ -98,7 +97,7 @@ class Builds extends Action * @param callable $getFunctionsDevice * @param Func $queueForFunctions * @param Event $queueForEvents - * @param Stats $usage + * @param Usage $queueForUsage * @param Database $dbForConsole * @param Database $dbForProject * @param GitHub $github @@ -110,7 +109,7 @@ class Builds extends Action * @throws \Utopia\Database\Exception * @throws Exception */ - protected function buildDeployment(callable $getFunctionsDevice, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template): void + protected function buildDeployment(callable $getFunctionsDevice, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template): void { $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); @@ -529,19 +528,16 @@ class Builds extends Action roles: $target['roles'] ); - /** Update usage stats */ - if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { - $usage - ->setParam('projectInternalId', $project->getInternalId()) - ->setParam('projectId', $project->getId()) - ->setParam('functionId', $function->getId()) - ->setParam('builds.{scope}.compute', 1) - ->setParam('buildStatus', $build->getAttribute('status', '')) - ->setParam('buildTime', $build->getAttribute('duration')) - ->setParam('networkRequestSize', 0) - ->setParam('networkResponseSize', 0) - ->submit(); - } + /** Trigger usage queue */ + $queueForUsage + ->setProject($project) + ->addMetric(METRIC_BUILDS, 1) // per project + ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) + ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0)) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000) + ->trigger(); } } diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 2130499257..c4a40a711f 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -386,6 +386,7 @@ class Deletes extends Action */ private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, Document $document): void { + $projects = $dbForConsole->find('projects', [ Query::equal('teamInternalId', [$document->getInternalId()]) ]); diff --git a/src/Appwrite/Platform/Workers/Functions.php b/src/Appwrite/Platform/Workers/Functions.php index 7a6cfe115f..247286e47e 100644 --- a/src/Appwrite/Platform/Workers/Functions.php +++ b/src/Appwrite/Platform/Workers/Functions.php @@ -2,9 +2,9 @@ namespace Appwrite\Platform\Workers; -use Appwrite\Usage\Stats; use Appwrite\Event\Event; use Appwrite\Event\Func; +use Appwrite\Event\Usage; use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Utopia\Response\Model\Execution; use Exception; @@ -43,9 +43,9 @@ class Functions extends Action ->inject('dbForProject') ->inject('queueForFunctions') ->inject('queueForEvents') - ->inject('usage') + ->inject('queueForUsage') ->inject('log') - ->callback(fn(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $usage, $log)); + ->callback(fn(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForUsage, $log)); } /** @@ -53,7 +53,7 @@ class Functions extends Action * @param Database $dbForProject * @param Func $queueForFunctions * @param Event $queueForEvents - * @param Stats $usage + * @param Usage $queueForUsage * @param Log $log * @return void * @throws Authorization @@ -61,7 +61,7 @@ class Functions extends Action * @throws \Utopia\Database\Exception * @throws Conflict */ - public function action(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Log $log): void + public function action(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log): void { $payload = $message->getPayload() ?? []; @@ -117,7 +117,7 @@ class Functions extends Action log: $log, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, - usage: $usage, + queueForUsage: $queueForUsage, queueForEvents: $queueForEvents, project: $project, function: $function, @@ -153,7 +153,7 @@ class Functions extends Action log: $log, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, - usage: $usage, + queueForUsage: $queueForUsage, queueForEvents: $queueForEvents, project: $project, function: $function, @@ -174,7 +174,7 @@ class Functions extends Action log: $log, dbForProject: $dbForProject, queueForFunctions: $queueForFunctions, - usage: $usage, + queueForUsage: $queueForUsage, queueForEvents: $queueForEvents, project: $project, function: $function, @@ -197,7 +197,7 @@ class Functions extends Action * @param Log $log * @param Database $dbForProject * @param Func $queueForFunctions - * @param Stats $usage + * @param Usage $queueForUsage * @param Event $queueForEvents * @param Document $project * @param Document $function @@ -221,7 +221,7 @@ class Functions extends Action Log $log, Database $dbForProject, Func $queueForFunctions, - stats $usage, + Usage $queueForUsage, Event $queueForEvents, Document $project, Document $function, @@ -472,18 +472,14 @@ class Functions extends Action throw new Exception($error, $errorCode); } - /** Update usage stats */ - if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') { - $usage - ->setParam('projectId', $project->getId()) - ->setParam('projectInternalId', $project->getInternalId()) - ->setParam('functionId', $function->getId()) // TODO: We should use functionInternalId in usage stats - ->setParam('executions.{scope}.compute', 1) - ->setParam('executionStatus', $execution->getAttribute('status', '')) - ->setParam('executionTime', $execution->getAttribute('duration')) - ->setParam('networkRequestSize', 0) - ->setParam('networkResponseSize', 0) - ->submit(); - } + /** Trigger usage queue */ + $queueForUsage + ->setProject($project) + ->addMetric(METRIC_EXECUTIONS, 1) + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1) + ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project + ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) + ->trigger() + ; } } diff --git a/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php b/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php index 91a0d089ae..886d703241 100644 --- a/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php +++ b/tests/e2e/Services/Databases/DatabasesConsoleClientTest.php @@ -193,6 +193,44 @@ class DatabasesConsoleClientTest extends Scope $this->assertEquals($response['body'], ""); } + /** + * @depends testCreateCollection + */ + public function testGetDatabaseUsage(array $data) + { + $databaseId = $data['databaseId']; + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(count($response['body']), 3); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['documentsTotal']); + $this->assertIsArray($response['body']['collectionsTotal']); + } + + /** * @depends testCreateCollection */ @@ -230,15 +268,10 @@ class DatabasesConsoleClientTest extends Scope ], $this->getHeaders()), [ 'range' => '24h' ]); - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(count($response['body']), 6); + $this->assertEquals(count($response['body']), 2); $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['documentsCount']); - $this->assertIsArray($response['body']['documentsCreate']); - $this->assertIsArray($response['body']['documentsRead']); - $this->assertIsArray($response['body']['documentsUpdate']); - $this->assertIsArray($response['body']['documentsDelete']); + $this->assertIsArray($response['body']['documentsTotal']); } /** diff --git a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php index b458a4316b..3dad6cb386 100644 --- a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php @@ -92,16 +92,15 @@ class FunctionsConsoleClientTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 200); - $this->assertEquals(count($response['body']), 9); + $this->assertEquals(count($response['body']), 8); $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['executionsTotal']); - $this->assertIsArray($response['body']['executionsFailure']); - $this->assertIsArray($response['body']['executionsSuccess']); - $this->assertIsArray($response['body']['executionsTime']); + $this->assertIsArray($response['body']['deploymentsTotal']); + $this->assertIsArray($response['body']['deploymentsStorage']); $this->assertIsArray($response['body']['buildsTotal']); - $this->assertIsArray($response['body']['buildsFailure']); - $this->assertIsArray($response['body']['buildsSuccess']); + $this->assertIsArray($response['body']['buildsStorage']); $this->assertIsArray($response['body']['buildsTime']); + $this->assertIsArray($response['body']['executionsTotal']); + $this->assertIsArray($response['body']['executionsTime']); } /** diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index ff3f8e8e94..826c90cef8 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -445,7 +445,7 @@ class ProjectsConsoleClientTest extends Scope /** * Test for SUCCESS */ - $response = $this->client->call(Client::METHOD_GET, '/projects/' . $id . '/usage', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/project/usage', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders())); @@ -454,14 +454,14 @@ class ProjectsConsoleClientTest extends Scope $this->assertEquals(count($response['body']), 9); $this->assertNotEmpty($response['body']); $this->assertEquals('30d', $response['body']['range']); - $this->assertIsArray($response['body']['requests']); + $this->assertIsArray($response['body']['requestsTotal']); $this->assertIsArray($response['body']['network']); - $this->assertIsArray($response['body']['executions']); - $this->assertIsArray($response['body']['documents']); - $this->assertIsArray($response['body']['databases']); - $this->assertIsArray($response['body']['buckets']); - $this->assertIsArray($response['body']['users']); - $this->assertIsArray($response['body']['storage']); + $this->assertIsArray($response['body']['executionsTotal']); + $this->assertIsArray($response['body']['documentsTotal']); + $this->assertIsArray($response['body']['databasesTotal']); + $this->assertIsArray($response['body']['bucketsTotal']); + $this->assertIsArray($response['body']['usersTotal']); + $this->assertIsArray($response['body']['filesStorage']); /** * Test for FAILURE diff --git a/tests/e2e/Services/Storage/StorageConsoleClientTest.php b/tests/e2e/Services/Storage/StorageConsoleClientTest.php index 8fda8e0464..aff055f18d 100644 --- a/tests/e2e/Services/Storage/StorageConsoleClientTest.php +++ b/tests/e2e/Services/Storage/StorageConsoleClientTest.php @@ -39,10 +39,11 @@ class StorageConsoleClientTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 200); - $this->assertEquals(12, count($response['body'])); + $this->assertEquals(4, count($response['body'])); $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['storage']); - $this->assertIsArray($response['body']['filesCount']); + $this->assertIsArray($response['body']['bucketsTotal']); + $this->assertIsArray($response['body']['filesTotal']); + $this->assertIsArray($response['body']['filesStorage']); } public function testGetStorageBucketUsage() @@ -94,13 +95,9 @@ class StorageConsoleClientTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 200); - $this->assertEquals(count($response['body']), 7); + $this->assertEquals(count($response['body']), 3); $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['filesCount']); - $this->assertIsArray($response['body']['filesCreate']); - $this->assertIsArray($response['body']['filesRead']); - $this->assertIsArray($response['body']['filesUpdate']); - $this->assertIsArray($response['body']['filesDelete']); + $this->assertIsArray($response['body']['filesTotal']); $this->assertIsArray($response['body']['filesStorage']); } } diff --git a/tests/e2e/Services/Users/UsersConsoleClientTest.php b/tests/e2e/Services/Users/UsersConsoleClientTest.php index 9cc23e825a..8943bfab63 100644 --- a/tests/e2e/Services/Users/UsersConsoleClientTest.php +++ b/tests/e2e/Services/Users/UsersConsoleClientTest.php @@ -23,17 +23,6 @@ class UsersConsoleClientTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'] ], $this->getHeaders()), [ 'range' => '32h', - 'provider' => 'email' - ]); - - $this->assertEquals($response['headers']['status-code'], 400); - - $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'] - ], $this->getHeaders()), [ - 'range' => '24h', - 'provider' => 'some-random-provider' ]); $this->assertEquals($response['headers']['status-code'], 400); @@ -46,38 +35,12 @@ class UsersConsoleClientTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'] ], $this->getHeaders()), [ 'range' => '24h', - 'provider' => 'email' ]); $this->assertEquals($response['headers']['status-code'], 200); - $this->assertEquals(count($response['body']), 9); + $this->assertEquals(count($response['body']), 3); $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['usersCount']); - $this->assertIsArray($response['body']['usersCreate']); - $this->assertIsArray($response['body']['usersRead']); - $this->assertIsArray($response['body']['usersUpdate']); - $this->assertIsArray($response['body']['usersDelete']); - $this->assertIsArray($response['body']['sessionsCreate']); - $this->assertIsArray($response['body']['sessionsProviderCreate']); - $this->assertIsArray($response['body']['sessionsDelete']); - - $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'] - ], $this->getHeaders()), [ - 'range' => '24h' - ]); - - $this->assertEquals($response['headers']['status-code'], 200); - $this->assertEquals(count($response['body']), 9); - $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['usersCount']); - $this->assertIsArray($response['body']['usersCreate']); - $this->assertIsArray($response['body']['usersRead']); - $this->assertIsArray($response['body']['usersUpdate']); - $this->assertIsArray($response['body']['usersDelete']); - $this->assertIsArray($response['body']['sessionsCreate']); - $this->assertIsArray($response['body']['sessionsProviderCreate']); - $this->assertIsArray($response['body']['sessionsDelete']); + $this->assertIsArray($response['body']['usersTotal']); + $this->assertIsArray($response['body']['sessionsTotal']); } } diff --git a/tests/unit/Usage/StatsTest.php b/tests/unit/Usage/StatsTest.php index f021841396..794caebdf5 100644 --- a/tests/unit/Usage/StatsTest.php +++ b/tests/unit/Usage/StatsTest.php @@ -2,70 +2,53 @@ namespace Tests\Unit\Usage; -use Appwrite\Usage\Stats; +use Appwrite\URL\URL as AppwriteURL; use PHPUnit\Framework\TestCase; use Utopia\App; +use Utopia\DSN\DSN; +use Utopia\Queue; +use Utopia\Queue\Client; +use Utopia\Queue\Connection; class StatsTest extends TestCase { - /** - * @var Stats - */ - protected $object = null; + protected ?Connection $connection = null; + protected ?Client $client = null; + + protected const QUEUE_NAME = 'usage-test-q'; public function setUp(): void { - $host = App::getEnv('_APP_STATSD_HOST', 'telegraf'); - $port = App::getEnv('_APP_STATSD_PORT', 8125); + $env = App::getEnv('_APP_CONNECTIONS_QUEUE', AppwriteURL::unparse([ + 'scheme' => 'redis', + 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), + 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), + 'user' => App::getEnv('_APP_REDIS_USER', ''), + 'pass' => App::getEnv('_APP_REDIS_PASS', ''), + ])); - $connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port); - $statsd = new \Domnikl\Statsd\Client($connection); - - $this->object = new Stats($statsd); + $dsn = explode('=', $env); + $dsn = count($dsn) > 1 ? $dsn[1] : $dsn[0]; + $dsn = new DSN($dsn); + $this->connection = new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()); + $this->client = new Client(self::QUEUE_NAME, $this->connection); } public function tearDown(): void { } - public function testNamespace(): void + public function testSamePayload(): void { - $this->object->setNamespace('appwritetest.usage'); - $this->assertEquals('appwritetest.usage', $this->object->getNamespace()); - } + $inToQueue = [ + 'key_1' => 'value_1', + 'key_2' => 'value_2', + ]; - public function testParams(): void - { - $this->object - ->setParam('projectId', 'appwrite_test') - ->setParam('projectInternalId', 1) - ->setParam('networkRequestSize', 100) - ; - - $this->assertEquals('appwrite_test', $this->object->getParam('projectId')); - $this->assertEquals(1, $this->object->getParam('projectInternalId')); - $this->assertEquals(100, $this->object->getParam('networkRequestSize')); - - $this->object->submit(); - - $this->assertEquals(null, $this->object->getParam('projectId')); - $this->assertEquals(null, $this->object->getParam('networkRequestSize')); - } - - public function testReset(): void - { - $this->object - ->setParam('projectId', 'appwrite_test') - ->setParam('networkRequestSize', 100) - ; - - $this->assertEquals('appwrite_test', $this->object->getParam('projectId')); - $this->assertEquals(100, $this->object->getParam('networkRequestSize')); - - $this->object->reset(); - - $this->assertEquals(null, $this->object->getParam('projectId')); - $this->assertEquals(null, $this->object->getParam('networkRequestSize')); - $this->assertEquals('appwrite.usage', $this->object->getNamespace()); + $result = $this->client->enqueue($inToQueue); + $this->assertTrue($result); + $outFromQueue = $this->connection->leftPopArray('utopia-queue.queue.' . self::QUEUE_NAME, 0)['payload']; + $this->assertNotEmpty($outFromQueue); + $this->assertSame($inToQueue, $outFromQueue); } }