diff --git a/app/config/variables.php b/app/config/variables.php index b39eb9c41d..ce04638d44 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -317,54 +317,6 @@ return [ ], ], ], - [ - 'category' => 'InfluxDB', - 'description' => 'Appwrite uses an InfluxDB server for managing time-series data and server stats. The InfluxDB env vars are used to allow Appwrite server to connect to the InfluxDB container.', - 'variables' => [ - [ - 'name' => '_APP_INFLUXDB_HOST', - 'description' => 'InfluxDB server host name address. Default value is: \'influxdb\'.', - 'introduction' => '', - 'default' => 'influxdb', - 'required' => false, - 'question' => '', - 'filter' => '' - ], - [ - 'name' => '_APP_INFLUXDB_PORT', - 'description' => 'InfluxDB server TCP port. Default value is: \'8086\'.', - 'introduction' => '', - 'default' => '8086', - 'required' => false, - 'question' => '', - 'filter' => '' - ], - ], - ], - [ - 'category' => 'StatsD', - 'description' => 'Appwrite uses a StatsD server for aggregating and sending stats data over a fast UDP connection. The StatsD env vars are used to allow Appwrite server to connect to the StatsD container.', - 'variables' => [ - [ - 'name' => '_APP_STATSD_HOST', - 'description' => 'StatsD server host name address. Default value is: \'telegraf\'.', - 'introduction' => '', - 'default' => 'telegraf', - 'required' => false, - 'question' => '', - 'filter' => '' - ], - [ - 'name' => '_APP_STATSD_PORT', - 'description' => 'StatsD server TCP port. Default value is: \'8125\'.', - 'introduction' => '', - 'default' => '8125', - 'required' => false, - 'question' => '', - 'filter' => '' - ], - ], - ], [ 'category' => 'SMTP', 'description' => "Appwrite is using an SMTP server for emailing your projects users and server admins. The SMTP env vars are used to allow Appwrite server to connect to the SMTP container.\n\nIf running in production, it might be easier to use a 3rd party SMTP server as it might be a little more difficult to set up a production SMTP server that will not send all your emails into your user\'s SPAM folder.", diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 0428c39d7f..f364ac89db 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -147,7 +147,6 @@ App::post('/v1/users/bcrypt') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createBcryptUser') @@ -178,7 +177,6 @@ App::post('/v1/users/md5') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createMD5User') @@ -209,7 +207,6 @@ App::post('/v1/users/argon2') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createArgon2User') @@ -240,7 +237,6 @@ App::post('/v1/users/sha') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createSHAUser') @@ -278,7 +274,6 @@ App::post('/v1/users/phpass') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createPHPassUser') @@ -309,7 +304,6 @@ App::post('/v1/users/scrypt') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createScryptUser') @@ -353,7 +347,6 @@ App::post('/v1/users/scrypt-modified') ->label('scope', 'users.write') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.create') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'createScryptModifiedUser') @@ -384,7 +377,6 @@ App::get('/v1/users') ->desc('List Users') ->groups(['api', 'users']) ->label('scope', 'users.read') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'list') @@ -433,7 +425,6 @@ App::get('/v1/users/:userId') ->desc('Get User') ->groups(['api', 'users']) ->label('scope', 'users.read') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'get') @@ -459,7 +450,6 @@ App::get('/v1/users/:userId/prefs') ->desc('Get User Preferences') ->groups(['api', 'users']) ->label('scope', 'users.read') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'getPrefs') @@ -487,7 +477,6 @@ App::get('/v1/users/:userId/sessions') ->desc('List User Sessions') ->groups(['api', 'users']) ->label('scope', 'users.read') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'listSessions') @@ -529,7 +518,6 @@ App::get('/v1/users/:userId/memberships') ->desc('List User Memberships') ->groups(['api', 'users']) ->label('scope', 'users.read') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'listMemberships') @@ -569,7 +557,6 @@ App::get('/v1/users/:userId/logs') ->desc('List User Logs') ->groups(['api', 'users']) ->label('scope', 'users.read') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'listLogs') @@ -651,7 +638,6 @@ App::get('/v1/users/identities') ->desc('List Identities') ->groups(['api', 'users']) ->label('scope', 'users.read') - ->label('usage.metric', 'users.{scope}.requests.read') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'listIdentities') @@ -704,7 +690,6 @@ App::patch('/v1/users/:userId/status') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updateStatus') @@ -740,7 +725,6 @@ App::put('/v1/users/:userId/labels') ->label('scope', 'users.write') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updateLabels') @@ -778,7 +762,6 @@ App::patch('/v1/users/:userId/verification/phone') ->label('scope', 'users.write') ->label('audits.event', 'verification.update') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updatePhoneVerification') @@ -815,7 +798,6 @@ App::patch('/v1/users/:userId/name') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updateName') @@ -853,7 +835,6 @@ App::patch('/v1/users/:userId/password') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updatePassword') @@ -918,7 +899,6 @@ App::patch('/v1/users/:userId/email') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updateEmail') @@ -974,7 +954,6 @@ App::patch('/v1/users/:userId/phone') ->label('scope', 'users.write') ->label('audits.event', 'user.update') ->label('audits.resource', 'user/{response.$id}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updatePhone') @@ -1019,7 +998,6 @@ App::patch('/v1/users/:userId/verification') ->label('audits.event', 'verification.update') ->label('audits.resource', 'user/{request.userId}') ->label('audits.userId', '{request.userId}') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updateEmailVerification') @@ -1052,7 +1030,6 @@ App::patch('/v1/users/:userId/prefs') ->groups(['api', 'users']) ->label('event', 'users.[userId].update.prefs') ->label('scope', 'users.write') - ->label('usage.metric', 'users.{scope}.requests.update') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'updatePrefs') @@ -1088,7 +1065,6 @@ App::delete('/v1/users/:userId/sessions/:sessionId') ->label('scope', 'users.write') ->label('audits.event', 'session.delete') ->label('audits.resource', 'user/{request.userId}') - ->label('usage.metric', 'sessions.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'deleteSession') @@ -1132,7 +1108,6 @@ App::delete('/v1/users/:userId/sessions') ->label('scope', 'users.write') ->label('audits.event', 'session.delete') ->label('audits.resource', 'user/{user.$id}') - ->label('usage.metric', 'sessions.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'deleteSessions') @@ -1175,7 +1150,6 @@ App::delete('/v1/users/:userId') ->label('scope', 'users.write') ->label('audits.event', 'user.delete') ->label('audits.resource', 'user/{request.userId}') - ->label('usage.metric', 'users.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'delete') @@ -1218,7 +1192,6 @@ App::delete('/v1/users/identities/:identityId') ->label('scope', 'users.write') ->label('audits.event', 'identity.delete') ->label('audits.resource', 'identity/{request.$identityId}') - ->label('usage.metric', 'users.{scope}.requests.delete') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'deleteIdentity') @@ -1254,96 +1227,59 @@ App::get('/v1/users/usage') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USAGE_USERS) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) - ->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(fn ($value) => "oauth-" . $value, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true) ->inject('response') ->inject('dbForProject') ->inject('register') - ->action(function (string $range, string $provider, Response $response, Database $dbForProject) { + ->action(function (string $range, Response $response, Database $dbForProject) { - $usage = []; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $periods = [ - '24h' => [ - 'period' => '1h', - 'limit' => 24, - ], - '7d' => [ - 'period' => '1d', - 'limit' => 7, - ], - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ], - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + METRIC_USERS, + METRIC_SESSIONS, + ]; - $metrics = [ - 'users.$all.count.total', - 'users.$all.requests.create', - 'users.$all.requests.read', - 'users.$all.requests.update', - 'users.$all.requests.delete', - 'sessions.$all.requests.create', - 'sessions.$all.requests.delete', - "sessions.$provider.requests.create", - ]; - - $stats = []; - - Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) { - foreach ($metrics as $metric) { - $limit = $periods[$range]['limit']; - $period = $periods[$range]['period']; - - $requestDocs = $dbForProject->find('stats', [ - Query::equal('period', [$period]), - Query::equal('metric', [$metric]), - Query::limit($limit), - Query::orderDesc('time'), - ]); - - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - // backfill metrics with empty values for graphs - $backfill = $limit - \count($requestDocs); - while ($backfill > 0) { - $last = $limit - $backfill - 1; // array index of last added metric - $diff = match ($period) { // convert period to seconds for unix timestamp math - '1h' => 3600, - '1d' => 86400, - }; - $stats[$metric][] = [ - 'value' => 0, - 'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)), - ]; - $backfill--; - } - $stats[$metric] = array_reverse($stats[$metric]); + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('period', [$period]), + Query::equal('metric', [$metric]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric] = []; + foreach ($results as $result) { + $stats[$metric][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; } - }); + } + }); - $usage = new Document([ - 'range' => $range, - 'usersCount' => $stats['users.$all.count.total'] ?? [], - 'usersCreate' => $stats['users.$all.requests.create'] ?? [], - 'usersRead' => $stats['users.$all.requests.read'] ?? [], - 'usersUpdate' => $stats['users.$all.requests.update'] ?? [], - 'usersDelete' => $stats['users.$all.requests.delete'] ?? [], - 'sessionsCreate' => $stats['sessions.$all.requests.create'] ?? [], - 'sessionsProviderCreate' => $stats["sessions.$provider.requests.create"] ?? [], - 'sessionsDelete' => $stats['sessions.$all.requests.delete' ?? []] - ]); + $format = match ($days['period']) { + '1h' => 'Y-m-d\TH:00:00.000P', + '1d' => 'Y-m-d\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric][] = [ + 'value' => $stats[$metric][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; } + } - $response->dynamic($usage, Response::MODEL_USAGE_USERS); + $response->dynamic(new Document([ + 'range' => $range, + 'usersTotal' => $usage[$metrics[0]], + 'sessionsTotal' => $usage[$metrics[1]], + ]), Response::MODEL_USAGE_USERS); }); diff --git a/composer.json b/composer.json index 34793bbcab..ae5e1302ce 100644 --- a/composer.json +++ b/composer.json @@ -75,7 +75,6 @@ "chillerlan/php-qrcode": "4.3.4", "adhocore/jwt": "1.1.2", "webonyx/graphql-php": "14.11.*", - "slickdeals/statsd": "3.1.0", "league/csv": "9.7.1" }, "repositories": [ diff --git a/tests/unit/Usage/StatsTest.php b/tests/unit/Usage/StatsTest.php index 0c9914f4bd..794caebdf5 100644 --- a/tests/unit/Usage/StatsTest.php +++ b/tests/unit/Usage/StatsTest.php @@ -28,7 +28,7 @@ class StatsTest extends TestCase ])); $dsn = explode('=', $env); - $dsn = $dsn[1] ?? ''; + $dsn = count($dsn) > 1 ? $dsn[1] : $dsn[0]; $dsn = new DSN($dsn); $this->connection = new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()); $this->client = new Client(self::QUEUE_NAME, $this->connection);