Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4770 from appwrite/feat-new-usage-stats

Browse Source 
Feat new usage stats
This commit is contained in:
Christy Jacob
2023-02-16 01:28:31 +05:30
committed by GitHub
parent 2ea619533c d1153ee0d7
commit 1832c3c15f
122 changed files with 2425 additions and 5112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
-4
View File
@@ -36,10 +36,6 @@ _APP_CONNECTIONS_STORAGE=local://localhost
_APP_STORAGE_ANTIVIRUS=disabled
_APP_STORAGE_ANTIVIRUS_HOST=clamav
_APP_STORAGE_ANTIVIRUS_PORT=3310
_APP_INFLUXDB_HOST=influxdb
_APP_INFLUXDB_PORT=8086
_APP_STATSD_HOST=telegraf
_APP_STATSD_PORT=8125
_APP_SMTP_HOST=maildev
_APP_SMTP_PORT=1025
_APP_SMTP_SECURE=
.github/workflows/tests.yml
+1 -1
View File
@@ -33,7 +33,7 @@ jobs:
docker compose build appwrite
docker compose up -d
sleep 30
- name: Doctor
run: docker compose exec -T appwrite doctor
CHANGES.md
+34
View File
@@ -6,6 +6,40 @@
- Get default region from environment on project create [#4780](https://github.com/appwrite/appwrite/pull/4780)
- Store build output file size [#4844](https://github.com/appwrite/appwrite/pull/4844)
- Fix max mimetype size [#4814](https://github.com/appwrite/appwrite/pull/4814)
- New usage metrics collection flow [#4770](https://github.com/appwrite/appwrite/pull/4770)
- Deprecated influxdb, telegraf containers and removed all of their occurrences from the code.
- Removed _APP_INFLUXDB_HOST, _APP_INFLUXDB_PORT, _APP_STATSD_HOST, _APP_STATSD_PORT env variables.
- Removed usage labels dependency.
- Dropped type attribute from stats collection.
- Usage metrics are processed via new usage worker.
- Metrics changes:
- Storage
- deprecated
- filesCreate, filesRead, filesUpdate, filesDelete, bucketsCreate, bucketsRead, bucketsUpdate, bucketsDelete.
- renamed
- filesCount to filesTotal, storage to filesStorage, bucketsCount to bucketsTotal.
- Auth
- deprecated
- usersCreate, usersRead, usersUpdate, usersDelete, sessionsCreate sessionsProviderCreate, sessionsDelete.
- renamed
- usersCount to usersTotal.
- added
- sessionsTotal.
- Databases
- deprecated
- databasesCreate, databasesRead, databasesDelete, documentsCreate, documentsRead, documentsUpdate, documentsDelete, collectionsCreate, collectionsRead, collectionsUpdate, collectionsDelete.
- renamed
- databasesCount to databasesTotal, collectionsCount to collectionsTotal, documentsCount to documentsTotal.
- Functions
- deprecated
- executionsFailure, executionsSuccess, buildsFailure, buildsSuccess, executionsFailure, executionsSuccess.
- renamed
- executionsTime to executionsCompute, buildsTime to buildsCompute, documentsCount to documentsTotal.
- added
- functionsTotal, buildsStorage, deploymentsTotal, deploymentsStorage.
- Project
- renamed
- executions to executionsTotal, builds to buildsTotal, requests to requestsTotal, storage to filesStorage, buckets to bucketsTotal, users to usersTotal, documents to documentsTotal, collections to collectionsTotal, databases to databasesTotal.
# Version 1.1.2
## Changes
CONTRIBUTING.md
-2
View File
@@ -235,8 +235,6 @@ Appwrite stack is combined from a variety of open-source technologies and tools.
- Redis - for managing cache and in-memory data (currently, we do not use Redis for persistent data)
- MariaDB - for database storage and queries
- InfluxDB - for managing stats and time-series based data
- Statsd - for sending data over UDP protocol (using Telegraf)
- ClamAV - for validating and scanning storage files
- Imagemagick - for manipulating and managing image media files.
- Webp - for better compression of images on supporting clients
Dockerfile
+2 -6
View File
@@ -208,10 +208,6 @@ ENV _APP_SERVER=swoole \
_APP_DB_USER=root \
_APP_DB_PASS=password \
_APP_DB_SCHEMA=appwrite \
_APP_INFLUXDB_HOST=influxdb \
_APP_INFLUXDB_PORT=8086 \
_APP_STATSD_HOST=telegraf \
_APP_STATSD_PORT=8125 \
_APP_FUNCTIONS_SIZE_LIMIT=30000000 \
_APP_FUNCTIONS_TIMEOUT=900 \
_APP_FUNCTIONS_CPUS=1 \
@@ -306,7 +302,6 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/patch-create-missing-schedules && \
chmod +x /usr/local/bin/maintenance && \
chmod +x /usr/local/bin/volume-sync && \
chmod +x /usr/local/bin/usage && \
chmod +x /usr/local/bin/install && \
chmod +x /usr/local/bin/migrate && \
chmod +x /usr/local/bin/realtime && \
@@ -324,7 +319,8 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/worker-builds && \
chmod +x /usr/local/bin/worker-mails && \
chmod +x /usr/local/bin/worker-messaging && \
chmod +x /usr/local/bin/worker-webhooks
chmod +x /usr/local/bin/worker-webhooks && \
chmod +x /usr/local/bin/worker-usage
# Letsencrypt Permissions
RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/
app/cli.php
-23
View File
@@ -116,29 +116,6 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
return $getProjectDB;
}, ['pools', 'dbForConsole', 'cache']);
CLI::setResource('influxdb', function (Registry $register) {
$client = $register->get('influxdb'); /** @var InfluxDB\Client $client */
$attempts = 0;
$max = 10;
$sleep = 1;
do { // check if telegraf database is ready
try {
$attempts++;
$database = $client->selectDB('telegraf');
if (in_array('telegraf', $client->listDatabases())) {
break; // leave the do-while if successful
}
} catch (\Throwable $th) {
Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('InfluxDB database not ready yet');
}
sleep($sleep);
}
} while ($attempts < $max);
return $database;
}, ['register']);
CLI::setResource('queueForFunctions', function (Group $pools) {
return new Func($pools->get('queue')->pop()->getResource());
app/config/collections.php
+64 -37
View File
@@ -3,7 +3,7 @@
use Appwrite\Auth\Auth;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
$providers = Config::getParam('providers', []);
$auth = Config::getParam('auth', []);
@@ -192,7 +192,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -214,7 +214,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -369,7 +369,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -391,7 +391,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -496,7 +496,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -791,7 +791,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -892,7 +892,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -986,7 +986,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -1091,7 +1091,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -1203,7 +1203,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -1579,7 +1579,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -1673,7 +1673,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -2014,7 +2014,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -2036,7 +2036,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -2240,7 +2240,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -2284,7 +2284,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -2409,7 +2409,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -2442,7 +2442,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -2633,7 +2633,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -2760,7 +2760,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -2782,7 +2782,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
@@ -3208,7 +3208,7 @@ $collections = [
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 8,
'signed' => false,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
@@ -3236,17 +3236,6 @@ $collections = [
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('type'),
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 1,
'signed' => false,
'required' => true,
'default' => 0, // 0 -> count, 1 -> sum
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
@@ -3265,14 +3254,51 @@ $collections = [
],
[
'$id' => ID::custom('_key_metric_period_time'),
'type' => Database::INDEX_KEY,
'type' => Database::INDEX_UNIQUE,
'attributes' => ['metric', 'period', 'time'],
'lengths' => [],
'orders' => [Database::ORDER_DESC],
],
],
],
'statsLogger' => [
'$collection' => ID::custom(Database::METADATA),
'$id' => ID::custom('statsLogger'),
'name' => 'StatsLogger',
'attributes' => [
[
'$id' => ID::custom('time'),
'type' => Database::VAR_DATETIME,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => ID::custom('metrics'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 5012,
'signed' => true,
'required' => false,
'default' => [],
'array' => false,
'filters' => ['json'],
],
],
'indexes' => [
[
'$id' => ID::custom('_key_time'),
'type' => Database::INDEX_KEY,
'attributes' => ['time'],
'lengths' => [],
'orders' => [Database::ORDER_DESC],
],
],
],
'realtime' => [
'$collection' => ID::custom(Database::METADATA),
'$id' => ID::custom('realtime'),
@@ -3322,6 +3348,7 @@ $collections = [
],
]
],
'cache' => [
'$collection' => Database::METADATA,
'$id' => 'cache',
@@ -3642,7 +3669,7 @@ $collections = [
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
app/config/specs/open-api3-latest-client.json
+1 -1
View File
File diff suppressed because one or more lines are too long
app/config/specs/open-api3-latest-console.json
+1 -1
View File
File diff suppressed because one or more lines are too long
app/config/specs/open-api3-latest-server.json
+1 -1
View File
File diff suppressed because one or more lines are too long
app/config/specs/swagger2-latest-client.json
+1 -1
View File
File diff suppressed because one or more lines are too long
app/config/specs/swagger2-latest-console.json
+1 -1
View File
File diff suppressed because one or more lines are too long
app/config/specs/swagger2-latest-server.json
+1 -1
View File
File diff suppressed because one or more lines are too long
app/config/usage.php
+24
View File
@@ -0,0 +1,24 @@
<?php
return [
'24h' => [
'period' => '1h',
'limit' => 24,
'factor' => 3600,
],
'7d' => [
'period' => '1d',
'limit' => 7,
'factor' => 86400,
],
'30d' => [
'period' => '1d',
'limit' => 30,
'factor' => 86400,
],
'90d' => [
'period' => '1d',
'limit' => 90,
'factor' => 86400,
],
];
app/config/variables.php
+1 -49
View File
@@ -143,7 +143,7 @@ return [
],
[
'name' => '_APP_USAGE_STATS',
'description' => 'This variable allows you to disable the collection and displaying of usage stats. This value is set to \'enabled\' by default, to disable the usage stats set the value to \'disabled\'. When disabled, it\'s recommended to turn off the Worker Usage, Influxdb and Telegraf containers for better resource usage.',
'description' => 'This variable allows you to disable the collection and displaying of usage stats. This value is set to \'enabled\' by default, to disable the usage stats set the value to \'disabled\'. When disabled, it\'s recommended to turn off the Worker Usage container for better resource usage.',
'introduction' => '0.7.0',
'default' => 'enabled',
'required' => false,
@@ -344,54 +344,6 @@ return [
// ]
],
],
[
'category' => 'InfluxDB',
'description' => 'Appwrite uses an InfluxDB server for managing time-series data and server stats. The InfluxDB env vars are used to allow Appwrite server to connect to the InfluxDB container.',
'variables' => [
[
'name' => '_APP_INFLUXDB_HOST',
'description' => 'InfluxDB server host name address. Default value is: \'influxdb\'.',
'introduction' => '',
'default' => 'influxdb',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_INFLUXDB_PORT',
'description' => 'InfluxDB server TCP port. Default value is: \'8086\'.',
'introduction' => '',
'default' => '8086',
'required' => false,
'question' => '',
'filter' => ''
],
],
],
[
'category' => 'StatsD',
'description' => 'Appwrite uses a StatsD server for aggregating and sending stats data over a fast UDP connection. The StatsD env vars are used to allow Appwrite server to connect to the StatsD container.',
'variables' => [
[
'name' => '_APP_STATSD_HOST',
'description' => 'StatsD server host name address. Default value is: \'telegraf\'.',
'introduction' => '',
'default' => 'telegraf',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_STATSD_PORT',
'description' => 'StatsD server TCP port. Default value is: \'8125\'.',
'introduction' => '',
'default' => '8125',
'required' => false,
'question' => '',
'filter' => ''
],
],
],
[
'category' => 'SMTP',
'description' => "Appwrite is using an SMTP server for emailing your projects users and server admins. The SMTP env vars are used to allow Appwrite server to connect to the SMTP container.\n\nIf running in production, it might be easier to use a 3rd party SMTP server as it might be a little more difficult to set up a production SMTP server that will not send all your emails into your user\'s SPAM folder.",
app/controllers/api/account.php
+3 -35
View File
@@ -29,10 +29,10 @@ use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\DateTime;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Query;
use Utopia\Database\Role;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\UID;
use Utopia\Locale\Locale;
@@ -53,7 +53,6 @@ App::post('/v1/account/invite')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createWithInviteCode')
@@ -145,7 +144,6 @@ App::post('/v1/account')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'create')
@@ -239,8 +237,6 @@ App::post('/v1/account/sessions/email')
->label('audits.event', 'session.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:email'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createEmailSession')
@@ -467,8 +463,6 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
->label('abuse-limit', 50)
->label('abuse-key', 'ip:{ip}')
->label('docs', false)
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:{request.provider}'])
->param('provider', '', new WhiteList(\array_keys(Config::getParam('providers')), true), 'OAuth2 provider.')
->param('code', '', new Text(2048), 'OAuth2 code.')
->param('state', '', new Text(2048), 'OAuth2 state params.', true)
@@ -838,8 +832,6 @@ App::put('/v1/account/sessions/magic-url')
->label('audits.event', 'session.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:magic-url'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateMagicURLSession')
@@ -1077,8 +1069,6 @@ App::put('/v1/account/sessions/phone')
->groups(['api', 'account'])
->label('scope', 'public')
->label('event', 'users.[userId].sessions.[sessionId].create')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:phone'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePhoneSession')
@@ -1196,8 +1186,6 @@ App::post('/v1/account/sessions/anonymous')
->label('audits.event', 'session.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:anonymous'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createAnonymousSession')
@@ -1372,7 +1360,6 @@ App::get('/v1/account')
->desc('Get Account')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'get')
@@ -1391,7 +1378,6 @@ App::get('/v1/account/prefs')
->desc('Get Account Preferences')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'getPrefs')
@@ -1412,7 +1398,6 @@ App::get('/v1/account/sessions')
->desc('List Sessions')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'listSessions')
@@ -1449,7 +1434,6 @@ App::get('/v1/account/logs')
->desc('List Logs')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'listLogs')
@@ -1510,7 +1494,6 @@ App::get('/v1/account/sessions/:sessionId')
->desc('Get Session')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'getSession')
@@ -1556,7 +1539,6 @@ App::patch('/v1/account/name')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateName')
@@ -1588,7 +1570,6 @@ App::patch('/v1/account/password')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePassword')
@@ -1627,7 +1608,6 @@ App::patch('/v1/account/email')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateEmail')
@@ -1679,7 +1659,6 @@ App::patch('/v1/account/phone')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePhone')
@@ -1727,7 +1706,6 @@ App::patch('/v1/account/prefs')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePrefs')
@@ -1756,7 +1734,6 @@ App::patch('/v1/account/status')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateStatus')
@@ -1791,7 +1768,6 @@ App::delete('/v1/account/sessions/:sessionId')
->label('event', 'users.[userId].sessions.[sessionId].delete')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{user.$id}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'deleteSession')
@@ -1865,7 +1841,6 @@ App::patch('/v1/account/sessions/:sessionId')
->label('audits.event', 'session.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateSession')
@@ -1950,7 +1925,6 @@ App::delete('/v1/account/sessions')
->label('event', 'users.[userId].sessions.[sessionId].delete')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{user.$id}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'deleteSessions')
@@ -2012,7 +1986,6 @@ App::post('/v1/account/recovery')
->label('audits.event', 'recovery.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createRecovery')
@@ -2119,7 +2092,6 @@ App::put('/v1/account/recovery')
->label('audits.event', 'recovery.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateRecovery')
@@ -2187,7 +2159,6 @@ App::post('/v1/account/verification')
->label('event', 'users.[userId].verification.[tokenId].create')
->label('audits.event', 'verification.create')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createVerification')
@@ -2277,7 +2248,6 @@ App::put('/v1/account/verification')
->label('event', 'users.[userId].verification.[tokenId].update')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateVerification')
@@ -2336,7 +2306,6 @@ App::post('/v1/account/verification/phone')
->label('event', 'users.[userId].verification.[tokenId].create')
->label('audits.event', 'verification.create')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createPhoneVerification')
@@ -2421,7 +2390,6 @@ App::put('/v1/account/verification/phone')
->label('event', 'users.[userId].verification.[tokenId].update')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePhoneVerification')
app/controllers/api/databases.php
+149 -329
View File
@@ -4,10 +4,10 @@ use Utopia\App;
use Appwrite\Event\Delete;
use Appwrite\Extend\Exception;
use Utopia\Audit\Audit;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\DatetimeValidator;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
use Utopia\Validator\Boolean;
use Utopia\Validator\FloatValidator;
use Utopia\Validator\Integer;
@@ -18,7 +18,6 @@ use Utopia\Validator\ArrayList;
use Utopia\Validator\JSON;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\DateTime;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\Key;
@@ -154,7 +153,6 @@ App::post('/v1/databases')
->label('scope', 'databases.write')
->label('audits.event', 'database.create')
->label('audits.resource', 'database/{response.$id}')
->label('usage.metric', 'databases.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'create')
@@ -226,7 +224,6 @@ App::get('/v1/databases')
->desc('List Databases')
->groups(['api', 'database'])
->label('scope', 'databases.read')
->label('usage.metric', 'databases.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'list')
@@ -273,7 +270,6 @@ App::get('/v1/databases/:databaseId')
->desc('Get Database')
->groups(['api', 'database'])
->label('scope', 'databases.read')
->label('usage.metric', 'databases.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'get')
@@ -388,7 +384,6 @@ App::put('/v1/databases/:databaseId')
->label('event', 'databases.[databaseId].update')
->label('audits.event', 'database.update')
->label('audits.resource', 'database/{response.$id}')
->label('usage.metric', 'databases.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'update')
@@ -431,7 +426,6 @@ App::delete('/v1/databases/:databaseId')
->label('event', 'databases.[databaseId].delete')
->label('audits.event', 'database.delete')
->label('audits.resource', 'database/{request.databaseId}')
->label('usage.metric', 'databases.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'delete')
@@ -478,8 +472,6 @@ App::post('/v1/databases/:databaseId/collections')
->label('scope', 'collections.write')
->label('audits.event', 'collection.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{response.$id}')
->label('usage.metric', 'collections.{scope}.requests.create')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'createCollection')
@@ -543,8 +535,6 @@ App::get('/v1/databases/:databaseId/collections')
->desc('List Collections')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listCollections')
@@ -599,8 +589,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId')
->desc('Get Collection')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getCollection')
@@ -634,8 +622,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs')
->desc('List Collection Logs')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listCollectionLogs')
@@ -733,8 +719,6 @@ App::put('/v1/databases/:databaseId/collections/:collectionId')
->label('event', 'databases.[databaseId].collections.[collectionId].update')
->label('audits.event', 'collection.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'updateCollection')
@@ -801,8 +785,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId')
->label('event', 'databases.[databaseId].collections.[collectionId].delete')
->label('audits.event', 'collection.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.delete')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'deleteCollection')
@@ -858,8 +840,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/string
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'createStringAttribute')
@@ -908,8 +888,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/email'
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createEmailAttribute')
@@ -952,8 +930,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createEnumAttribute')
@@ -1012,8 +988,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/ip')
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createIpAttribute')
@@ -1056,8 +1030,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/url')
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createUrlAttribute')
@@ -1100,8 +1072,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/intege
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createIntegerAttribute')
@@ -1173,8 +1143,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/float'
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createFloatAttribute')
@@ -1249,8 +1217,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/boolea
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createBooleanAttribute')
@@ -1293,8 +1259,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/dateti
->label('scope', 'collections.write')
->label('audits.event', 'attribute.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.namespace', 'databases')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.method', 'createDatetimeAttribute')
@@ -1335,8 +1299,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes')
->desc('List Attributes')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listAttributes')
@@ -1374,8 +1336,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes/:key')
->desc('Get Attribute')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getAttribute')
@@ -1448,8 +1408,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key
->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete')
->label('audits.event', 'attribute.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'deleteAttribute')
@@ -1536,8 +1494,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes')
->label('scope', 'collections.write')
->label('audits.event', 'index.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'createIndex')
@@ -1683,8 +1639,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes')
->desc('List Indexes')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listIndexes')
@@ -1722,8 +1676,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes/:key')
->desc('Get Index')
->groups(['api', 'database'])
->label('scope', 'collections.read')
->label('usage.metric', 'collections.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getIndex')
@@ -1773,8 +1725,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/indexes/:key')
->label('event', 'databases.[databaseId].collections.[collectionId].indexes.[indexId].delete')
->label('audits.event', 'index.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'collections.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}'])
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'deleteIndex')
@@ -1841,8 +1791,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents')
->label('scope', 'documents.write')
->label('audits.event', 'document.create')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
->label('usage.metric', 'documents.{scope}.requests.create')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -1966,8 +1914,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents')
->desc('List Documents')
->groups(['api', 'database'])
->label('scope', 'documents.read')
->label('usage.metric', 'documents.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listDocuments')
@@ -2063,8 +2009,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
->desc('Get Document')
->groups(['api', 'database'])
->label('scope', 'documents.read')
->label('usage.metric', 'documents.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'getDocument')
@@ -2125,8 +2069,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
->desc('List Document Logs')
->groups(['api', 'database'])
->label('scope', 'documents.read')
->label('usage.metric', 'documents.{scope}.requests.read')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'databases')
->label('sdk.method', 'listDocumentLogs')
@@ -2229,8 +2171,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum
->label('scope', 'documents.write')
->label('audits.event', 'document.update')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}')
->label('usage.metric', 'documents.{scope}.requests.update')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -2363,8 +2303,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].delete')
->label('audits.event', 'document.delete')
->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{request.documentId}')
->label('usage.metric', 'documents.{scope}.requests.delete')
->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -2462,107 +2400,57 @@ App::get('/v1/databases/usage')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_DATABASES,
METRIC_COLLECTIONS,
METRIC_DOCUMENTS,
];
$metrics = [
'databases.$all.count.total',
'documents.$all.count.total',
'collections.$all.count.total',
'databases.$all.requests.create',
'databases.$all.requests.read',
'databases.$all.requests.update',
'databases.$all.requests.delete',
'collections.$all.requests.create',
'collections.$all.requests.read',
'collections.$all.requests.update',
'collections.$all.requests.delete',
'documents.$all.requests.create',
'documents.$all.requests.read',
'documents.$all.requests.update',
'documents.$all.requests.delete'
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
// Added 3'rd level to Index [period, metric, time] because of order by.
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'databasesCount' => $stats['databases.$all.count.total'] ?? [],
'documentsCount' => $stats['documents.$all.count.total'] ?? [],
'collectionsCount' => $stats['collections.$all.count.total'] ?? [],
'documentsCreate' => $stats['documents.$all.requests.create'] ?? [],
'documentsRead' => $stats['documents.$all.requests.read'] ?? [],
'documentsUpdate' => $stats['documents.$all.requests.update'] ?? [],
'documentsDelete' => $stats['documents.$all.requests.delete'] ?? [],
'collectionsCreate' => $stats['collections.$all.requests.create'] ?? [],
'collectionsRead' => $stats['collections.$all.requests.read'] ?? [],
'collectionsUpdate' => $stats['collections.$all.requests.update'] ?? [],
'collectionsDelete' => $stats['collections.$all.requests.delete'] ?? [],
'databasesCreate' => $stats['databases.$all.requests.create'] ?? [],
'databasesRead' => $stats['databases.$all.requests.read'] ?? [],
'databasesUpdate' => $stats['databases.$all.requests.update'] ?? [],
'databasesDelete' => $stats['databases.$all.requests.delete'] ?? [],
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
$response->dynamic($usage, Response::MODEL_USAGE_DATABASES);
}
$response->dynamic(new Document([
'range' => $range,
'databasesTotal' => $usage[$metrics[0]],
'collectionsTotal' => $usage[$metrics[1]],
'documentsTotal' => $usage[$metrics[2]],
]), Response::MODEL_USAGE_DATABASES);
});
App::get('/v1/databases/:databaseId/usage')
@@ -2581,97 +2469,62 @@ App::get('/v1/databases/:databaseId/usage')
->inject('dbForProject')
->action(function (string $databaseId, string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$database = $dbForProject->getDocument('databases', $databaseId);
$metrics = [
'collections.' . $databaseId . '.count.total',
'collections.' . $databaseId . '.requests.create',
'collections.' . $databaseId . '.requests.read',
'collections.' . $databaseId . '.requests.update',
'collections.' . $databaseId . '.requests.delete',
'documents.' . $databaseId . '.count.total',
'documents.' . $databaseId . '.requests.create',
'documents.' . $databaseId . '.requests.read',
'documents.' . $databaseId . '.requests.update',
'documents.' . $databaseId . '.requests.delete'
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
// TODO@kodumbeats explore performance if query is ordered by time ASC
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'collectionsCount' => $stats["collections.{$databaseId}.count.total"] ?? [],
'collectionsCreate' => $stats["collections.{$databaseId}.requests.create"] ?? [],
'collectionsRead' => $stats["collections.{$databaseId}.requests.read"] ?? [],
'collectionsUpdate' => $stats["collections.{$databaseId}.requests.update"] ?? [],
'collectionsDelete' => $stats["collections.{$databaseId}.requests.delete"] ?? [],
'documentsCount' => $stats["documents.{$databaseId}.count.total"] ?? [],
'documentsCreate' => $stats["documents.{$databaseId}.requests.create"] ?? [],
'documentsRead' => $stats["documents.{$databaseId}.requests.read"] ?? [],
'documentsUpdate' => $stats["documents.{$databaseId}.requests.update"] ?? [],
'documentsDelete' => $stats["documents.{$databaseId}.requests.delete"] ?? [],
]);
if ($database->isEmpty()) {
throw new Exception(Exception::DATABASE_NOT_FOUND);
}
$response->dynamic($usage, Response::MODEL_USAGE_DATABASE);
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS),
str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS),
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic(new Document([
'range' => $range,
'collectionsTotal' => $usage[$metrics[0]],
'documentsTotal' => $usage[$metrics[1]],
]), Response::MODEL_USAGE_DATABASE);
});
App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
@@ -2692,93 +2545,60 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
->inject('dbForProject')
->action(function (string $databaseId, string $range, string $collectionId, Response $response, Database $dbForProject) {
$database = $dbForProject->getDocument('databases', $databaseId);
$collectionDocument = $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId);
$collection = $dbForProject->getCollection('database_' . $database->getInternalId() . '_collection_' . $collectionDocument->getInternalId());
$database = $dbForProject->getDocument('databases', $databaseId);
$collectionDocument = $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId);
$collection = $dbForProject->getCollection('database_' . $database->getInternalId() . '_collection_' . $collectionDocument->getInternalId());
if ($collection->isEmpty()) {
throw new Exception(Exception::COLLECTION_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collectionDocument->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS),
];
$metrics = [
"documents.{$databaseId}/{$collectionId}.count.total",
"documents.{$databaseId}/{$collectionId}.requests.create",
"documents.{$databaseId}/{$collectionId}.requests.read",
"documents.{$databaseId}/{$collectionId}.requests.update",
"documents.{$databaseId}/{$collectionId}.requests.delete",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'documentsCount' => $stats["documents.{$databaseId}/{$collectionId}.count.total"] ?? [],
'documentsCreate' => $stats["documents.{$databaseId}/{$collectionId}.requests.create"] ?? [],
'documentsRead' => $stats["documents.{$databaseId}/{$collectionId}.requests.read"] ?? [],
'documentsUpdate' => $stats["documents.{$databaseId}/{$collectionId}.requests.update"] ?? [],
'documentsDelete' => $stats["documents.{$databaseId}/{$collectionId}.requests.delete" ?? []]
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_COLLECTION);
$response->dynamic(new Document([
'range' => $range,
'documentsTotal' => $usage[$metrics[0]],
]), Response::MODEL_USAGE_COLLECTION);
});
app/controllers/api/functions.php
+129 -177
View File
@@ -6,14 +6,14 @@ use Appwrite\Event\Build;
use Appwrite\Event\Delete;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Event\Validator\Event as ValidatorEvent;
use Appwrite\Extend\Exception;
use Appwrite\Utopia\Database\Validator\CustomId;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\UID;
use Appwrite\Usage\Stats;
use Utopia\Storage\Device;
use Utopia\Storage\Validator\File;
use Utopia\Storage\Validator\FileExt;
@@ -85,6 +85,7 @@ App::post('/v1/functions')
'deployment' => '',
'events' => $events,
'schedule' => $schedule,
'scheduleInternalId' => '',
'scheduleUpdatedAt' => DateTime::now(),
'timeout' => $timeout,
'search' => implode(' ', [$functionId, $name, $runtime])
@@ -95,6 +96,7 @@ App::post('/v1/functions')
'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region
'resourceType' => 'function',
'resourceId' => $function->getId(),
'resourceInternalId' => $function->getInternalId(),
'resourceUpdatedAt' => DateTime::now(),
'projectId' => $project->getId(),
'schedule' => $function->getAttribute('schedule'),
@@ -232,92 +234,66 @@ App::get('/v1/functions/:functionId/usage')
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace(['{resourceType}', '{resourceInternalId}'], ['function', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS),
str_replace(['{resourceType}', '{resourceInternalId}'], ['function', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE),
];
$metrics = [
"executions.$functionId.compute.total",
"executions.$functionId.compute.success",
"executions.$functionId.compute.failure",
"executions.$functionId.compute.time",
"builds.$functionId.compute.total",
"builds.$functionId.compute.success",
"builds.$functionId.compute.failure",
"builds.$functionId.compute.time",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'executionsTotal' => $stats["executions.$functionId.compute.total"] ?? [],
'executionsFailure' => $stats["executions.$functionId.compute.failure"] ?? [],
'executionsSuccesse' => $stats["executions.$functionId.compute.success"] ?? [],
'executionsTime' => $stats["executions.$functionId.compute.time"] ?? [],
'buildsTotal' => $stats["builds.$functionId.compute.total"] ?? [],
'buildsFailure' => $stats["builds.$functionId.compute.failure"] ?? [],
'buildsSuccess' => $stats["builds.$functionId.compute.success"] ?? [],
'buildsTime' => $stats["builds.$functionId.compute.time" ?? []]
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_FUNCTION);
$response->dynamic(new Document([
'range' => $range,
'deploymentsTotal' => $usage[$metrics[0]],
'deploymentsStorage' => $usage[$metrics[1]],
'buildsTotal' => $usage[$metrics[2]],
'buildsStorage' => $usage[$metrics[3]],
'buildsTime' => $usage[$metrics[4]],
'executionsTotal' => $usage[$metrics[5]],
'executionsTime' => $usage[$metrics[6]],
]), Response::MODEL_USAGE_FUNCTION);
});
App::get('/v1/functions/usage')
@@ -335,92 +311,67 @@ App::get('/v1/functions/usage')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_FUNCTIONS,
METRIC_DEPLOYMENTS,
METRIC_DEPLOYMENTS_STORAGE,
METRIC_BUILDS,
METRIC_BUILDS_STORAGE,
METRIC_BUILDS_COMPUTE,
METRIC_EXECUTIONS,
METRIC_EXECUTIONS_COMPUTE,
];
$metrics = [
'executions.$all.compute.total',
'executions.$all.compute.failure',
'executions.$all.compute.success',
'executions.$all.compute.time',
'builds.$all.compute.total',
'builds.$all.compute.failure',
'builds.$all.compute.success',
'builds.$all.compute.time',
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'executionsTotal' => $stats[$metrics[0]] ?? [],
'executionsFailure' => $stats[$metrics[1]] ?? [],
'executionsSuccess' => $stats[$metrics[2]] ?? [],
'executionsTime' => $stats[$metrics[3]] ?? [],
'buildsTotal' => $stats[$metrics[4]] ?? [],
'buildsFailure' => $stats[$metrics[5]] ?? [],
'buildsSuccess' => $stats[$metrics[6]] ?? [],
'buildsTime' => $stats[$metrics[7]] ?? [],
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
$response->dynamic($usage, Response::MODEL_USAGE_FUNCTIONS);
}
$response->dynamic(new Document([
'range' => $range,
'functionsTotal' => $usage[$metrics[0]],
'deploymentsTotal' => $usage[$metrics[1]],
'deploymentsStorage' => $usage[$metrics[2]],
'buildsTotal' => $usage[$metrics[3]],
'buildsStorage' => $usage[$metrics[4]],
'buildsTime' => $usage[$metrics[5]],
'executionsTotal' => $usage[$metrics[6]],
'executionsTime' => $usage[$metrics[7]],
]), Response::MODEL_USAGE_FUNCTIONS);
});
App::put('/v1/functions/:functionId')
@@ -698,7 +649,6 @@ App::post('/v1/functions/:functionId/deployments')
// Save to storage
$fileSize ??= $deviceLocal->getFileSize($fileTmpName);
$path = $deviceFunctions->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
$metadata = ['content_type' => $deviceLocal->getFileMimeType($fileTmpName)];
@@ -746,6 +696,7 @@ App::post('/v1/functions/:functionId/deployments')
'resourceInternalId' => $function->getInternalId(),
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'buildInternalId' => '',
'entrypoint' => $entrypoint,
'path' => $path,
'size' => $fileSize,
@@ -777,6 +728,7 @@ App::post('/v1/functions/:functionId/deployments')
'resourceInternalId' => $function->getInternalId(),
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'buildInternalId' => '',
'entrypoint' => $entrypoint,
'path' => $path,
'size' => $fileSize,
@@ -1069,10 +1021,10 @@ App::post('/v1/functions/:functionId/executions')
->inject('dbForProject')
->inject('user')
->inject('events')
->inject('usage')
->inject('mode')
->inject('queueForFunctions')
->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Func $queueForFunctions) {
->inject('queueForUsage')
->action(function (string $functionId, string $data, bool $async, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, string $mode, Func $queueForFunctions, Usage $queueForUsage) {
$function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId));
@@ -1216,6 +1168,13 @@ App::post('/v1/functions/:functionId/executions')
$execution->setAttribute('stdout', $executionResponse['stdout']);
$execution->setAttribute('stderr', $executionResponse['stderr']);
$execution->setAttribute('duration', $executionResponse['duration']);
/**
* Sync execution compute usage from
*/
$queueForUsage
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($executionResponse['duration'] * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($executionResponse['duration'] * 1000))// per function
;
} catch (\Throwable $th) {
$interval = (new \DateTime())->diff(new \DateTime($execution->getCreatedAt()));
$execution
@@ -1228,13 +1187,6 @@ App::post('/v1/functions/:functionId/executions')
Authorization::skip(fn () => $dbForProject->updateDocument('executions', $executionId, $execution));
// TODO revise this later using route label
$usage
->setParam('functionId', $function->getId())
->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', ''))
->setParam('executionTime', $execution->getAttribute('duration')); // ms
$roles = Authorization::getRoles();
$isPrivilegedUser = Auth::isPrivilegedUser($roles);
$isAppUser = Auth::isAppUser($roles);
app/controllers/api/project.php
+61 -79
View File
@@ -2,6 +2,7 @@
use Appwrite\Utopia\Response;
use Utopia\App;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
@@ -23,90 +24,71 @@ App::get('/v1/project/usage')
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$metrics = [
'project.$all.network.requests',
'project.$all.network.bandwidth',
'project.$all.storage.size',
'users.$all.count.total',
'databases.$all.count.total',
'documents.$all.count.total',
'executions.$all.compute.total',
'buckets.$all.count.total'
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_NETWORK_REQUESTS,
METRIC_NETWORK_INBOUND,
METRIC_NETWORK_OUTBOUND,
METRIC_EXECUTIONS,
METRIC_DOCUMENTS,
METRIC_DATABASES,
METRIC_USERS,
METRIC_BUCKETS,
METRIC_FILES_STORAGE
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'requests' => $stats[$metrics[0]] ?? [],
'network' => $stats[$metrics[1]] ?? [],
'storage' => $stats[$metrics[2]] ?? [],
'users' => $stats[$metrics[3]] ?? [],
'databases' => $stats[$metrics[4]] ?? [],
'documents' => $stats[$metrics[5]] ?? [],
'executions' => $stats[$metrics[6]] ?? [],
'buckets' => $stats[$metrics[7]] ?? [],
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_PROJECT);
$response->dynamic(new Document([
'range' => $range,
'requestsTotal' => ($usage[$metrics[0]]),
'network' => ($usage[$metrics[1]] + $usage[$metrics[2]]),
'executionsTotal' => $usage[$metrics[3]],
'documentsTotal' => $usage[$metrics[4]],
'databasesTotal' => $usage[$metrics[5]],
'usersTotal' => $usage[$metrics[6]],
'bucketsTotal' => $usage[$metrics[7]],
'filesStorage' => $usage[$metrics[8]],
]), Response::MODEL_USAGE_PROJECT);
});
app/controllers/api/projects.php
+3 -3
View File
@@ -17,11 +17,11 @@ use Utopia\Audit\Audit;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
use Utopia\Database\DateTime;
use Utopia\Database\Permission;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Query;
use Utopia\Database\Role;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\DatetimeValidator;
use Utopia\Database\Validator\UID;
use Utopia\Domains\Domain;
app/controllers/api/storage.php
+119 -212
View File
@@ -16,10 +16,10 @@ use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Exception\Authorization as AuthorizationException;
use Utopia\Database\Exception\Duplicate as DuplicateException;
use Utopia\Database\Exception\Structure as StructureException;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Query;
use Utopia\Database\Role;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\Permissions;
use Utopia\Database\Validator\UID;
@@ -51,7 +51,6 @@ App::post('/v1/storage/buckets')
->label('event', 'buckets.[bucketId].create')
->label('audits.event', 'bucket.create')
->label('audits.resource', 'bucket/{response.$id}')
->label('usage.metric', 'buckets.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'createBucket')
@@ -147,7 +146,6 @@ App::get('/v1/storage/buckets')
->desc('List buckets')
->groups(['api', 'storage'])
->label('scope', 'buckets.read')
->label('usage.metric', 'buckets.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'listBuckets')
@@ -194,7 +192,6 @@ App::get('/v1/storage/buckets/:bucketId')
->desc('Get Bucket')
->groups(['api', 'storage'])
->label('scope', 'buckets.read')
->label('usage.metric', 'buckets.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getBucket')
@@ -223,7 +220,6 @@ App::put('/v1/storage/buckets/:bucketId')
->label('event', 'buckets.[bucketId].update')
->label('audits.event', 'bucket.update')
->label('audits.resource', 'bucket/{response.$id}')
->label('usage.metric', 'buckets.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'updateBucket')
@@ -290,7 +286,6 @@ App::delete('/v1/storage/buckets/:bucketId')
->label('audits.event', 'bucket.delete')
->label('event', 'buckets.[bucketId].delete')
->label('audits.resource', 'bucket/{request.bucketId}')
->label('usage.metric', 'buckets.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'deleteBucket')
@@ -333,8 +328,6 @@ App::post('/v1/storage/buckets/:bucketId/files')
->label('audits.event', 'file.create')
->label('event', 'buckets.[bucketId].files.[fileId].create')
->label('audits.resource', 'file/{response.$id}')
->label('usage.metric', 'files.{scope}.requests.create')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -660,8 +653,6 @@ App::get('/v1/storage/buckets/:bucketId/files')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'listFiles')
->label('sdk.description', '/docs/references/storage/list-files.md')
@@ -737,8 +728,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFile')
->label('sdk.description', '/docs/references/storage/get-file.md')
@@ -786,8 +775,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
->label('cache', true)
->label('cache.resourceType', 'bucket/{request.bucketId}')
->label('cache.resource', 'file/{request.fileId}')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFilePreview')
@@ -949,8 +936,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
->desc('Get File for Download')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFileDownload')
@@ -1089,8 +1074,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
->desc('Get File for View')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFileView')
@@ -1243,8 +1226,6 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
->label('event', 'buckets.[bucketId].files.[fileId].update')
->label('audits.event', 'file.update')
->label('audits.resource', 'file/{response.$id}')
->label('usage.metric', 'files.{scope}.requests.update')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -1346,8 +1327,6 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
->label('event', 'buckets.[bucketId].files.[fileId].delete')
->label('audits.event', 'file.delete')
->label('audits.resource', 'file/{request.fileId}')
->label('usage.metric', 'files.{scope}.requests.delete')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -1434,118 +1413,8 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
$response->noContent();
});
App::get('/v1/storage/usage')
->desc('Get usage stats for storage')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_STORAGE)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$metrics = [
'project.$all.storage.size',
'buckets.$all.count.total',
'buckets.$all.requests.create',
'buckets.$all.requests.read',
'buckets.$all.requests.update',
'buckets.$all.requests.delete',
'files.$all.storage.size',
'files.$all.count.total',
'files.$all.requests.create',
'files.$all.requests.read',
'files.$all.requests.update',
'files.$all.requests.delete',
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'bucketsCount' => $stats['buckets.$all.count.total'],
'bucketsCreate' => $stats['buckets.$all.requests.create'],
'bucketsRead' => $stats['buckets.$all.requests.read'],
'bucketsUpdate' => $stats['buckets.$all.requests.update'],
'bucketsDelete' => $stats['buckets.$all.requests.delete'],
'storage' => $stats['project.$all.storage.size'],
'filesCount' => $stats['files.$all.count.total'],
'filesCreate' => $stats['files.$all.requests.create'],
'filesRead' => $stats['files.$all.requests.read'],
'filesUpdate' => $stats['files.$all.requests.update'],
'filesDelete' => $stats['files.$all.requests.delete'],
]);
}
$response->dynamic($usage, Response::MODEL_USAGE_STORAGE);
});
App::get('/v1/storage/:bucketId/usage')
->desc('Get usage stats for a storage bucket')
->desc('Get usage stats for storage bucket')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
@@ -1566,86 +1435,124 @@ App::get('/v1/storage/:bucketId/usage')
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES),
str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE),
];
$metrics = [
"files.{$bucketId}.count.total",
"files.{$bucketId}.storage.size",
"files.{$bucketId}.requests.create",
"files.{$bucketId}.requests.read",
"files.{$bucketId}.requests.update",
"files.{$bucketId}.requests.delete",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'filesCount' => $stats[$metrics[0]],
'filesStorage' => $stats[$metrics[1]],
'filesCreate' => $stats[$metrics[2]],
'filesRead' => $stats[$metrics[3]],
'filesUpdate' => $stats[$metrics[4]],
'filesDelete' => $stats[$metrics[5]],
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_BUCKETS);
$response->dynamic(new Document([
'range' => $range,
'filesTotal' => $usage[$metrics[0]],
'filesStorage' => $usage[$metrics[1]],
]), Response::MODEL_USAGE_BUCKETS);
});
App::get('/v1/storage/usage')
->desc('Get usage stats for storage')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_STORAGE)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_BUCKETS,
METRIC_FILES,
METRIC_FILES_STORAGE,
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic(new Document([
'range' => $range,
'bucketsTotal' => $usage[$metrics[0]],
'filesTotal' => $usage[$metrics[1]],
'filesStorage' => $usage[$metrics[2]],
]), Response::MODEL_USAGE_STORAGE);
});
app/controllers/api/teams.php
+3 -3
View File
@@ -25,11 +25,11 @@ use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Exception\Authorization as AuthorizationException;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Query;
use Utopia\Database\DateTime;
use Utopia\Database\Role;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\Key;
use Utopia\Database\Validator\UID;
app/controllers/api/users.php
+49 -112
View File
@@ -16,9 +16,9 @@ use Appwrite\Utopia\Response;
use Utopia\App;
use Utopia\Audit\Audit;
use Utopia\Config\Config;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Locale\Locale;
use Appwrite\Extend\Exception;
use Utopia\Database\Document;
@@ -90,7 +90,6 @@ App::post('/v1/users')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'create')
@@ -121,7 +120,6 @@ App::post('/v1/users/bcrypt')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createBcryptUser')
@@ -151,7 +149,6 @@ App::post('/v1/users/md5')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createMD5User')
@@ -181,7 +178,6 @@ App::post('/v1/users/argon2')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createArgon2User')
@@ -211,7 +207,6 @@ App::post('/v1/users/sha')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createSHAUser')
@@ -248,7 +243,6 @@ App::post('/v1/users/phpass')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createPHPassUser')
@@ -278,7 +272,6 @@ App::post('/v1/users/scrypt')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createScryptUser')
@@ -321,7 +314,6 @@ App::post('/v1/users/scrypt-modified')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createScryptModifiedUser')
@@ -351,7 +343,6 @@ App::get('/v1/users')
->desc('List Users')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'list')
@@ -398,7 +389,6 @@ App::get('/v1/users/:userId')
->desc('Get User')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'get')
@@ -424,7 +414,6 @@ App::get('/v1/users/:userId/prefs')
->desc('Get User Preferences')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'getPrefs')
@@ -452,7 +441,6 @@ App::get('/v1/users/:userId/sessions')
->desc('List User Sessions')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listSessions')
@@ -494,7 +482,6 @@ App::get('/v1/users/:userId/memberships')
->desc('List User Memberships')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listMemberships')
@@ -534,7 +521,6 @@ App::get('/v1/users/:userId/logs')
->desc('List User Logs')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listLogs')
@@ -620,7 +606,6 @@ App::patch('/v1/users/:userId/status')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateStatus')
@@ -656,7 +641,6 @@ App::patch('/v1/users/:userId/verification')
->label('scope', 'users.write')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateEmailVerification')
@@ -692,7 +676,6 @@ App::patch('/v1/users/:userId/verification/phone')
->label('scope', 'users.write')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePhoneVerification')
@@ -729,7 +712,6 @@ App::patch('/v1/users/:userId/name')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateName')
@@ -770,7 +752,6 @@ App::patch('/v1/users/:userId/password')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePassword')
@@ -812,7 +793,6 @@ App::patch('/v1/users/:userId/email')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateEmail')
@@ -858,7 +838,6 @@ App::patch('/v1/users/:userId/phone')
->label('scope', 'users.write')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePhone')
@@ -904,7 +883,6 @@ App::patch('/v1/users/:userId/verification')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{request.userId}')
->label('audits.userId', '{request.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateEmailVerification')
@@ -937,7 +915,6 @@ App::patch('/v1/users/:userId/prefs')
->groups(['api', 'users'])
->label('event', 'users.[userId].update.prefs')
->label('scope', 'users.write')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePrefs')
@@ -973,7 +950,6 @@ App::delete('/v1/users/:userId/sessions/:sessionId')
->label('scope', 'users.write')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{request.userId}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'deleteSession')
@@ -1016,7 +992,6 @@ App::delete('/v1/users/:userId/sessions')
->label('scope', 'users.write')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{user.$id}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'deleteSessions')
@@ -1059,7 +1034,6 @@ App::delete('/v1/users/:userId')
->label('scope', 'users.write')
->label('audits.event', 'user.delete')
->label('audits.resource', 'user/{request.userId}')
->label('usage.metric', 'users.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'delete')
@@ -1106,96 +1080,59 @@ App::get('/v1/users/usage')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_USERS)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(fn ($value) => "oauth-" . $value, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true)
->inject('response')
->inject('dbForProject')
->inject('register')
->action(function (string $range, string $provider, Response $response, Database $dbForProject) {
->action(function (string $range, Response $response, Database $dbForProject) {
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_USERS,
METRIC_SESSIONS,
];
$metrics = [
'users.$all.count.total',
'users.$all.requests.create',
'users.$all.requests.read',
'users.$all.requests.update',
'users.$all.requests.delete',
'sessions.$all.requests.create',
'sessions.$all.requests.delete',
"sessions.$provider.requests.create",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
});
}
});
$usage = new Document([
'range' => $range,
'usersCount' => $stats['users.$all.count.total'] ?? [],
'usersCreate' => $stats['users.$all.requests.create'] ?? [],
'usersRead' => $stats['users.$all.requests.read'] ?? [],
'usersUpdate' => $stats['users.$all.requests.update'] ?? [],
'usersDelete' => $stats['users.$all.requests.delete'] ?? [],
'sessionsCreate' => $stats['sessions.$all.requests.create'] ?? [],
'sessionsProviderCreate' => $stats["sessions.$provider.requests.create"] ?? [],
'sessionsDelete' => $stats['sessions.$all.requests.delete' ?? []]
]);
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
}
$response->dynamic($usage, Response::MODEL_USAGE_USERS);
$response->dynamic(new Document([
'range' => $range,
'usersTotal' => $usage[$metrics[0]],
'sessionsTotal' => $usage[$metrics[1]],
]), Response::MODEL_USAGE_USERS);
});
app/controllers/general.php
+2 -1
View File
@@ -3,7 +3,7 @@
require_once __DIR__ . '/../init.php';
use Utopia\App;
use Utopia\Database\Role;
use Utopia\Database\Helpers\Role;
use Utopia\Locale\Locale;
use Utopia\Logger\Logger;
use Utopia\Logger\Log;
@@ -631,6 +631,7 @@ App::get('/.well-known/acme-challenge')
});
include_once __DIR__ . '/shared/api.php';
include_once __DIR__ . '/shared/api/auth.php';
foreach (Config::getParam('services', []) as $service) {
include_once $service['controller'];
app/controllers/mock.php
+1 -2
View File
@@ -7,14 +7,13 @@ use Utopia\Database\Document;
use Appwrite\Network\Validator\Host;
use Appwrite\Utopia\Request;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
use Utopia\App;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Integer;
use Utopia\Validator\Text;
use Utopia\Storage\Validator\File;
use Utopia\Validator\WhiteList;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
App::get('/v1/mock/tests/foo')
->desc('Get Foo')
app/controllers/shared/api.php
+122 -136
View File
@@ -7,8 +7,8 @@ use Appwrite\Event\Delete;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Mail;
use Appwrite\Event\Usage;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Usage\Stats;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Request;
use Utopia\App;
@@ -17,7 +17,6 @@ use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\Cache\Adapter\Filesystem;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
@@ -49,43 +48,101 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar
return $label;
};
$databaseListener = function (string $event, Document $document, Stats $usage) {
$multiplier = 1;
$databaseListener = function (string $event, Document $document, Document $project, Usage $queueForUsage, Database $dbForProject) {
$value = 1;
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$multiplier = -1;
$value = -1;
}
$collection = $document->getCollection();
switch ($collection) {
case 'users':
$usage->setParam('users.{scope}.count.total', 1 * $multiplier);
switch (true) {
case $document->getCollection() === 'teams':
$queueForUsage
->addMetric(METRIC_TEAMS, $value); // per project
break;
case 'databases':
$usage->setParam('databases.{scope}.count.total', 1 * $multiplier);
case $document->getCollection() === 'users':
$queueForUsage
->addMetric(METRIC_USERS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case 'buckets':
$usage->setParam('buckets.{scope}.count.total', 1 * $multiplier);
case $document->getCollection() === 'sessions': // sessions
$queueForUsage
->addMetric(METRIC_SESSIONS, $value); //per project
break;
case 'deployments':
$usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size') * $multiplier);
case $document->getCollection() === 'databases': // databases
$queueForUsage
->addMetric(METRIC_DATABASES, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$queueForUsage
->addMetric(METRIC_COLLECTIONS, $value) // per project
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database
;
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$collectionInternalId = $parts[3] ?? 0;
$queueForUsage
->addMetric(METRIC_DOCUMENTS, $value) // per project
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection
break;
case $document->getCollection() === 'buckets': //buckets
$queueForUsage
->addMetric(METRIC_BUCKETS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'bucket_'): // files
$parts = explode('_', $document->getCollection());
$bucketInternalId = $parts[1];
$queueForUsage
->addMetric(METRIC_FILES, $value) // per project
->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project
->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES), $value) // per bucket
->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket
break;
case $document->getCollection() === 'functions':
$queueForUsage
->addMetric(METRIC_FUNCTIONS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case $document->getCollection() === 'deployments':
$queueForUsage
->addMetric(METRIC_DEPLOYMENTS, $value) // per project
->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value)// per function
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), $document->getAttribute('size') * $value);// per function
break;
case $document->getCollection() === 'executions':
$queueForUsage
->addMetric(METRIC_EXECUTIONS, $value) // per project
->addMetric(str_replace('{functionInternalId}', $document->getAttribute('functionInternalId'), METRIC_FUNCTION_ID_EXECUTIONS), $value);// per function
break;
default:
if (strpos($collection, 'bucket_') === 0) {
$usage
->setParam('bucketId', $document->getAttribute('bucketId'))
->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal') * $multiplier)
->setParam('files.{scope}.count.total', 1 * $multiplier);
} elseif (strpos($collection, 'database_') === 0) {
$usage
->setParam('databaseId', $document->getAttribute('databaseId'));
if (strpos($collection, '_collection_') !== false) {
$usage
->setParam('collectionId', $document->getAttribute('$collectionId'))
->setParam('documents.{scope}.count.total', 1 * $multiplier);
} else {
$usage->setParam('collections.{scope}.count.total', 1 * $multiplier);
}
}
break;
}
};
@@ -100,12 +157,12 @@ App::init()
->inject('events')
->inject('audits')
->inject('mails')
->inject('usage')
->inject('deletes')
->inject('database')
->inject('dbForProject')
->inject('queueForUsage')
->inject('mode')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Mail $mails, Stats $usage, Delete $deletes, EventDatabase $database, Database $dbForProject, string $mode) use ($databaseListener) {
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Mail $mails, Delete $deletes, EventDatabase $database, Database $dbForProject, Usage $queueForUsage, string $mode) use ($databaseListener) {
$route = $utopia->match($request);
@@ -124,11 +181,11 @@ App::init()
foreach ($abuseKeyLabel as $abuseKey) {
$timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $dbForProject);
$timeLimit
->setParam('{userId}', $user->getId())
->setParam('{userAgent}', $request->getUserAgent(''))
->setParam('{ip}', $request->getIP())
->setParam('{url}', $request->getHostname() . $route->getPath())
->setParam('{method}', $request->getMethod());
->setParam('{userId}', $user->getId())
->setParam('{userAgent}', $request->getUserAgent(''))
->setParam('{ip}', $request->getIP())
->setParam('{url}', $request->getHostname() . $route->getPath())
->setParam('{method}', $request->getMethod());
$timeLimitArray[] = $timeLimit;
}
@@ -191,20 +248,13 @@ App::init()
->setProject($project)
->setUser($user);
$usage
->setParam('projectInternalId', $project->getInternalId())
->setParam('projectId', $project->getId())
->setParam('project.{scope}.network.requests', 1)
->setParam('httpMethod', $request->getMethod())
->setParam('project.{scope}.network.inbound', 0)
->setParam('project.{scope}.network.outbound', 0);
$deletes->setProject($project);
$database->setProject($project);
$dbForProject->on(Database::EVENT_DOCUMENT_CREATE, fn ($event, Document $document) => $databaseListener($event, $document, $usage));
$dbForProject->on(Database::EVENT_DOCUMENT_DELETE, fn ($event, Document $document) => $databaseListener($event, $document, $usage));
$dbForProject
->on(Database::EVENT_DOCUMENT_CREATE, fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject))
->on(Database::EVENT_DOCUMENT_DELETE, fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject))
;
$useCache = $route->getLabel('cache', false);
@@ -265,60 +315,6 @@ App::init()
}
});
App::init()
->groups(['auth'])
->inject('utopia')
->inject('request')
->inject('project')
->action(function (App $utopia, Request $request, Document $project) {
$route = $utopia->match($request);
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
$isAppUser = Auth::isAppUser(Authorization::getRoles());
if ($isAppUser || $isPrivilegedUser) { // Skip limits for app and console devs
return;
}
$auths = $project->getAttribute('auths', []);
switch ($route->getLabel('auth.type', '')) {
case 'emailPassword':
if (($auths['emailPassword'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Email / Password authentication is disabled for this project');
}
break;
case 'magic-url':
if ($project->getAttribute('usersAuthMagicURL', true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Magic URL authentication is disabled for this project');
}
break;
case 'anonymous':
if (($auths['anonymous'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Anonymous authentication is disabled for this project');
}
break;
case 'invites':
if (($auths['invites'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Invites authentication is disabled for this project');
}
break;
case 'jwt':
if (($auths['JWT'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'JWT authentication is disabled for this project');
}
break;
default:
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Unsupported authentication route');
break;
}
});
App::shutdown()
->groups(['api'])
->inject('utopia')
@@ -327,13 +323,13 @@ App::shutdown()
->inject('project')
->inject('events')
->inject('audits')
->inject('usage')
->inject('deletes')
->inject('database')
->inject('mode')
->inject('dbForProject')
->inject('queueForFunctions')
->action(function (App $utopia, Request $request, Response $response, Document $project, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, string $mode, Database $dbForProject, Func $queueForFunctions) use ($parseLabel) {
->inject('queueForUsage')
->inject('mode')
->action(function (App $utopia, Request $request, Response $response, Document $project, Event $events, Audit $audits, Delete $deletes, EventDatabase $database, Database $dbForProject, Func $queueForFunctions, Usage $queueForUsage, string $mode) use ($parseLabel) {
$responsePayload = $response->getPayload();
@@ -341,6 +337,7 @@ App::shutdown()
if (empty($events->getPayload())) {
$events->setPayload($responsePayload);
}
/**
* Trigger functions.
*/
@@ -458,10 +455,10 @@ App::shutdown()
$key = md5($request->getURI() . implode('*', $request->getParams())) . '*' . APP_CACHE_BUSTER;
$data = json_encode([
'resourceType' => $resourceType,
'resource' => $resource,
'contentType' => $response->getContentType(),
'payload' => base64_encode($data['payload']),
'resourceType' => $resourceType,
'resource' => $resource,
'contentType' => $response->getContentType(),
'payload' => base64_encode($data['payload']),
]) ;
$signature = md5($data);
@@ -489,35 +486,24 @@ App::shutdown()
}
}
if (
App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled'
&& $project->getId()
&& !empty($route->getLabel('sdk.namespace', null))
) { // Don't calculate console usage on admin mode
$metric = $route->getLabel('usage.metric', '');
$usageParams = $route->getLabel('usage.params', []);
if (!empty($metric)) {
$usage->setParam($metric, 1);
foreach ($usageParams as $param) {
$param = $parseLabel($param, $responsePayload, $requestParams, $user);
$parts = explode(':', $param);
if (count($parts) != 2) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Usage params not properly set');
}
$usage->setParam($parts[0], $parts[1]);
if ($project->getId() !== 'console') {
if ($mode !== APP_MODE_ADMIN) {
$fileSize = 0;
$file = $request->getFiles('file');
if (!empty($file)) {
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$queueForUsage
->addMetric(METRIC_NETWORK_REQUESTS, 1)
->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize)
->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize());
}
$fileSize = 0;
$file = $request->getFiles('file');
if (!empty($file)) {
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$usage
->setParam('project.{scope}.network.inbound', $request->getSize() + $fileSize)
->setParam('project.{scope}.network.outbound', $response->getSize())
->submit();
$queueForUsage
->setProject($project)
->trigger();
}
});
app/controllers/shared/api/auth.php
+62
View File
@@ -0,0 +1,62 @@
<?php
use Appwrite\Auth\Auth;
use Appwrite\Utopia\Request;
use Utopia\App;
use Appwrite\Extend\Exception;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
App::init()
->groups(['auth'])
->inject('utopia')
->inject('request')
->inject('project')
->action(function (App $utopia, Request $request, Document $project) {
$route = $utopia->match($request);
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
$isAppUser = Auth::isAppUser(Authorization::getRoles());
if ($isAppUser || $isPrivilegedUser) { // Skip limits for app and console devs
return;
}
$auths = $project->getAttribute('auths', []);
switch ($route->getLabel('auth.type', '')) {
case 'emailPassword':
if (($auths['emailPassword'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Email / Password authentication is disabled for this project');
}
break;
case 'magic-url':
if ($project->getAttribute('usersAuthMagicURL', true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Magic URL authentication is disabled for this project');
}
break;
case 'anonymous':
if (($auths['anonymous'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Anonymous authentication is disabled for this project');
}
break;
case 'invites':
if (($auths['invites'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Invites authentication is disabled for this project');
}
break;
case 'jwt':
if (($auths['JWT'] ?? true) === false) {
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'JWT authentication is disabled for this project');
}
break;
default:
throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Unsupported authentication route');
break;
}
});
app/http.php
+3 -3
View File
@@ -10,9 +10,9 @@ use Swoole\Http\Response as SwooleResponse;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\Authorization;
use Utopia\Audit\Audit;
use Utopia\Abuse\Adapters\TimeLimit;
app/init.php
+39 -34
View File
@@ -18,6 +18,7 @@ ini_set('display_startup_errors', 1);
ini_set('default_socket_timeout', -1);
error_reporting(E_ALL);
use Appwrite\Event\Usage;
use Appwrite\Extend\Exception;
use Appwrite\Auth\Auth;
use Appwrite\SMS\Adapter\Mock;
@@ -37,11 +38,10 @@ use Appwrite\Network\Validator\IP;
use Appwrite\Network\Validator\URL;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\URL\URL as AppwriteURL;
use Appwrite\Usage\Stats;
use Utopia\App;
use Utopia\Validator\Range;
use Utopia\Validator\WhiteList;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Document;
use Utopia\Database\Database;
use Utopia\Database\Query;
@@ -174,6 +174,39 @@ const APP_AUTH_TYPE_KEY = 'Key';
const APP_AUTH_TYPE_ADMIN = 'Admin';
// Response related
const MAX_OUTPUT_CHUNK_SIZE = 2 * 1024 * 1024; // 2MB
// Usage metrics
const METRIC_TEAMS = 'teams';
const METRIC_USERS = 'users';
const METRIC_SESSIONS = 'sessions';
const METRIC_DATABASES = 'databases';
const METRIC_COLLECTIONS = 'collections';
const METRIC_DATABASE_ID_COLLECTIONS = '{databaseInternalId}.collections';
const METRIC_DOCUMENTS = 'documents';
const METRIC_DATABASE_ID_DOCUMENTS = '{databaseInternalId}.documents';
const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS = '{databaseInternalId}.{collectionInternalId}.documents';
const METRIC_BUCKETS = 'buckets';
const METRIC_FILES = 'files';
const METRIC_FILES_STORAGE = 'files.storage';
const METRIC_BUCKET_ID_FILES = '{bucketInternalId}.files';
const METRIC_BUCKET_ID_FILES_STORAGE = '{bucketInternalId}.files.storage';
const METRIC_FUNCTIONS = 'functions';
const METRIC_DEPLOYMENTS = 'deployments';
const METRIC_DEPLOYMENTS_STORAGE = 'deployments.storage';
const METRIC_BUILDS = 'builds';
const METRIC_BUILDS_STORAGE = 'builds.storage';
const METRIC_BUILDS_COMPUTE = 'builds.compute';
const METRIC_FUNCTION_ID_BUILDS = '{functionInternalId}.builds';
const METRIC_FUNCTION_ID_BUILDS_STORAGE = '{functionInternalId}.builds.storage';
const METRIC_FUNCTION_ID_BUILDS_COMPUTE = '{functionInternalId}.builds.compute';
const METRIC_FUNCTION_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments';
const METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE = '{resourceType}.{resourceInternalId}.deployments.storage';
const METRIC_EXECUTIONS = 'executions';
const METRIC_EXECUTIONS_COMPUTE = 'executions.compute';
const METRIC_FUNCTION_ID_EXECUTIONS = '{functionInternalId}.executions';
const METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE = '{functionInternalId}.executions.compute';
const METRIC_NETWORK_REQUESTS = 'network.requests';
const METRIC_NETWORK_INBOUND = 'network.inbound';
const METRIC_NETWORK_OUTBOUND = 'network.outbound';
$register = new Registry();
@@ -189,6 +222,7 @@ Config::load('providers', __DIR__ . '/config/providers.php');
Config::load('platforms', __DIR__ . '/config/platforms.php');
Config::load('collections', __DIR__ . '/config/collections.php');
Config::load('runtimes', __DIR__ . '/config/runtimes.php');
Config::load('usage', __DIR__ . '/config/usage.php');
Config::load('roles', __DIR__ . '/config/roles.php'); // User roles and scopes
Config::load('scopes', __DIR__ . '/config/scopes.php'); // User roles and scopes
Config::load('services', __DIR__ . '/config/services.php'); // List of services
@@ -576,13 +610,11 @@ $register->set('pools', function () {
$schemes = $connection['schemes'] ?? [];
$config = [];
$dsns = explode(',', $connection['dsns'] ?? '');
foreach ($dsns as &$dsn) {
$dsn = explode('=', $dsn);
$name = ($multipe) ? $key . '_' . $dsn[0] : $key;
$dsn = $dsn[1] ?? '';
$config[] = $name;
if (empty($dsn)) {
//throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}");
continue;
@@ -607,7 +639,6 @@ $register->set('pools', function () {
*
* Resource assignment to an adapter will happen below.
*/
switch ($dsnScheme) {
case 'mysql':
case 'mariadb':
@@ -645,7 +676,6 @@ $register->set('pools', function () {
$pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) {
// Get Adapter
$adapter = null;
switch ($type) {
case 'database':
$adapter = match ($dsn->getScheme()) {
@@ -689,30 +719,6 @@ $register->set('pools', function () {
return $group;
});
$register->set('influxdb', function () {
// Register DB connection
$host = App::getEnv('_APP_INFLUXDB_HOST', '');
$port = App::getEnv('_APP_INFLUXDB_PORT', '');
if (empty($host) || empty($port)) {
return;
}
$driver = new InfluxDB\Driver\Curl("http://{$host}:{$port}");
$client = new InfluxDB\Client($host, $port, '', '', false, false, 5);
$client->setDriver($driver);
return $client;
});
$register->set('statsd', function () {
// Register DB connection
$host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
$port = App::getEnv('_APP_STATSD_PORT', 8125);
$connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port);
$statsd = new \Domnikl\Statsd\Client($connection);
return $statsd;
});
$register->set('smtp', function () {
$mail = new PHPMailer(true);
@@ -844,7 +850,6 @@ App::setResource('loggerBreadcrumbs', function () {
});
App::setResource('register', fn() => $register);
App::setResource('locale', fn() => new Locale(App::getEnv('_APP_LOCALE', 'en')));
// Queues
@@ -860,9 +865,9 @@ App::setResource('queue', function (Group $pools) {
App::setResource('queueForFunctions', function (Connection $queue) {
return new Func($queue);
}, ['queue']);
App::setResource('usage', function ($register) {
return new Stats($register->get('statsd'));
}, ['register']);
App::setResource('queueForUsage', function (Connection $queue) {
return new Usage($queue);
}, ['queue']);
App::setResource('clients', function ($request, $console, $project) {
$console->setAttribute('platforms', [ // Always allow current host
'$collection' => ID::custom('platforms'),
app/realtime.php
+2 -2
View File
@@ -13,8 +13,8 @@ use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\ID;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Role;
use Utopia\Logger\Log;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
app/views/install/compose.phtml
+31 -51
View File
@@ -80,7 +80,6 @@ services:
- mariadb
- redis
# - clamav
- influxdb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
@@ -114,8 +113,6 @@ services:
- _APP_SMTP_USERNAME
- _APP_SMTP_PASSWORD
- _APP_USAGE_STATS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_STORAGE_LIMIT
- _APP_STORAGE_PREVIEW_LIMIT
- _APP_STORAGE_ANTIVIRUS
@@ -132,8 +129,6 @@ services:
- _APP_EXECUTOR_HOST
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_STATSD_HOST
- _APP_STATSD_PORT
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
@@ -465,35 +460,6 @@ services:
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
appwrite-usage:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: usage
container_name: appwrite-usage
<<: *x-logging
restart: unless-stopped
networks:
- appwrite
depends_on:
- influxdb
- mariadb
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-schedule:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: schedule
@@ -578,26 +544,41 @@ services:
# volumes:
# - appwrite-uploads:/storage/uploads
influxdb:
image: appwrite/influxdb:1.5.0
container_name: appwrite-influxdb
appwrite-worker-usage:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: worker-usage
<<: *x-logging
restart: unless-stopped
container_name: appwrite-worker-usage
networks:
- appwrite
volumes:
- appwrite-influxdb:/var/lib/influxdb:rw
telegraf:
image: appwrite/telegraf:1.4.0
container_name: appwrite-telegraf
<<: *x-logging
restart: unless-stopped
networks:
- appwrite
environment:
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_USAGE_STATS
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
networks:
gateway:
@@ -613,7 +594,6 @@ volumes:
appwrite-cache:
appwrite-uploads:
appwrite-certificates:
appwrite-influxdb:
appwrite-config:
appwrite-functions:
appwrite-builds:
app/worker.php
+11 -4
View File
@@ -3,6 +3,7 @@
require_once __DIR__ . '/init.php';
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Swoole\Runtime;
use Utopia\App;
use Utopia\Cache\Adapter\Sharding;
@@ -85,12 +86,18 @@ Server::setResource('queueForFunctions', function (Registry $register) {
);
}, ['register']);
Server::setResource('logger', function ($register) {
return $register->get('logger');
Server::setResource('queueForUsage', function (Registry $register) {
$pools = $register->get('pools');
return new Usage(
$pools
->get('queue')
->pop()
->getResource()
);
}, ['register']);
Server::setResource('statsd', function ($register) {
return $register->get('statsd');
Server::setResource('logger', function ($register) {
return $register->get('logger');
}, ['register']);
Server::setResource('pools', function ($register) {
app/workers/audits.php
+1
View File
@@ -39,6 +39,7 @@ class AuditsV1 extends Worker
$dbForProject = $this->getProjectDB($project);
$audit = new Audit($dbForProject);
$audit->log(
userInternalId: $user->getInternalId(),
userId: $user->getId(),
// Pass first, most verbose event pattern
event: $event,
app/workers/builds.php
+20 -20
View File
@@ -6,11 +6,10 @@ use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Resque\Worker;
use Appwrite\Utopia\Response\Model\Deployment;
use Executor\Executor;
use Appwrite\Usage\Stats;
use Utopia\Database\DateTime;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
use Utopia\DSN\DSN;
use Utopia\Database\Document;
use Utopia\Config\Config;
@@ -57,6 +56,11 @@ class BuildsV1 extends Worker
}
}
/**
* @throws \Utopia\Database\Exception\Authorization
* @throws \Utopia\Database\Exception\Structure
* @throws Throwable
*/
protected function buildDeployment(Document $project, Document $function, Document $deployment)
{
global $register;
@@ -172,8 +176,8 @@ class BuildsV1 extends Worker
try {
$response = $this->executor->createRuntime(
projectId: $project->getId(),
deploymentId: $deployment->getId(),
projectId: $project->getId(),
source: $source,
image: $runtime['image'],
remove: true,
@@ -249,24 +253,20 @@ class BuildsV1 extends Worker
channels: $target['channels'],
roles: $target['roles']
);
/** Update usage stats */
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$statsd = $register->get('statsd');
$usage = new Stats($statsd);
$usage
->setParam('projectInternalId', $project->getInternalId())
->setParam('projectId', $project->getId())
->setParam('functionId', $function->getId())
->setParam('builds.{scope}.compute', 1)
->setParam('buildStatus', $build->getAttribute('status', ''))
->setParam('buildTime', $build->getAttribute('duration'))
->setParam('buildSize', $build->getAttribute('size'))
->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0)
->submit();
}
}
/** Trigger usage queue */
$this
->getUsageQueue()
->setProject($project)
->addMetric(METRIC_BUILDS, 1) // per project
->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000)
->trigger()
;
}
public function shutdown(): void
app/workers/certificates.php
+1 -1
View File
@@ -8,7 +8,7 @@ use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\DateTime;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Domains\Domain;
app/workers/functions.php
+27 -26
View File
@@ -2,23 +2,22 @@
require_once __DIR__ . '/../worker.php';
use Appwrite\Event\Usage;
use Utopia\Queue\Message;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Usage\Stats;
use Appwrite\Utopia\Response\Model\Execution;
use Domnikl\Statsd\Client;
use Executor\Executor;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Query;
use Utopia\Database\Role;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\Authorization;
use Utopia\Queue\Server;
@@ -29,7 +28,7 @@ Server::setResource('execute', function () {
return function (
Func $queueForFunctions,
Database $dbForProject,
Client $statsd,
Usage $queueForUsage,
Document $project,
Document $function,
string $trigger,
@@ -40,6 +39,7 @@ Server::setResource('execute', function () {
string $eventData = null,
string $executionId = null,
) {
$user ??= new Document();
$functionId = $function->getId();
$deploymentId = $function->getAttribute('deployment', '');
@@ -97,6 +97,14 @@ Server::setResource('execute', function () {
if ($execution->isEmpty()) {
throw new Exception('Failed to create or read execution');
}
/**
* Usage
*/
$queueForUsage
->addMetric(METRIC_EXECUTIONS, 1) // per project
->addMetric(str_replace('{functionInternalId}', $function->getId(), METRIC_FUNCTION_ID_EXECUTIONS), 1); // per function
}
$execution->setAttribute('status', 'processing');
@@ -203,21 +211,13 @@ Server::setResource('execute', function () {
roles: $target['roles']
);
/** Update usage stats */
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$usage = new Stats($statsd);
$usage
->setParam('projectId', $project->getId())
->setParam('projectInternalId', $project->getInternalId())
->setParam('functionId', $function->getId())
->setParam('functionInternalId', $function->getInternalId())
->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', ''))
->setParam('executionTime', $execution->getAttribute('duration'))
->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0)
->submit();
}
/** Trigger usage queue */
$queueForUsage
->setProject($project)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
->trigger()
;
};
});
@@ -225,9 +225,9 @@ $server->job()
->inject('message')
->inject('dbForProject')
->inject('queueForFunctions')
->inject('statsd')
->inject('queueForUsage')
->inject('execute')
->action(function (Message $message, Database $dbForProject, Func $queueForFunctions, Client $statsd, callable $execute) {
->action(function (Message $message, Database $dbForProject, Func $queueForFunctions, Usage $queueForUsage, callable $execute) {
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
@@ -269,8 +269,9 @@ $server->job()
continue;
}
Console::success('Iterating function: ' . $function->getAttribute('name'));
$execute(
statsd: $statsd,
queueForUsage: $queueForUsage,
dbForProject: $dbForProject,
project: $project,
function: $function,
@@ -309,7 +310,7 @@ $server->job()
data: $data,
user: $user,
jwt: $jwt,
statsd: $statsd,
queueForUsage: $queueForUsage,
);
break;
case 'schedule':
@@ -325,7 +326,7 @@ $server->job()
data: null,
user: null,
jwt: null,
statsd: $statsd,
queueForUsage: $queueForUsage,
);
break;
}
app/workers/usage.php
+289
View File
@@ -0,0 +1,289 @@
<?php
require_once __DIR__ . '/../worker.php';
use Swoole\Timer;
use Utopia\App;
use Utopia\Cache\Cache;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Validator\Authorization;
use Utopia\Queue\Message;
use Utopia\CLI\Console;
use Utopia\Queue\Server;
use Utopia\Registry\Registry;
Authorization::disable();
Authorization::setDefaultStatus(false);
$stats = [];
$periods['1h'] = 'Y-m-d H:00';
$periods['1d'] = 'Y-m-d 00:00';
//$periods['1m'] = 'Y-m-1 00:00';
$periods['inf'] = '0000-00-00 00:00';
const INFINITI_PERIOD = '_inf_';
/**
* On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files
* When we remove a parent document we need to deduct his children aggregation from the project scope
*/
Server::setResource('reduce', function (Cache $cache, Registry $register, $pools) {
return function ($database, $projectInternalId, Document $document, array &$metrics) use ($pools, $cache, $register): void {
try {
$dbForProject = new Database(
$pools
->get($database)
->pop()
->getResource(),
$cache
);
$dbForProject->setNamespace('_' . $projectInternalId);
switch (true) {
case $document->getCollection() === 'users': // users
$sessions = count($document->getAttribute(METRIC_SESSIONS, 0));
if (!empty($sessions)) {
$metrics[] = [
'key' => METRIC_SESSIONS,
'value' => ($sessions * -1),
];
}
break;
case $document->getCollection() === 'databases': // databases
$collections = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
$documents = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
if (!empty($collections['value'])) {
$metrics[] = [
'key' => METRIC_COLLECTIONS,
'value' => ($collections['value'] * -1),
];
}
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
}
break;
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$documents = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
$metrics[] = [
'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS),
'value' => ($documents['value'] * -1),
];
}
break;
case $document->getCollection() === 'buckets':
$files = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
$storage = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
if (!empty($files['value'])) {
$metrics[] = [
'key' => METRIC_FILES,
'value' => ($files['value'] * -1),
];
}
if (!empty($storage['value'])) {
$metrics[] = [
'key' => METRIC_FILES_STORAGE,
'value' => ($storage['value'] * -1),
];
}
break;
case $document->getCollection() === 'functions':
$deployments = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['function', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
$deploymentsStorage = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['function', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
$builds = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
$buildsStorage = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
$buildsCompute = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
$executions = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
$executionsCompute = $dbForProject->getDocument('stats', md5(INFINITI_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
if (!empty($deployments['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS,
'value' => ($deployments['value'] * -1),
];
}
if (!empty($deploymentsStorage['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS_STORAGE,
'value' => ($deploymentsStorage['value'] * -1),
];
}
if (!empty($builds['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS,
'value' => ($builds['value'] * -1),
];
}
if (!empty($buildsStorage['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_STORAGE,
'value' => ($buildsStorage['value'] * -1),
];
}
if (!empty($buildsCompute['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_COMPUTE,
'value' => ($buildsCompute['value'] * -1),
];
}
if (!empty($executions['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS,
'value' => ($executionsCompute['value'] * -1),
];
}
if (!empty($executionsCompute['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS_COMPUTE,
'value' => ($executionsCompute['value'] * -1),
];
}
break;
default:
break;
}
} catch (\Exception $e) {
console::error("[reducer] " . " {DateTime::now()} " . " {$projectInternalId} " . " {$e->getMessage()}");
} finally {
$pools->reclaim();
}
};
}, ['cache', 'register', 'pools']);
$server->job()
->inject('message')
->inject('reduce')
->action(function (Message $message, callable $reduce) use (&$stats) {
$payload = $message->getPayload() ?? [];
$project = new Document($payload['project'] ?? []);
$projectId = $project->getInternalId();
foreach ($payload['reduce'] ?? [] as $document) {
if (empty($document)) {
continue;
}
$reduce(
database: $project->getAttribute('database'),
projectInternalId: $project->getInternalId(),
document: new Document($document),
metrics: $payload['metrics'],
);
}
$stats[$projectId]['database'] = $project->getAttribute('database');
foreach ($payload['metrics'] ?? [] as $metric) {
if (!isset($stats[$projectId]['keys'][$metric['key']])) {
$stats[$projectId]['keys'][$metric['key']] = $metric['value'];
continue;
}
$stats[$projectId]['keys'][$metric['key']] += $metric['value'];
}
});
$server
->workerStart()
->inject('register')
->inject('cache')
->inject('pools')
->action(function ($register, $cache, $pools) use ($periods, &$stats) {
Timer::tick(30000, function () use ($register, $cache, $pools, $periods, &$stats) {
$offset = count($stats);
$projects = array_slice($stats, 0, $offset, true);
array_splice($stats, 0, $offset);
foreach ($projects as $projectInternalId => $project) {
try {
$dbForProject = new Database(
$pools
->get($project['database'])
->pop()
->getResource(),
$cache
);
$dbForProject->setNamespace('_' . $projectInternalId);
foreach ($project['keys'] ?? [] as $key => $value) {
if ($value == 0) {
continue;
}
foreach ($periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
try {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => App::getEnv('_APP_REGION', 'default'),
]));
} catch (Duplicate $th) {
if ($value < 0) {
$dbForProject->decreaseDocumentAttribute(
'stats',
$id,
'value',
abs($value)
);
} else {
$dbForProject->increaseDocumentAttribute(
'stats',
$id,
'value',
$value
);
}
}
}
}
if (!empty($project['keys'])) {
$dbForProject->createDocument('statsLogger', new Document([
'time' => DateTime::now(),
'metrics' => $project['keys'],
]));
}
} catch (\Exception $e) {
console::error("[logger] " . " {DateTime::now()} " . " {$projectInternalId} " . " {$e->getMessage()}");
} finally {
$pools->reclaim();
}
}
});
});
$server->start();
bin/usage
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php usage $@
bin/worker-usage
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
QUEUE=v1-usage php /usr/src/code/app/workers/usage.php $@
composer.json
+4 -6
View File
@@ -43,13 +43,13 @@
"ext-sockets": "*",
"appwrite/php-clamav": "1.1.*",
"appwrite/php-runtimes": "0.11.*",
"utopia-php/abuse": "0.16.*",
"utopia-php/abuse": "0.18.*",
"utopia-php/analytics": "0.2.*",
"utopia-php/audit": "0.17.*",
"utopia-php/audit": "0.20.*",
"utopia-php/cache": "0.8.*",
"utopia-php/cli": "0.14.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.28.*",
"utopia-php/database": "0.30.*",
"utopia-php/queue": "0.5.*",
"utopia-php/orchestration": "0.9.*",
"utopia-php/platform": "0.3.*",
@@ -69,11 +69,9 @@
"resque/php-resque": "1.3.6",
"matomo/device-detector": "6.0.0",
"dragonmantank/cron-expression": "3.3.1",
"influxdb/influxdb-php": "1.15.2",
"phpmailer/phpmailer": "6.6.0",
"chillerlan/php-qrcode": "4.3.3",
"adhocore/jwt": "1.1.2",
"slickdeals/statsd": "3.1.0"
"adhocore/jwt": "1.1.2"
},
"repositories": [
{
composer.lock
Generated
+301 -708
View File
File diff suppressed because it is too large Load Diff
docker-compose.yml
+11 -60
View File
@@ -54,7 +54,7 @@ services:
TESTING: true
VERSION: dev
VITE_CONSOLE_MODE: self-hosted
ports:
ports:
- 9501:80
networks:
- appwrite
@@ -136,8 +136,6 @@ services:
- _APP_SMTP_USERNAME
- _APP_SMTP_PASSWORD
- _APP_USAGE_STATS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_STORAGE_LIMIT
- _APP_STORAGE_PREVIEW_LIMIT
- _APP_STORAGE_ANTIVIRUS
@@ -154,8 +152,6 @@ services:
- _APP_EXECUTOR_HOST
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_STATSD_HOST
- _APP_STATSD_PORT
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
@@ -580,22 +576,20 @@ services:
- _APP_MAINTENANCE_RETENTION_CACHE
- _APP_MAINTENANCE_RETENTION_ABUSE
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_MAINTENANCE_RETENTION_SCHEDULES
appwrite-usage:
entrypoint: usage
appwrite-worker-usage:
entrypoint: worker-usage
<<: *x-logging
container_name: appwrite-usage
container_name: appwrite-worker-usage
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./dev:/usr/local/dev
depends_on:
- influxdb
- redis
- mariadb
environment:
- _APP_ENV
@@ -608,9 +602,6 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@@ -618,8 +609,10 @@ services:
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_CONNECTIONS_QUEUE
- _APP_USAGE_STATS
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
appwrite-schedule:
entrypoint: schedule
@@ -732,28 +725,8 @@ services:
# - appwrite
# volumes:
# - appwrite-uploads:/storage/uploads
influxdb:
image: appwrite/influxdb:1.5.0
container_name: appwrite-influxdb
<<: *x-logging
networks:
- appwrite
volumes:
- appwrite-influxdb:/var/lib/influxdb:rw
telegraf:
image: appwrite/telegraf:1.4.0
container_name: appwrite-telegraf
<<: *x-logging
networks:
- appwrite
environment:
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
# Dev Tools Start ------------------------------------------------------------------------------------------
#
#
# The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack
#
# Here is a description of the different tools and why are we using them:
@@ -762,7 +735,6 @@ services:
# RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks
# RedisCommander - A nice UI for exploring Redis data
# Resque - A nice UI for exploring Redis pub/sub, view the different queues workloads, pending and failed tasks
# Chronograf - A nice UI for exploring InfluxDB data
# Webgrind - A nice UI for exploring and debugging code-level stuff
maildev: # used mainly for dev tests
@@ -825,25 +797,6 @@ services:
# - RESQUE_WEB_PORT=6379
# - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user
# - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password
# chronograf:
# image: chronograf:1.6
# container_name: appwrite-chronograf
# networks:
# - appwrite
# volumes:
# - appwrite-chronograf:/var/lib/chronograf
# ports:
# - "8888:8888"
# environment:
# - INFLUXDB_URL=http://influxdb:8086
# - KAPACITOR_URL=http://kapacitor:9092
# - AUTH_DURATION=48h
# - TOKEN_SECRET=duperduper5674829!jwt
# - GH_CLIENT_ID=d86f7145a41eacfc52cc
# - GH_CLIENT_SECRET=9e0081062367a2134e7f2ea95ba1a32d08b6c8ab
# - GH_ORGS=appwrite
# webgrind:
# image: 'jokkedk/webgrind:latest'
# volumes:
@@ -867,8 +820,6 @@ volumes:
appwrite-cache:
appwrite-uploads:
appwrite-certificates:
appwrite-influxdb:
appwrite-config:
appwrite-functions:
appwrite-builds:
# appwrite-chronograf:
appwrite-config:
docs/specs/overview.drawio.svg
+4 -880
View File
File diff suppressed because one or more lines are too long
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 76 KiB

After

Width:  |  Height:  |  Size: 57 KiB

docs/tutorials/add-environment-variable.md
+1 -1
View File
@@ -12,7 +12,7 @@ This tutorial will cover how to properly add a new environment variable in Appwr
### Naming environment variable
The environment variables in Appwrite are prefixed with `_APP_`. If it belongs to a specific category, the category name is appended as `_APP_REDIS` for the Redis category. The available categories are General, Redis, MariaDB, InfluxDB, StatsD, SMTP, Storage, Functions and Maintenance. Finally, a properly describing name is given to the variable. For example, `_APP_REDIS_HOST` is an environment variable for the hostname of your Redis instance. You can find more information on available categories and existing environment variables in the [environment variables doc](https://appwrite.io/docs/environment-variables).
The environment variables in Appwrite are prefixed with `_APP_`. If it belongs to a specific category, the category name is appended as `_APP_REDIS` for the Redis category. The available categories are General, Redis, MariaDB, SMTP, Storage, Functions and Maintenance. Finally, a properly describing name is given to the variable. For example, `_APP_REDIS_HOST` is an environment variable for the hostname of your Redis instance. You can find more information on available categories and existing environment variables in the [environment variables doc](https://appwrite.io/docs/environment-variables).
### Describe new environment variable
docs/tutorials/add-route.md
-5
View File
@@ -127,11 +127,6 @@ App::post('/v1/storage/buckets/:bucketId/files')
->label('event', 'buckets.[bucketId].files.[fileId].create')
```
#### Usage
- usage.metric - The metric the route generates.
- usage.params - Additional parameters the metrics can have.
```php
App::post('/v1/storage/buckets/:bucketId/files')
->label('usage.metric', 'files.{scope}.requests.create')
docs/tutorials/multi-architecture-support.md
-2
View File
@@ -6,8 +6,6 @@ A list of Appwrite CPU architecture support status. We use this list to track th
|---|---|---|---|---|---|---|---|
| appwrite/appwrite | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 |
| appwrite/mariadb | 🟢 | 🟢 | 🔴 | 🔴 | 🔴 | 🟢 | 🔴 |
| appwrite/influxdb | 🟢 | 🟢 | 🟢 | 🟢 | 🔴 | 🔴 | 🔴 |
| appwrite/telegraf | 🟢 | 🟢 | 🟢 | 🟢 | 🔴 | 🔴 | 🔴 |
| appwrite/clamav | 🟢 | 🟢 | 🟢 | 🟢 | 🔴 | 🟢 | 🟢 |
| traefik | 🟢 | 🔴 | 🟢 | 🔴 | 🟢 | 🔴 | 🔴 |
| redis | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 | 🟢 |
src/Appwrite/Auth/Auth.php
+1 -2
View File
@@ -9,10 +9,9 @@ use Appwrite\Auth\Hash\Phpass;
use Appwrite\Auth\Hash\Scrypt;
use Appwrite\Auth\Hash\Scryptmodified;
use Appwrite\Auth\Hash\Sha;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\DateTime;
use Utopia\Database\Role;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\Roles;
src/Appwrite/Event/Event.php
+3
View File
@@ -23,6 +23,9 @@ class Event
public const FUNCTIONS_QUEUE_NAME = 'v1-functions';
public const FUNCTIONS_CLASS_NAME = 'FunctionsV1';
public const USAGE_QUEUE_NAME = 'v1-usage';
public const USAGE_CLASS_NAME = 'UsageV1';
public const WEBHOOK_QUEUE_NAME = 'v1-webhooks';
public const WEBHOOK_CLASS_NAME = 'WebhooksV1';
src/Appwrite/Event/Usage.php
+64
View File
@@ -0,0 +1,64 @@
<?php
namespace Appwrite\Event;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
use Utopia\Database\Document;
class Usage extends Event
{
protected array $metrics = [];
protected array $reduce = [];
public function __construct(protected Connection $connection)
{
parent::__construct(Event::USAGE_QUEUE_NAME, Event::USAGE_CLASS_NAME);
}
/**
* Add reduce.
*
* @param Document $document
* @return self
*/
public function addReduce(Document $document): self
{
$this->reduce[] = $document;
return $this;
}
/**
* Add metric.
*
* @param string $key
* @param int $value
* @return self
*/
public function addMetric(string $key, int $value): self
{
$this->metrics[] = [
'key' => $key,
'value' => $value,
];
return $this;
}
/**
* Sends metrics to the usage worker.
*
* @return string|bool
*/
public function trigger(): string|bool
{
$client = new Client($this->queue, $this->connection);
return $client->enqueue([
'project' => $this->getProject(),
'reduce' => $this->reduce,
'metrics' => $this->metrics,
]);
}
}
src/Appwrite/Messaging/Adapter/Realtime.php
+2 -2
View File
@@ -6,8 +6,8 @@ use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Appwrite\Messaging\Adapter;
use Utopia\App;
use Utopia\Database\ID;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Role;
class Realtime extends Adapter
{
src/Appwrite/Migration/Migration.php
+1 -1
View File
@@ -10,7 +10,7 @@ use Utopia\CLI\Console;
use Utopia\Config\Config;
use Exception;
use Utopia\App;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Validator\Authorization;
Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
src/Appwrite/Migration/Version/V15.php
+3 -3
View File
@@ -11,9 +11,9 @@ use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
class V15 extends Migration
{
src/Appwrite/Migration/Version/V17.php
+12
View File
@@ -99,6 +99,18 @@ class V17 extends Migration
}
break;
case 'stats':
try {
/**
* Delete 'type' attribute
*/
$this->projectDB->deleteAttribute($id, 'type');
$this->projectDB->deleteCachedCollection($id);
} catch (\Throwable $th) {
Console::warning("'type' from {$id}: {$th->getMessage()}");
}
break;
case 'schedules':
try {
/**
src/Appwrite/Platform/Services/Tasks.php
-2
View File
@@ -12,7 +12,6 @@ use Appwrite\Platform\Tasks\PatchCreateMissingSchedules;
use Appwrite\Platform\Tasks\SDKs;
use Appwrite\Platform\Tasks\Specs;
use Appwrite\Platform\Tasks\SSL;
use Appwrite\Platform\Tasks\Usage;
use Appwrite\Platform\Tasks\Vars;
use Appwrite\Platform\Tasks\Version;
use Appwrite\Platform\Tasks\VolumeSync;
@@ -24,7 +23,6 @@ class Tasks extends Service
$this->type = self::TYPE_CLI;
$this
->addAction(Version::getName(), new Version())
->addAction(Usage::getName(), new Usage())
->addAction(Vars::getName(), new Vars())
->addAction(SSL::getName(), new SSL())
->addAction(Doctor::getName(), new Doctor())
src/Appwrite/Platform/Tasks/Doctor.php
-20
View File
@@ -187,26 +187,6 @@ class Doctor extends Action
Console::error('🔴 ' . str_pad("SMTP", 47, '.') . 'disconnected');
}
$host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
$port = App::getEnv('_APP_STATSD_PORT', 8125);
if ($fp = @\fsockopen('udp://' . $host, $port, $errCode, $errStr, 2)) {
Console::success('🟢 ' . str_pad("StatsD", 50, '.') . 'connected');
\fclose($fp);
} else {
Console::error('🔴 ' . str_pad("StatsD", 47, '.') . 'disconnected');
}
$host = App::getEnv('_APP_INFLUXDB_HOST', '');
$port = App::getEnv('_APP_INFLUXDB_PORT', '');
if ($fp = @\fsockopen($host, $port, $errCode, $errStr, 2)) {
Console::success('🟢 ' . str_pad("InfluxDB", 50, '.') . 'connected');
\fclose($fp);
} else {
Console::error('🔴 ' . str_pad("InfluxDB", 47, '.') . 'disconnected');
}
\sleep(0.2);
Console::log('');
src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php
+1 -1
View File
@@ -8,7 +8,7 @@ use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Validator\Authorization;
class PatchCreateMissingSchedules extends Action
src/Appwrite/Platform/Tasks/SDKs.php
+8 -8
View File
@@ -65,19 +65,19 @@ class SDKs extends Action
Console::info('Fetching API Spec for ' . $language['name'] . ' for ' . $platform['name'] . ' (version: ' . $version . ')');
$spec = file_get_contents(__DIR__ . '/../../../app/config/specs/swagger2-' . $version . '-' . $language['family'] . '.json');
$spec = file_get_contents(__DIR__ . '/../../../../app/config/specs/swagger2-' . $version . '-' . $language['family'] . '.json');
$cover = 'https://appwrite.io/images/github.png';
$result = \realpath(__DIR__ . '/../../../app') . '/sdks/' . $key . '-' . $language['key'];
$resultExamples = \realpath(__DIR__ . '/../../..') . '/docs/examples/' . $version . '/' . $key . '-' . $language['key'];
$target = \realpath(__DIR__ . '/../../../app') . '/sdks/git/' . $language['key'] . '/';
$readme = \realpath(__DIR__ . '/../../../docs/sdks/' . $language['key'] . '/README.md');
$result = \realpath(__DIR__ . '/../../../../app') . '/sdks/' . $key . '-' . $language['key'];
$resultExamples = \realpath(__DIR__ . '/../../../..') . '/docs/examples/' . $version . '/' . $key . '-' . $language['key'];
$target = \realpath(__DIR__ . '/../../../../app') . '/sdks/git/' . $language['key'] . '/';
$readme = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/README.md');
$readme = ($readme) ? \file_get_contents($readme) : '';
$gettingStarted = \realpath(__DIR__ . '/../../../docs/sdks/' . $language['key'] . '/GETTING_STARTED.md');
$gettingStarted = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/GETTING_STARTED.md');
$gettingStarted = ($gettingStarted) ? \file_get_contents($gettingStarted) : '';
$examples = \realpath(__DIR__ . '/../../../docs/sdks/' . $language['key'] . '/EXAMPLES.md');
$examples = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/EXAMPLES.md');
$examples = ($examples) ? \file_get_contents($examples) : '';
$changelog = \realpath(__DIR__ . '/../../../docs/sdks/' . $language['key'] . '/CHANGELOG.md');
$changelog = \realpath(__DIR__ . '/../../../../docs/sdks/' . $language['key'] . '/CHANGELOG.md');
$changelog = ($changelog) ? \file_get_contents($changelog) : '# Change Log';
$warning = '**This SDK is compatible with Appwrite server version ' . $version . '. For older versions, please check [previous releases](' . $language['url'] . '/releases).**';
$license = 'BSD-3-Clause';
src/Appwrite/Platform/Tasks/Specs.php
+3 -3
View File
@@ -40,8 +40,8 @@ class Specs extends Action
public function action(string $version, string $mode, Registry $register): void
{
$db = $register->get('db');
$redis = $register->get('cache');
//$db = $register->get('db');
//$redis = $register->get('cache');
$appRoutes = App::getRoutes();
$response = new Response(new HttpResponse());
$mocks = ($mode === 'mocks');
@@ -266,7 +266,7 @@ class Specs extends Action
continue;
}
$path = __DIR__ . '/../../../app/config/specs/' . $format . '-' . $version . '-' . $platform . '.json';
$path = __DIR__ . '/../../../../app/config/specs/' . $format . '-' . $version . '-' . $platform . '.json';
if (!file_put_contents($path, json_encode($specs->parse()))) {
throw new Exception('Failed to save spec file: ' . $path);
src/Appwrite/Platform/Tasks/Usage.php
-60
View File
@@ -1,60 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Usage\Calculators\TimeSeries;
use InfluxDB\Database as InfluxDatabase;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Database as UtopiaDatabase;
use Throwable;
use Utopia\Platform\Action;
use Utopia\Registry\Registry;
class Usage extends Action
{
public static function getName(): string
{
return 'usage';
}
public function __construct()
{
$this
->desc('Schedules syncing data from influxdb to Appwrite console db')
->inject('dbForConsole')
->inject('influxdb')
->inject('register')
->inject('getProjectDB')
->inject('logError')
->callback(fn ($dbForConsole, $influxDB, $register, $getProjectDB, $logError) => $this->action($dbForConsole, $influxDB, $register, $getProjectDB, $logError));
}
protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void
{
}
public function action(UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register, callable $getProjectDB, callable $logError)
{
Console::title('Usage Aggregation V1');
Console::success(APP_NAME . ' usage aggregation process v1 has started');
$errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action);
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
$region = App::getEnv('region', 'default');
$usage = new TimeSeries($region, $dbForConsole, $influxDB, $getProjectDB, $register, $errorLogger);
Console::loop(function () use ($interval, $usage) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds");
$loopStart = microtime(true);
$usage->collect();
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
}, $interval);
}
}
src/Appwrite/Resque/Worker.php
+24 -5
View File
@@ -2,6 +2,7 @@
namespace Appwrite\Resque;
use Appwrite\Event\Usage;
use Exception;
use Utopia\App;
use Utopia\Cache\Cache;
@@ -9,6 +10,7 @@ use Utopia\Config\Config;
use Utopia\Cache\Adapter\Sharding;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Pools\Group;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Device\DOSpaces;
@@ -141,7 +143,7 @@ abstract class Worker
global $register;
try {
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools'); /** @var Group $pools */
$pools->reclaim();
$this->shutdown();
@@ -176,7 +178,7 @@ abstract class Worker
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools'); /** @var Group $pools */
if ($project->isEmpty() || $project->getId() === 'console') {
return $this->getConsoleDB();
@@ -213,7 +215,7 @@ abstract class Worker
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools'); /** @var Group $pools */
$dbAdapter = $pools
->get('console')
@@ -237,7 +239,7 @@ abstract class Worker
{
global $register;
$pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */
$pools = $register->get('pools'); /** @var Group $pools */
$list = Config::getParam('pools-cache', []);
$adapters = [];
@@ -253,6 +255,24 @@ abstract class Worker
return new Cache(new Sharding($adapters));
}
/**
* Get usage queue
* @return Usage
* @throws Exception
*/
protected function getUsageQueue(): Usage
{
global $register;
$pools = $register->get('pools'); /** @var Group $pools */
$queue = $pools
->get('queue')
->pop()
->getResource();
return new Usage($queue);
}
/**
* Get Functions Storage Device
* @param string $projectId of the project
@@ -291,7 +311,6 @@ abstract class Worker
public function getDevice($root): Device
{
$connection = App::getEnv('_APP_CONNECTIONS_STORAGE', '');
$acl = 'private';
$device = Storage::DEVICE_LOCAL;
$accessKey = '';
src/Appwrite/Specification/Format/OpenAPI3.php
+2 -2
View File
@@ -5,8 +5,8 @@ namespace Appwrite\Specification\Format;
use Appwrite\Specification\Format;
use Appwrite\Template\Template;
use Appwrite\Utopia\Response\Model;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Validator;
class OpenAPI3 extends Format
src/Appwrite/Specification/Format/Swagger2.php
+2 -2
View File
@@ -5,8 +5,8 @@ namespace Appwrite\Specification\Format;
use Appwrite\Specification\Format;
use Appwrite\Template\Template;
use Appwrite\Utopia\Response\Model;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Validator;
class Swagger2 extends Format
src/Appwrite/Usage/Calculator.php
-15
View File
@@ -1,15 +0,0 @@
<?php
namespace Appwrite\Usage;
abstract class Calculator
{
protected string $region;
public function __construct(string $region)
{
$this->region = $region;
}
abstract public function collect(): void;
}
src/Appwrite/Usage/Calculators/TimeSeries.php
-557
View File
@@ -1,557 +0,0 @@
<?php
namespace Appwrite\Usage\Calculators;
use Utopia\App;
use Appwrite\Usage\Calculator;
use Utopia\Database\Database;
use Utopia\Database\Document;
use InfluxDB\Database as InfluxDatabase;
use DateTime;
use Utopia\Registry\Registry;
class TimeSeries extends Calculator
{
/**
* InfluxDB
*
* @var InfluxDatabase
*/
protected InfluxDatabase $influxDB;
/**
* Utopia Database
*
* @var Database
*/
protected Database $database;
/**
* Error Handler Callback
*
* @var callable
*/
protected $errorHandler;
/**
* Callback to get project DB
*
* @var callable
*/
protected mixed $getProjectDB;
/**
* Registry
*
* @var Registry
*/
protected Registry $register;
/**
* Latest times for metric that was synced to the database
*
* @var array
*/
private array $latestTime = [];
/**
* Periods the metrics are collected for
* @var array
*/
protected array $periods = [
[
'key' => '1h',
'startTime' => '-24 hours'
],
[
'key' => '1d',
'startTime' => '-30 days'
]
];
/**
* All the metrics that we are collecting
*
* @var array
*/
protected array $metrics = [
'project.$all.network.requests' => [
'table' => 'appwrite_usage_project_{scope}_network_requests',
],
'project.$all.network.bandwidth' => [
'table' => 'appwrite_usage_project_{scope}_network_bandwidth',
],
'project.$all.network.inbound' => [
'table' => 'appwrite_usage_project_{scope}_network_inbound',
],
'project.$all.network.outbound' => [
'table' => 'appwrite_usage_project_{scope}_network_outbound',
],
/* Users service metrics */
'users.$all.requests.create' => [
'table' => 'appwrite_usage_users_{scope}_requests_create',
],
'users.$all.requests.read' => [
'table' => 'appwrite_usage_users_{scope}_requests_read',
],
'users.$all.requests.update' => [
'table' => 'appwrite_usage_users_{scope}_requests_update',
],
'users.$all.requests.delete' => [
'table' => 'appwrite_usage_users_{scope}_requests_delete',
],
'databases.$all.requests.create' => [
'table' => 'appwrite_usage_databases_{scope}_requests_create',
],
'databases.$all.requests.read' => [
'table' => 'appwrite_usage_databases_{scope}_requests_read',
],
'databases.$all.requests.update' => [
'table' => 'appwrite_usage_databases_{scope}_requests_update',
],
'databases.$all.requests.delete' => [
'table' => 'appwrite_usage_databases_{scope}_requests_delete',
],
'collections.$all.requests.create' => [
'table' => 'appwrite_usage_collections_{scope}_requests_create',
],
'collections.$all.requests.read' => [
'table' => 'appwrite_usage_collections_{scope}_requests_read',
],
'collections.$all.requests.update' => [
'table' => 'appwrite_usage_collections_{scope}_requests_update',
],
'collections.$all.requests.delete' => [
'table' => 'appwrite_usage_collections_{scope}_requests_delete',
],
'documents.$all.requests.create' => [
'table' => 'appwrite_usage_documents_{scope}_requests_create',
],
'documents.$all.requests.read' => [
'table' => 'appwrite_usage_documents_{scope}_requests_read',
],
'documents.$all.requests.update' => [
'table' => 'appwrite_usage_documents_{scope}_requests_update',
],
'documents.$all.requests.delete' => [
'table' => 'appwrite_usage_documents_{scope}_requests_delete',
],
'collections.databaseId.requests.create' => [
'table' => 'appwrite_usage_collections_{scope}_requests_create',
'groupBy' => ['databaseId'],
],
'collections.databaseId.requests.read' => [
'table' => 'appwrite_usage_collections_{scope}_requests_read',
'groupBy' => ['databaseId'],
],
'collections.databaseId.requests.update' => [
'table' => 'appwrite_usage_collections_{scope}_requests_update',
'groupBy' => ['databaseId'],
],
'collections.databaseId.requests.delete' => [
'table' => 'appwrite_usage_collections_{scope}_requests_delete',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.create' => [
'table' => 'appwrite_usage_documents_{scope}_requests_create',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.read' => [
'table' => 'appwrite_usage_documents_{scope}_requests_read',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.update' => [
'table' => 'appwrite_usage_documents_{scope}_requests_update',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.delete' => [
'table' => 'appwrite_usage_documents_{scope}_requests_delete',
'groupBy' => ['databaseId'],
],
'documents.databaseId/collectionId.requests.create' => [
'table' => 'appwrite_usage_documents_{scope}_requests_create',
'groupBy' => ['databaseId', 'collectionId'],
],
'documents.databaseId/collectionId.requests.read' => [
'table' => 'appwrite_usage_documents_{scope}_requests_read',
'groupBy' => ['databaseId', 'collectionId'],
],
'documents.databaseId/collectionId.requests.update' => [
'table' => 'appwrite_usage_documents_{scope}_requests_update',
'groupBy' => ['databaseId', 'collectionId'],
],
'documents.databaseId/collectionId.requests.delete' => [
'table' => 'appwrite_usage_documents_{scope}_requests_delete',
'groupBy' => ['databaseId', 'collectionId'],
],
'buckets.$all.requests.create' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_create',
],
'buckets.$all.requests.read' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_read',
],
'buckets.$all.requests.update' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_update',
],
'buckets.$all.requests.delete' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_delete',
],
'files.$all.requests.create' => [
'table' => 'appwrite_usage_files_{scope}_requests_create',
],
'files.$all.requests.read' => [
'table' => 'appwrite_usage_files_{scope}_requests_read',
],
'files.$all.requests.update' => [
'table' => 'appwrite_usage_files_{scope}_requests_update',
],
'files.$all.requests.delete' => [
'table' => 'appwrite_usage_files_{scope}_requests_delete',
],
'files.bucketId.requests.create' => [
'table' => 'appwrite_usage_files_{scope}_requests_create',
'groupBy' => ['bucketId'],
],
'files.bucketId.requests.read' => [
'table' => 'appwrite_usage_files_{scope}_requests_read',
'groupBy' => ['bucketId'],
],
'files.bucketId.requests.update' => [
'table' => 'appwrite_usage_files_{scope}_requests_update',
'groupBy' => ['bucketId'],
],
'files.bucketId.requests.delete' => [
'table' => 'appwrite_usage_files_{scope}_requests_delete',
'groupBy' => ['bucketId'],
],
'sessions.$all.requests.create' => [
'table' => 'appwrite_usage_sessions__{scope}_requests_create',
],
'sessions.provider.requests.create' => [
'table' => 'appwrite_usage_sessions_{scope}_requests_create',
'groupBy' => ['provider'],
],
'sessions.$all.requests.delete' => [
'table' => 'appwrite_usage_sessions_{scope}_requests_delete',
],
'executions.$all.compute.total' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
],
'builds.$all.compute.total' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
],
'executions.$all.compute.failure' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'filters' => [
'functionStatus' => 'failed',
],
],
'builds.$all.compute.failure' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'filters' => [
'functionStatus' => 'failed',
],
],
'executions.$all.compute.success' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'filters' => [
'functionStatus' => 'success',
],
],
'builds.$all.compute.success' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'filters' => [
'functionStatus' => 'success',
],
],
'executions.functionId.compute.total' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
],
'builds.functionId.compute.total' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
],
'executions.functionId.compute.failure' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionStatus' => 'failed',
],
],
'builds.functionId.compute.failure' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionBuildStatus' => 'failed',
],
],
'executions.functionId.compute.success' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionStatus' => 'success',
],
],
'builds.functionId.compute.success' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionBuildStatus' => 'success',
],
],
// counters
'users.$all.count.total' => [
'table' => 'appwrite_usage_users_{scope}_count_total',
],
'buckets.$all.count.total' => [
'table' => 'appwrite_usage_buckets_{scope}_count_total',
],
'files.$all.count.total' => [
'table' => 'appwrite_usage_files_{scope}_count_total',
],
'files.bucketId.count.total' => [
'table' => 'appwrite_usage_files_{scope}_count_total',
'groupBy' => ['bucketId']
],
'databases.$all.count.total' => [
'table' => 'appwrite_usage_databases_{scope}_count_total',
],
'collections.$all.count.total' => [
'table' => 'appwrite_usage_collections_{scope}_count_total',
],
'documents.$all.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
],
'collections.databaseId.count.total' => [
'table' => 'appwrite_usage_collections_{scope}_count_total',
'groupBy' => ['databaseId']
],
'documents.databaseId.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
'groupBy' => ['databaseId']
],
'documents.databaseId/collectionId.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
'groupBy' => ['databaseId', 'collectionId']
],
'deployments.$all.storage.size' => [
'table' => 'appwrite_usage_deployments_{scope}_storage_size',
],
'project.$all.storage.size' => [
'table' => 'appwrite_usage_project_{scope}_storage_size',
],
'files.$all.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
],
'files.$bucketId.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
'groupBy' => ['bucketId']
],
'builds.$all.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
],
'executions.$all.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
],
'executions.functionId.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'builds.functionId.compute.time' => [
'table' => 'appwrite_usage_builds_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'project.$all.compute.time' => [ // Built time + execution time
'table' => 'appwrite_usage_project_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'deployments.$all.storage.size' => [
'table' => 'appwrite_usage_deployments_{scope}_storage_size'
],
'project.$all.storage.size' => [
'table' => 'appwrite_usage_project_{scope}_storage_size'
],
'files.$all.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size'
],
'files.bucketId.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
'groupBy' => ['bucketId']
]
];
public function __construct(string $region, Database $database, InfluxDatabase $influxDB, callable $getProjectDB, Registry $register, callable $errorHandler = null)
{
parent::__construct($region);
$this->database = $database;
$this->influxDB = $influxDB;
$this->getProjectDB = $getProjectDB;
$this->register = $register;
$this->errorHandler = $errorHandler;
}
/**
* Create or Update Mertic
* Create or update each metric in the stats collection for the given project
*
* @param string $projectId
* @param int $time
* @param string $period
* @param string $metric
* @param int $value
* @param int $type
*
* @return void
*/
private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void
{
$id = \md5("{$time}_{$period}_{$metric}");
$project = $this->database->getDocument('projects', $projectId);
$database = call_user_func($this->getProjectDB, $project);
try {
$document = $database->getDocument('stats', $id);
if ($document->isEmpty()) {
$database->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $metric,
'value' => $value,
'type' => $type,
'region' => $this->region,
]));
} else {
$database->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $value)
);
}
} catch (\Exception $e) { // if projects are deleted this might fail
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}");
} else {
throw $e;
}
}
$this->register->get('pools')->reclaim();
}
/**
* Sync From InfluxDB
* Sync stats from influxDB to stats collection in the Appwrite database
*
* @param string $metric
* @param array $options
* @param array $period
*
* @return void
*/
private function syncFromInfluxDB(string $metric, array $options, array $period): void
{
$start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339);
if (!empty($this->latestTime[$metric][$period['key']])) {
$start = $this->latestTime[$metric][$period['key']];
}
$end = (new DateTime())->format(DateTime::RFC3339);
$table = $options['table']; //Which influxdb table to query for this metric
$groupBy = empty($options['groupBy']) ? '' : ', ' . implode(', ', array_map(fn($groupBy) => '"' . $groupBy . '" ', $options['groupBy'])); //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc
$filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status
if (!empty($filters)) {
$filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters)));
} else {
$filters = '';
}
$query = "SELECT sum(value) AS \"value\" ";
$query .= "FROM \"{$table}\" ";
$query .= "WHERE \"time\" > '{$start}' ";
$query .= "AND \"time\" < '{$end}' ";
$query .= "AND \"metric_type\"='counter' {$filters} ";
$query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} ";
$query .= "FILL(null)";
try {
$result = $this->influxDB->query($query);
$points = $result->getPoints();
foreach ($points as $point) {
$projectId = $point['projectId'];
if (!empty($projectId) && $projectId !== 'console') {
$metricUpdated = $metric;
if (!empty($groupBy)) {
foreach ($options['groupBy'] as $groupBy) {
$groupedBy = $point[$groupBy] ?? '';
if (empty($groupedBy)) {
continue;
}
$metricUpdated = str_replace($groupBy, $groupedBy, $metricUpdated);
}
}
$value = (!empty($point['value'])) ? $point['value'] : 0;
$this->createOrUpdateMetric(
$point['projectId'],
$point['time'],
$period['key'],
$metricUpdated,
$value,
0
);
$this->latestTime[$metric][$period['key']] = $point['time'];
}
}
} catch (\Exception $e) { // if projects are deleted this might fail
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e, "sync_metric_{$metric}_influxdb");
} else {
throw $e;
}
}
}
/**
* Collect Stats
* Collect all the stats from Influd DB to Database
*
* @return void
*/
public function collect(): void
{
foreach ($this->periods as $period) {
foreach ($this->metrics as $metric => $options) { //for each metrics
try {
$this->syncFromInfluxDB($metric, $options, $period);
} catch (\Exception $e) {
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e);
} else {
throw $e;
}
}
}
}
}
}
src/Appwrite/Usage/Stats.php
-227
View File
@@ -1,227 +0,0 @@
<?php
namespace Appwrite\Usage;
use Utopia\App;
class Stats
{
/**
* @var array
*/
protected $params = [];
/**
* @var mixed
*/
protected $statsd;
/**
* @var string
*/
protected $namespace = 'appwrite.usage';
/**
* Event constructor.
*
* @param mixed $statsd
*/
public function __construct($statsd)
{
$this->statsd = $statsd;
}
/**
* @param string $key
* @param mixed $value
*
* @return $this
*/
public function setParam(string $key, $value): self
{
$this->params[$key] = $value;
return $this;
}
/**
* @param string $key
*
* @return mixed|null
*/
public function getParam(string $key)
{
return (isset($this->params[$key])) ? $this->params[$key] : null;
}
/**
* @param string $namespace
*
* @return $this
*/
public function setNamespace(string $namespace): self
{
$this->namespace = $namespace;
return $this;
}
/**
* @return string
*/
public function getNamespace()
{
return $this->namespace;
}
/**
* Submit data to StatsD.
* Send various metrics to StatsD based on the parameters that are set
* @return void
*/
public function submit(): void
{
$projectId = $this->params['projectId'] ?? '';
$projectInternalId = $this->params['projectInternalId'];
$tags = ",projectInternalId={$projectInternalId},projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN');
// the global namespace is prepended to every key (optional)
$this->statsd->setNamespace($this->namespace);
$httpRequest = $this->params['project.{scope}.network.requests'] ?? 0;
$httpMethod = $this->params['httpMethod'] ?? '';
if ($httpRequest >= 1) {
$this->statsd->increment('project.{scope}.network.requests' . $tags . ',method=' . \strtolower($httpMethod));
}
$inbound = $this->params['project.{scope}.network.inbound'] ?? 0;
$outbound = $this->params['project.{scope}.network.outbound'] ?? 0;
$this->statsd->count('project.{scope}.network.inbound' . $tags, $inbound);
$this->statsd->count('project.{scope}.network.outbound' . $tags, $outbound);
$this->statsd->count('project.{scope}.network.bandwidth' . $tags, $inbound + $outbound);
$usersMetrics = [
'users.{scope}.requests.create',
'users.{scope}.requests.read',
'users.{scope}.requests.update',
'users.{scope}.requests.delete',
'users.{scope}.count.total',
];
foreach ($usersMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value === 1 || $value === -1) {
$this->statsd->count($metric . $tags, $value);
}
}
$dbMetrics = [
'databases.{scope}.requests.create',
'databases.{scope}.requests.read',
'databases.{scope}.requests.update',
'databases.{scope}.requests.delete',
'collections.{scope}.requests.create',
'collections.{scope}.requests.read',
'collections.{scope}.requests.update',
'collections.{scope}.requests.delete',
'documents.{scope}.requests.create',
'documents.{scope}.requests.read',
'documents.{scope}.requests.update',
'documents.{scope}.requests.delete',
'databases.{scope}.count.total',
'collections.{scope}.count.total',
'documents.{scope}.count.total'
];
foreach ($dbMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value === 1 || $value === -1) {
$dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? '');
$this->statsd->count($metric . $dbTags, $value);
}
}
$storageMertics = [
'buckets.{scope}.requests.create',
'buckets.{scope}.requests.read',
'buckets.{scope}.requests.update',
'buckets.{scope}.requests.delete',
'files.{scope}.requests.create',
'files.{scope}.requests.read',
'files.{scope}.requests.update',
'files.{scope}.requests.delete',
'buckets.{scope}.count.total',
'files.{scope}.count.total',
'files.{scope}.storage.size'
];
foreach ($storageMertics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value !== 0) {
$storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? '');
$this->statsd->count($metric . $storageTags, $value);
}
}
$sessionsMetrics = [
'sessions.{scope}.requests.create',
'sessions.{scope}.requests.update',
'sessions.{scope}.requests.delete',
];
foreach ($sessionsMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value >= 1) {
$sessionTags = $tags . ",provider=" . ($this->params['provider'] ?? '');
$this->statsd->count($metric . $sessionTags, $value);
}
}
$functionId = $this->params['functionId'] ?? '';
$functionExecution = $this->params['executions.{scope}.compute'] ?? 0;
$functionExecutionTime = ($this->params['executionTime'] ?? 0) * 1000; // ms
$functionExecutionStatus = $this->params['executionStatus'] ?? '';
$functionBuild = $this->params['builds.{scope}.compute'] ?? 0;
$functionBuildTime = ($this->params['buildTime'] ?? 0) * 1000; // ms
$functionBuildSize = ($this->params['buildSize'] ?? 0); // bytes
$functionBuildStatus = $this->params['buildStatus'] ?? '';
$functionCompute = $functionExecutionTime + $functionBuildTime;
$functionTags = $tags . ',functionId=' . $functionId;
$deploymentSize = $this->params['deployment.{scope}.storage.size'] ?? 0;
$storageSize = $this->params['files.{scope}.storage.size'] ?? 0;
if ($deploymentSize + $storageSize > 0 || $deploymentSize + $storageSize <= -1) {
$this->statsd->count('project.{scope}.storage.size' . $tags, $deploymentSize + $storageSize);
}
if ($deploymentSize !== 0) {
$this->statsd->count('deployments.{scope}.storage.size' . $functionTags, $deploymentSize);
}
if ($functionExecution >= 1) {
$this->statsd->increment('executions.{scope}.compute' . $functionTags . ',functionStatus=' . $functionExecutionStatus);
if ($functionExecutionTime > 0) {
$this->statsd->count('executions.{scope}.compute.time' . $functionTags, $functionExecutionTime);
}
}
if ($functionBuild >= 1) {
$this->statsd->increment('builds.{scope}.compute' . $functionTags . ',functionBuildStatus=' . $functionBuildStatus);
$this->statsd->count('builds.{scope}.compute.time' . $functionTags, $functionBuildTime);
$this->statsd->count('builds.{scope}.storage.size' . $functionTags, $functionBuildSize);
}
if ($functionBuild + $functionExecution >= 1) {
$this->statsd->count('project.{scope}.compute.time' . $functionTags, $functionCompute);
}
$this->reset();
}
public function reset(): self
{
$this->params = [];
$this->namespace = 'appwrite.usage';
return $this;
}
}
src/Appwrite/Utopia/Request/Filters/V15.php
+2 -2
View File
@@ -4,9 +4,9 @@ namespace Appwrite\Utopia\Request\Filters;
use Appwrite\Utopia\Request\Filter;
use Utopia\Database\Database;
use Utopia\Database\Permission;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Query;
use Utopia\Database\Role;
use Utopia\Database\Helpers\Role;
class V15 extends Filter
{
src/Appwrite/Utopia/Response/Filters/V15.php
+2 -2
View File
@@ -5,8 +5,8 @@ namespace Appwrite\Utopia\Response\Filters;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Filter;
use Utopia\Database\Database;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
class V15 extends Filter
{
src/Appwrite/Utopia/Response/Model/Execution.php
+1 -1
View File
@@ -4,7 +4,7 @@ namespace Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Response\Model;
use Utopia\Database\Role;
use Utopia\Database\Helpers\Role;
class Execution extends Model
{
src/Appwrite/Utopia/Response/Model/UsageBuckets.php
+1 -29
View File
@@ -16,7 +16,7 @@ class UsageBuckets extends Model
'default' => '',
'example' => '30d',
])
->addRule('filesCount', [
->addRule('filesTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of files in this bucket.',
'default' => [],
@@ -30,34 +30,6 @@ class UsageBuckets extends Model
'example' => [],
'array' => true
])
->addRule('filesCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files deleted.',
'default' => [],
'example' => [],
'array' => true
])
;
}
src/Appwrite/Utopia/Response/Model/UsageCollection.php
+1 -29
View File
@@ -16,41 +16,13 @@ class UsageCollection extends Model
'default' => '',
'example' => '30d',
])
->addRule('documentsCount', [
->addRule('documentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => [],
'array' => true
])
;
}
src/Appwrite/Utopia/Response/Model/UsageDatabase.php
+3 -59
View File
@@ -16,72 +16,16 @@ class UsageDatabase extends Model
'default' => '',
'example' => '30d',
])
->addRule('documentsCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCount', [
->addRule('collectionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCreate', [
->addRule('documentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections delete.',
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
src/Appwrite/Utopia/Response/Model/UsageDatabases.php
+9 -93
View File
@@ -16,111 +16,27 @@ class UsageDatabases extends Model
'default' => '',
'example' => '30d',
])
->addRule('databasesCount', [
->addRule('databasesTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCount', [
->addRule('collectionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections delete.',
'default' => [],
'example' => [],
'array' => true
])
;
}
src/Appwrite/Utopia/Response/Model/UsageFunction.php
+22 -28
View File
@@ -16,30 +16,16 @@ class UsageFunction extends Model
'default' => '',
'example' => '30d',
])
->addRule('executionsTotal', [
->addRule('deploymentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'description' => 'Aggregated stats for number of function deployments.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsFailure', [
->addRule('deploymentsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution failures.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsSuccess', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution successes.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution duration.',
'description' => 'Aggregated stats for function deployments storage.',
'default' => [],
'example' => [],
'array' => true
@@ -51,23 +37,31 @@ class UsageFunction extends Model
'example' => [],
'array' => true
])
->addRule('buildsFailure', [
->addRule('buildsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build failures.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsSuccess', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build successes.',
'description' => 'Aggregated stats for builds storage.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build duration.',
'description' => 'Aggregated stats for function build compute.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution compute.',
'default' => [],
'example' => [],
'array' => true
src/Appwrite/Utopia/Response/Model/UsageFunctions.php
+24 -23
View File
@@ -16,30 +16,23 @@ class UsageFunctions extends Model
'default' => '',
'example' => '30d',
])
->addRule('executionsTotal', [
->addRule('functionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'description' => 'Aggregated stats for number of functions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsFailure', [
->addRule('deploymentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution failures.',
'description' => 'Aggregated stats for number of function deployments.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsSuccess', [
->addRule('deploymentsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution successes.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution duration.',
'description' => 'Aggregated stats for function deployments storage.',
'default' => [],
'example' => [],
'array' => true
@@ -51,23 +44,31 @@ class UsageFunctions extends Model
'example' => [],
'array' => true
])
->addRule('buildsFailure', [
->addRule('buildsStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build failures.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsSuccess', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build successes.',
'description' => 'Aggregated stats for builds storage.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build duration.',
'description' => 'Aggregated stats for function build compute.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution compute.',
'default' => [],
'example' => [],
'array' => true
src/Appwrite/Utopia/Response/Model/UsageProject.php
+7 -7
View File
@@ -16,7 +16,7 @@ class UsageProject extends Model
'default' => '',
'example' => '30d',
])
->addRule('requests', [
->addRule('requestsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of requests.',
'default' => [],
@@ -30,42 +30,42 @@ class UsageProject extends Model
'example' => [],
'array' => true
])
->addRule('executions', [
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function executions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documents', [
->addRule('documentsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databases', [
->addRule('databasesTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of databases.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('users', [
->addRule('usersTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of users.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('storage', [
->addRule('filesStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buckets', [
->addRule('bucketsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of buckets.',
'default' => [],
src/Appwrite/Utopia/Response/Model/UsageStorage.php
+5 -61
View File
@@ -16,79 +16,23 @@ class UsageStorage extends Model
'default' => '',
'example' => '30d',
])
->addRule('storage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of files.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsCount', [
->addRule('bucketsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of buckets.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsCreate', [
->addRule('filesTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets created.',
'description' => 'Aggregated stats for total number of files.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsRead', [
->addRule('filesStorage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files deleted.',
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'default' => [],
'example' => [],
'array' => true
src/Appwrite/Utopia/Response/Model/UsageUsers.php
+3 -44
View File
@@ -16,62 +16,21 @@ class UsageUsers extends Model
'default' => '',
'example' => '30d',
])
->addRule('usersCount', [
->addRule('usersTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of users.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsCreate', [
->addRule('sessionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for sessions created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsProviderCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for sessions created for a provider ( email, anonymous or oauth2 ).',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for sessions deleted.',
'default' => [],
'example' => [],
'array' => true
])
;
}
src/Executor/Executor.php
-1
View File
@@ -4,7 +4,6 @@ namespace Executor;
use Exception;
use Utopia\App;
use Utopia\CLI\Console;
class Executor
{
tests/e2e/General/AbuseTest.php
+3 -3
View File
@@ -8,9 +8,9 @@ use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\SideNone;
use Utopia\App;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
class AbuseTest extends Scope
{
tests/e2e/General/UsageTest.php
+367 -419
View File
File diff suppressed because it is too large Load Diff
tests/e2e/Scopes/ProjectConsole.php
+1 -1
View File
@@ -2,7 +2,7 @@
namespace Tests\E2E\Scopes;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
trait ProjectConsole
{
tests/e2e/Scopes/ProjectCustom.php
+1 -1
View File
@@ -3,7 +3,7 @@
namespace Tests\E2E\Scopes;
use Tests\E2E\Client;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
trait ProjectCustom
{
tests/e2e/Scopes/Scope.php
+1 -1
View File
@@ -5,7 +5,7 @@ namespace Tests\E2E\Scopes;
use Appwrite\Tests\Retryable;
use Tests\E2E\Client;
use PHPUnit\Framework\TestCase;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
abstract class Scope extends TestCase
{
tests/e2e/Services/Account/AccountBase.php
+15 -14
View File
@@ -4,8 +4,9 @@ namespace Tests\E2E\Services\Account;
use Appwrite\Tests\Retry;
use Tests\E2E\Client;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
use Utopia\Database\DateTime;
use Utopia\Database\Validator\DatetimeValidator;
trait AccountBase
{
@@ -34,7 +35,7 @@ trait AccountBase
$this->assertEquals($response['headers']['status-code'], 201);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$this->assertEquals($response['body']['name'], $name);
@@ -198,7 +199,7 @@ trait AccountBase
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$this->assertEquals($response['body']['name'], $name);
@@ -343,7 +344,7 @@ trait AccountBase
$this->assertIsNumeric($response['body']['total']);
$this->assertContains($response['body']['logs'][1]['event'], ["session.create"]);
$this->assertEquals($response['body']['logs'][1]['ip'], filter_var($response['body']['logs'][1]['ip'], FILTER_VALIDATE_IP));
$this->assertEquals(true, DateTime::isValid($response['body']['logs'][1]['time']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['logs'][1]['time']));
$this->assertEquals('Windows', $response['body']['logs'][1]['osName']);
$this->assertEquals('WIN', $response['body']['logs'][1]['osCode']);
@@ -365,7 +366,7 @@ trait AccountBase
$this->assertContains($response['body']['logs'][2]['event'], ["user.create"]);
$this->assertEquals($response['body']['logs'][2]['ip'], filter_var($response['body']['logs'][2]['ip'], FILTER_VALIDATE_IP));
$this->assertEquals(true, DateTime::isValid($response['body']['logs'][2]['time']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['logs'][2]['time']));
$this->assertEquals('Windows', $response['body']['logs'][2]['osName']);
$this->assertEquals('WIN', $response['body']['logs'][2]['osCode']);
@@ -476,7 +477,7 @@ trait AccountBase
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$this->assertEquals($response['body']['name'], $newName);
@@ -543,7 +544,7 @@ trait AccountBase
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([
@@ -633,7 +634,7 @@ trait AccountBase
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $newEmail);
/**
@@ -675,7 +676,7 @@ trait AccountBase
$this->assertEquals($response['headers']['status-code'], 201);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $data['email']);
$this->assertEquals($response['body']['name'], $data['name']);
@@ -817,7 +818,7 @@ trait AccountBase
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEmpty($response['body']['secret']);
$this->assertEquals(true, DateTime::isValid($response['body']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['expire']));
$lastEmail = $this->getLastEmail();
@@ -1119,7 +1120,7 @@ trait AccountBase
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEmpty($response['body']['secret']);
$this->assertEquals(true, DateTime::isValid($response['body']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['expire']));
$lastEmail = $this->getLastEmail();
@@ -1273,7 +1274,7 @@ trait AccountBase
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEmpty($response['body']['secret']);
$this->assertEquals(true, DateTime::isValid($response['body']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['expire']));
$userId = $response['body']['userId'];
@@ -1379,7 +1380,7 @@ trait AccountBase
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$this->assertTrue($response['body']['emailVerification']);
@@ -1439,7 +1440,7 @@ trait AccountBase
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([
tests/e2e/Services/Account/AccountConsoleClientTest.php
+4 -3
View File
@@ -6,9 +6,10 @@ use Appwrite\Extend\Exception;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectConsole;
use Tests\E2E\Scopes\SideClient;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
use Utopia\Database\DateTime;
use Tests\E2E\Client;
use Utopia\Database\Validator\DatetimeValidator;
class AccountConsoleClientTest extends Scope
{
@@ -72,7 +73,7 @@ class AccountConsoleClientTest extends Scope
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$this->assertEquals($response['body']['name'], $name);
@@ -92,7 +93,7 @@ class AccountConsoleClientTest extends Scope
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$this->assertEquals($response['body']['name'], $name);
}
tests/e2e/Services/Account/AccountCustomClientTest.php
+8 -7
View File
@@ -9,7 +9,8 @@ use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\SideClient;
use Utopia\Database\DateTime;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Validator\DatetimeValidator;
use function sleep;
@@ -475,7 +476,7 @@ class AccountCustomClientTest extends Scope
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([
@@ -726,7 +727,7 @@ class AccountCustomClientTest extends Scope
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEmpty($response['body']['secret']);
$this->assertEquals(true, DateTime::isValid($response['body']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['expire']));
$userId = $response['body']['userId'];
@@ -826,7 +827,7 @@ class AccountCustomClientTest extends Scope
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['phone'], $number);
$this->assertTrue($response['body']['phoneVerification']);
@@ -877,7 +878,7 @@ class AccountCustomClientTest extends Scope
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['email'], $email);
$response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', array_merge([
@@ -919,7 +920,7 @@ class AccountCustomClientTest extends Scope
$this->assertIsArray($response['body']);
$this->assertNotEmpty($response['body']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['registration']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['registration']));
$this->assertEquals($response['body']['phone'], $newPhone);
/**
@@ -968,7 +969,7 @@ class AccountCustomClientTest extends Scope
$this->assertEquals(201, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEmpty($response['body']['secret']);
$this->assertEquals(true, DateTime::isValid($response['body']['expire']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['expire']));
\sleep(2);
tests/e2e/Services/Account/AccountCustomServerTest.php
+1 -1
View File
@@ -6,7 +6,7 @@ use Tests\E2E\Client;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\SideServer;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
class AccountCustomServerTest extends Scope
{
tests/e2e/Services/Databases/DatabasesBase.php
+6 -6
View File
@@ -3,11 +3,11 @@
namespace Tests\E2E\Services\Databases;
use Tests\E2E\Client;
use Utopia\Database\Database;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
use Utopia\Database\DateTime;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\DatetimeValidator;
trait DatabasesBase
{
@@ -1532,8 +1532,8 @@ trait DatabasesBase
$this->assertEquals($databaseId, $document['body']['$databaseId']);
$this->assertEquals($document['body']['title'], 'Thor: Ragnaroc');
$this->assertEquals($document['body']['releaseYear'], 2017);
$this->assertEquals(true, DateTime::isValid($document['body']['$createdAt']));
$this->assertEquals(true, DateTime::isValid($document['body']['birthDay']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($document['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($document['body']['birthDay']));
$this->assertContains(Permission::read(Role::user($this->getUser()['$id'])), $document['body']['$permissions']);
$this->assertContains(Permission::update(Role::user($this->getUser()['$id'])), $document['body']['$permissions']);
$this->assertContains(Permission::delete(Role::user($this->getUser()['$id'])), $document['body']['$permissions']);
tests/e2e/Services/Databases/DatabasesConsoleClientTest.php
+33 -46
View File
@@ -6,9 +6,9 @@ use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Client;
use Tests\E2E\Scopes\SideConsole;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
class DatabasesConsoleClientTest extends Scope
{
@@ -56,47 +56,39 @@ class DatabasesConsoleClientTest extends Scope
/**
* @depends testCreateCollection
*/
// public function testGetDatabaseUsage(array $data)
// {
// $databaseId = $data['databaseId'];
// /**
// * Test for FAILURE
// */
public function testGetDatabaseUsage(array $data)
{
$databaseId = $data['databaseId'];
/**
* Test for FAILURE
*/
// $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([
// 'content-type' => 'application/json',
// 'x-appwrite-project' => $this->getProject()['$id']
// ], $this->getHeaders()), [
// 'range' => '32h'
// ]);
$response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '32h'
]);
// $this->assertEquals(400, $response['headers']['status-code']);
$this->assertEquals(400, $response['headers']['status-code']);
// /**
// * Test for SUCCESS
// */
/**
* Test for SUCCESS
*/
// $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([
// 'content-type' => 'application/json',
// 'x-appwrite-project' => $this->getProject()['$id']
// ], $this->getHeaders()), [
// 'range' => '24h'
// ]);
$response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h'
]);
// $this->assertEquals(200, $response['headers']['status-code']);
// $this->assertEquals(count($response['body']), 11);
// $this->assertEquals($response['body']['range'], '24h');
// $this->assertIsArray($response['body']['documentsCount']);
// $this->assertIsArray($response['body']['collectionsCount']);
// $this->assertIsArray($response['body']['documentsCreate']);
// $this->assertIsArray($response['body']['documentsRead']);
// $this->assertIsArray($response['body']['documentsUpdate']);
// $this->assertIsArray($response['body']['documentsDelete']);
// $this->assertIsArray($response['body']['collectionsCreate']);
// $this->assertIsArray($response['body']['collectionsRead']);
// $this->assertIsArray($response['body']['collectionsUpdate']);
// $this->assertIsArray($response['body']['collectionsDelete']);
// }
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(count($response['body']), 3);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['documentsTotal']);
$this->assertIsArray($response['body']['collectionsTotal']);
}
/**
@@ -136,15 +128,10 @@ class DatabasesConsoleClientTest extends Scope
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(count($response['body']), 6);
$this->assertEquals(count($response['body']), 2);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['documentsCount']);
$this->assertIsArray($response['body']['documentsCreate']);
$this->assertIsArray($response['body']['documentsRead']);
$this->assertIsArray($response['body']['documentsUpdate']);
$this->assertIsArray($response['body']['documentsDelete']);
$this->assertIsArray($response['body']['documentsTotal']);
}
/**
tests/e2e/Services/Databases/DatabasesCustomClientTest.php
+3 -3
View File
@@ -6,9 +6,9 @@ use Tests\E2E\Client;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\SideClient;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
class DatabasesCustomClientTest extends Scope
{
tests/e2e/Services/Databases/DatabasesCustomServerTest.php
+3 -4
View File
@@ -6,10 +6,9 @@ use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\SideServer;
use Tests\E2E\Client;
use Utopia\Database\Database;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
class DatabasesCustomServerTest extends Scope
{
tests/e2e/Services/Databases/DatabasesPermissionsGuestTest.php
+3 -3
View File
@@ -6,9 +6,9 @@ use Tests\E2E\Client;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\SideClient;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\Authorization;
class DatabasesPermissionsGuestTest extends Scope
tests/e2e/Services/Databases/DatabasesPermissionsMemberTest.php
+3 -3
View File
@@ -6,9 +6,9 @@ use Tests\E2E\Client;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\SideClient;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
class DatabasesPermissionsMemberTest extends Scope
{
tests/e2e/Services/Databases/DatabasesPermissionsTeamTest.php
+3 -3
View File
@@ -6,9 +6,9 @@ use Tests\E2E\Client;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\SideClient;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
class DatabasesPermissionsTeamTest extends Scope
{
tests/e2e/Services/Functions/FunctionsConsoleClientTest.php
+10 -11
View File
@@ -6,8 +6,8 @@ use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Client;
use Tests\E2E\Scopes\SideConsole;
use Utopia\Database\ID;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Role;
class FunctionsConsoleClientTest extends Scope
{
@@ -90,16 +90,15 @@ class FunctionsConsoleClientTest extends Scope
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 9);
$this->assertEquals(count($response['body']), 8);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['executionsTotal']);
$this->assertIsArray($response['body']['executionsFailure']);
$this->assertIsArray($response['body']['executionsSuccess']);
$this->assertIsArray($response['body']['executionsTime']);
$this->assertIsArray($response['body']['buildsTotal']);
$this->assertIsArray($response['body']['buildsFailure']);
$this->assertIsArray($response['body']['buildsSuccess']);
$this->assertIsArray($response['body']['buildsTime']);
$this->assertIsArray($response['body']['deployments']);
$this->assertIsArray($response['body']['deploymentsStorage']);
$this->assertIsArray($response['body']['builds']);
$this->assertIsArray($response['body']['buildsStorage']);
$this->assertIsArray($response['body']['buildsCompute']);
$this->assertIsArray($response['body']['executions']);
$this->assertIsArray($response['body']['executionsCompute']);
}
/**
tests/e2e/Services/Functions/FunctionsCustomClientTest.php
+2 -4
View File
@@ -7,10 +7,8 @@ use Tests\E2E\Client;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\SideClient;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\ID;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Role;
class FunctionsCustomClientTest extends Scope
{
tests/e2e/Services/Functions/FunctionsCustomServerTest.php
+13 -15
View File
@@ -8,10 +8,9 @@ use Tests\E2E\Client;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\SideServer;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Validator\DatetimeValidator;
class FunctionsCustomServerTest extends Scope
{
@@ -45,8 +44,8 @@ class FunctionsCustomServerTest extends Scope
$this->assertNotEmpty($response1['body']['$id']);
$this->assertEquals('Test', $response1['body']['name']);
$this->assertEquals('php-8.0', $response1['body']['runtime']);
$this->assertEquals(true, DateTime::isValid($response1['body']['$createdAt']));
$this->assertEquals(true, DateTime::isValid($response1['body']['$updatedAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response1['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response1['body']['$updatedAt']));
$this->assertEquals('', $response1['body']['deployment']);
$this->assertEquals([
'users.*.create',
@@ -328,8 +327,8 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals(200, $response1['headers']['status-code']);
$this->assertNotEmpty($response1['body']['$id']);
$this->assertEquals('Test1', $response1['body']['name']);
$this->assertEquals(true, DateTime::isValid($response1['body']['$createdAt']));
$this->assertEquals(true, DateTime::isValid($response1['body']['$updatedAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response1['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response1['body']['$updatedAt']));
$this->assertEquals('', $response1['body']['deployment']);
$this->assertEquals([
'users.*.update.name',
@@ -369,7 +368,7 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals(202, $deployment['headers']['status-code']);
$this->assertNotEmpty($deployment['body']['$id']);
$this->assertEquals(true, DateTime::isValid($deployment['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($deployment['body']['$createdAt']));
$this->assertEquals('index.php', $deployment['body']['entrypoint']);
// Wait for deployment to build.
@@ -418,7 +417,7 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals(202, $largeTag['headers']['status-code']);
$this->assertNotEmpty($largeTag['body']['$id']);
$this->assertEquals(true, DateTime::isValid($largeTag['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($largeTag['body']['$createdAt']));
$this->assertEquals('index.php', $largeTag['body']['entrypoint']);
$this->assertGreaterThan(10000, $largeTag['body']['size']);
@@ -440,8 +439,8 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertNotEmpty($response['body']['$id']);
$this->assertEquals(true, DateTime::isValid($response['body']['$createdAt']));
$this->assertEquals(true, DateTime::isValid($response['body']['$updatedAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($response['body']['$updatedAt']));
$this->assertEquals($data['deploymentId'], $response['body']['deployment']);
/**
@@ -606,7 +605,7 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals(202, $execution['headers']['status-code']);
$this->assertNotEmpty($execution['body']['$id']);
$this->assertNotEmpty($execution['body']['functionId']);
$this->assertEquals(true, DateTime::isValid($execution['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($execution['body']['$createdAt']));
$this->assertEquals($data['functionId'], $execution['body']['functionId']);
$this->assertEquals('waiting', $execution['body']['status']);
$this->assertEquals(0, $execution['body']['statusCode']);
@@ -623,7 +622,7 @@ class FunctionsCustomServerTest extends Scope
$this->assertNotEmpty($execution['body']['$id']);
$this->assertNotEmpty($execution['body']['functionId']);
$this->assertEquals(true, DateTime::isValid($execution['body']['$createdAt']));
$this->assertEquals(true, (new DatetimeValidator())->isValid($execution['body']['$createdAt']));
$this->assertEquals($data['functionId'], $execution['body']['functionId']);
$this->assertEquals('completed', $execution['body']['status']);
$this->assertEquals(200, $execution['body']['statusCode']);
@@ -635,7 +634,7 @@ class FunctionsCustomServerTest extends Scope
$this->assertStringContainsString('8.0', $execution['body']['response']);
$this->assertStringContainsString('êä', $execution['body']['response']); // tests unknown utf-8 chars
$this->assertEquals('', $execution['body']['stderr']);
$this->assertLessThan(1.500, $execution['body']['duration']);
$this->assertLessThan(3, $execution['body']['duration']);
/**
* Test for FAILURE
@@ -915,7 +914,6 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals($executions['body']['executions'][0]['statusCode'], 500);
$this->assertGreaterThan(2, $executions['body']['executions'][0]['duration']);
$this->assertLessThan(6, $executions['body']['executions'][0]['duration']);
$this->assertGreaterThan(4, $executions['body']['executions'][0]['duration']);
$this->assertEquals($executions['body']['executions'][0]['response'], '');
$this->assertEquals($executions['body']['executions'][0]['stderr'], 'An internal curl error has occurred within the executor! Error Msg: Operation timed out');
tests/e2e/Services/Projects/ProjectsConsoleClientTest.php
+8 -10
View File
@@ -6,11 +6,9 @@ use Appwrite\Auth\Auth;
use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectConsole;
use Tests\E2E\Scopes\SideClient;
use Tests\E2E\Services\Projects\ProjectsBase;
use Tests\E2E\Client;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\ID;
use Utopia\Database\Helpers\ID;
class ProjectsConsoleClientTest extends Scope
{
@@ -355,14 +353,14 @@ class ProjectsConsoleClientTest extends Scope
$this->assertEquals(count($response['body']), 9);
$this->assertNotEmpty($response['body']);
$this->assertEquals('30d', $response['body']['range']);
$this->assertIsArray($response['body']['requests']);
$this->assertIsArray($response['body']['requestsTotal']);
$this->assertIsArray($response['body']['network']);
$this->assertIsArray($response['body']['executions']);
$this->assertIsArray($response['body']['documents']);
$this->assertIsArray($response['body']['databases']);
$this->assertIsArray($response['body']['buckets']);
$this->assertIsArray($response['body']['users']);
$this->assertIsArray($response['body']['storage']);
$this->assertIsArray($response['body']['executionsTotal']);
$this->assertIsArray($response['body']['documentsTotal']);
$this->assertIsArray($response['body']['databasesTotal']);
$this->assertIsArray($response['body']['bucketsTotal']);
$this->assertIsArray($response['body']['usersTotal']);
$this->assertIsArray($response['body']['filesStorage']);
/**
* Test for FAILURE
tests/e2e/Services/Realtime/RealtimeConsoleClientTest.php
+3 -3
View File
@@ -8,9 +8,9 @@ use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\SideConsole;
use Tests\E2E\Services\Functions\FunctionsBase;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
class RealtimeConsoleClientTest extends Scope
{
tests/e2e/Services/Realtime/RealtimeCustomClientTest.php
+4 -4
View File
@@ -8,9 +8,9 @@ use Tests\E2E\Scopes\Scope;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\SideClient;
use Utopia\CLI\Console;
use Utopia\Database\ID;
use Utopia\Database\Permission;
use Utopia\Database\Role;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use WebSocket\ConnectionException;
class RealtimeCustomClientTest extends Scope
@@ -1248,7 +1248,7 @@ class RealtimeCustomClientTest extends Scope
$this->assertNotEmpty($deployment['body']['$id']);
// Wait for deployment to be built.
sleep(5);
sleep(10);
$response = $this->client->call(Client::METHOD_PATCH, '/functions/' . $functionId . '/deployments/' . $deploymentId, array_merge([
'content-type' => 'application/json',

Some files were not shown because too many files have changed in this diff Show More