Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/feat-git-integration' into feat-1.4.x-response-filters

Browse Source 
# Conflicts:
#	app/controllers/api/proxy.php
This commit is contained in:
Jake Barnby
2023-08-22 13:37:03 -04:00
parent ee06ecf5c5 4a2f10f938
commit e7d5d25263
59 changed files with 4498 additions and 2225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+4
View File
@@ -38,6 +38,10 @@ _APP_CONNECTIONS_STORAGE=local://localhost
_APP_STORAGE_ANTIVIRUS=disabled
_APP_STORAGE_ANTIVIRUS_HOST=clamav
_APP_STORAGE_ANTIVIRUS_PORT=3310
_APP_INFLUXDB_HOST=influxdb
_APP_INFLUXDB_PORT=8086
_APP_STATSD_HOST=telegraf
_APP_STATSD_PORT=8125
_APP_SMTP_HOST=maildev
_APP_SMTP_PORT=1025
_APP_SMTP_SECURE=
.gitmodules
+1 -1
View File
@@ -1,4 +1,4 @@
[submodule "app/console"]
path = app/console
url = https://github.com/appwrite/console
branch = feat-g4
branch = 1.4.x
CHANGES.md
-35
View File
@@ -121,41 +121,6 @@
- Get default region from environment on project create [#4780](https://github.com/appwrite/appwrite/pull/4780)
- Fix french translation [#4782](https://github.com/appwrite/appwrite/pull/4782)
- Fix max mimetype size [#4814](https://github.com/appwrite/appwrite/pull/4814)
- New usage metrics collection flow [#4770](https://github.com/appwrite/appwrite/pull/4770)
- Deprecated influxdb, telegraf containers and removed all of their occurrences from the code.
- Removed _APP_INFLUXDB_HOST, _APP_INFLUXDB_PORT, _APP_STATSD_HOST, _APP_STATSD_PORT env variables.
- Removed usage labels dependency.
- Dropped type attribute from stats collection.
- Usage metrics are processed via new usage worker.
- Metrics changes:
- Storage
- deprecated
- filesCreate, filesRead, filesUpdate, filesDelete, bucketsCreate, bucketsRead, bucketsUpdate, bucketsDelete.
- renamed
- filesCount to filesTotal, storage to filesStorage, bucketsCount to bucketsTotal.
- Auth
- deprecated
- usersCreate, usersRead, usersUpdate, usersDelete, sessionsCreate sessionsProviderCreate, sessionsDelete.
- renamed
- usersCount to usersTotal.
- added
- sessionsTotal.
- Databases
- deprecated
- databasesCreate, databasesRead, databasesDelete, documentsCreate, documentsRead, documentsUpdate, documentsDelete, collectionsCreate, collectionsRead, collectionsUpdate, collectionsDelete.
- renamed
- databasesCount to databasesTotal, collectionsCount to collectionsTotal, documentsCount to documentsTotal.
- Functions
- deprecated
- executionsFailure, executionsSuccess, buildsFailure, buildsSuccess, executionsFailure, executionsSuccess.
- renamed
- executionsTime to executionsCompute, buildsTime to buildsCompute, documentsCount to documentsTotal.
- added
- functionsTotal, buildsStorage, deploymentsTotal, deploymentsStorage.
- Project
- renamed
- executions to executionsTotal, builds to buildsTotal, requests to requestsTotal, storage to filesStorage, buckets to bucketsTotal, users to usersTotal, documents to documentsTotal, collections to collectionsTotal, databases to databasesTotal.
## Bugs
- Fix invited account verified status [#4776](https://github.com/appwrite/appwrite/pull/4776)
CONTRIBUTING.md
+2
View File
@@ -238,6 +238,8 @@ Appwrite stack is a combination of a variety of open-source technologies and too
- Redis - for managing cache and in-memory data (currently, we do not use Redis for persistent data).
- MariaDB - for database storage and queries.
- InfluxDB - for managing stats and time-series based data
- Statsd - for sending data over UDP protocol (using Telegraf)
- ClamAV - for validating and scanning storage files.
- Imagemagick - for manipulating and managing image media files.
- Webp - for better compression of images on supporting clients.
Dockerfile
+6 -2
View File
@@ -92,6 +92,10 @@ ENV _APP_SERVER=swoole \
_APP_DB_USER=root \
_APP_DB_PASS=password \
_APP_DB_SCHEMA=appwrite \
_APP_INFLUXDB_HOST=influxdb \
_APP_INFLUXDB_PORT=8086 \
_APP_STATSD_HOST=telegraf \
_APP_STATSD_PORT=8125 \
_APP_FUNCTIONS_SIZE_LIMIT=30000000 \
_APP_FUNCTIONS_TIMEOUT=900 \
_APP_FUNCTIONS_CONTAINERS=10 \
@@ -161,6 +165,7 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/patch-delete-project-collections && \
chmod +x /usr/local/bin/maintenance && \
chmod +x /usr/local/bin/volume-sync && \
chmod +x /usr/local/bin/usage && \
chmod +x /usr/local/bin/install && \
chmod +x /usr/local/bin/migrate && \
chmod +x /usr/local/bin/realtime && \
@@ -180,8 +185,7 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/worker-mails && \
chmod +x /usr/local/bin/worker-messaging && \
chmod +x /usr/local/bin/worker-webhooks && \
chmod +x /usr/local/bin/worker-migrations && \
chmod +x /usr/local/bin/worker-usage
chmod +x /usr/local/bin/worker-migrations
# Letsencrypt Permissions
RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/
app/cli.php
+23
View File
@@ -116,6 +116,29 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
return $getProjectDB;
}, ['pools', 'dbForConsole', 'cache']);
CLI::setResource('influxdb', function (Registry $register) {
$client = $register->get('influxdb'); /** @var InfluxDB\Client $client */
$attempts = 0;
$max = 10;
$sleep = 1;
do { // check if telegraf database is ready
try {
$attempts++;
$database = $client->selectDB('telegraf');
if (in_array('telegraf', $client->listDatabases())) {
break; // leave the do-while if successful
}
} catch (\Throwable $th) {
Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('InfluxDB database not ready yet');
}
sleep($sleep);
}
} while ($attempts < $max);
return $database;
}, ['register']);
CLI::setResource('queueForFunctions', function (Group $pools) {
return new Func($pools->get('queue')->pop()->getResource());
app/config/collections.php
+14 -42
View File
@@ -1302,7 +1302,7 @@ $commonCollections = [
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 8,
'signed' => true,
'signed' => false,
'required' => true,
'default' => null,
'array' => false,
@@ -1330,6 +1330,17 @@ $commonCollections = [
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('type'),
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 1,
'signed' => false,
'required' => true,
'default' => 0, // 0 -> count, 1 -> sum
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
@@ -1348,53 +1359,14 @@ $commonCollections = [
],
[
'$id' => ID::custom('_key_metric_period_time'),
'type' => Database::INDEX_UNIQUE,
'type' => Database::INDEX_KEY,
'attributes' => ['metric', 'period', 'time'],
'lengths' => [],
'orders' => [Database::ORDER_DESC],
],
],
],
'statsLogger' => [
'$collection' => ID::custom(Database::METADATA),
'$id' => ID::custom('statsLogger'),
'name' => 'StatsLogger',
'attributes' => [
[
'$id' => ID::custom('time'),
'type' => Database::VAR_DATETIME,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => ID::custom('metrics'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 5012,
'signed' => true,
'required' => false,
'default' => [],
'array' => false,
'filters' => ['json'],
],
],
'indexes' => [
[
'$id' => ID::custom('_key_time'),
'type' => Database::INDEX_KEY,
'attributes' => ['time'],
'lengths' => [],
'orders' => [Database::ORDER_DESC],
],
],
],
];
];
$projectCollections = array_merge([
'databases' => [
app/config/specs/open-api3-latest-console.json
+1 -1
View File
File diff suppressed because one or more lines are too long
app/config/specs/open-api3-latest-server.json
+1 -1
View File
File diff suppressed because one or more lines are too long
app/config/specs/swagger2-latest-console.json
+1 -1
View File
File diff suppressed because one or more lines are too long
app/config/specs/swagger2-latest-server.json
+1 -1
View File
File diff suppressed because one or more lines are too long
app/console
+1 -1
Submodule app/console updated: 495a9c2530...9865be05f8
app/controllers/api/account.php
+33 -1
View File
@@ -58,6 +58,7 @@ App::post('/v1/account/invite')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createWithInviteCode')
@@ -153,6 +154,7 @@ App::post('/v1/account')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'create')
@@ -266,6 +268,8 @@ App::post('/v1/account/sessions/email')
->label('audits.event', 'session.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:email'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createEmailSession')
@@ -518,6 +522,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
->label('abuse-limit', 50)
->label('abuse-key', 'ip:{ip}')
->label('docs', false)
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:{request.provider}'])
->param('provider', '', new WhiteList(\array_keys(Config::getParam('providers')), true), 'OAuth2 provider.')
->param('code', '', new Text(2048, 0), 'OAuth2 code.', true)
->param('state', '', new Text(2048), 'OAuth2 state params.', true)
@@ -900,7 +906,9 @@ App::get('/v1/account/identities')
$queries[] = Query::equal('userInternalId', [$user->getInternalId()]);
// Get cursor document if there was a cursor query
$cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
$cursor = \array_filter($queries, function ($query) {
return \in_array($query->getMethod(), [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
});
$cursor = reset($cursor);
if ($cursor) {
/** @var Query $cursor */
@@ -1138,6 +1146,8 @@ App::put('/v1/account/sessions/magic-url')
->label('audits.event', 'session.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:magic-url'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateMagicURLSession')
@@ -1512,6 +1522,8 @@ App::post('/v1/account/sessions/anonymous')
->label('audits.event', 'session.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.create')
->label('usage.params', ['provider:anonymous'])
->label('sdk.auth', [])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createAnonymousSession')
@@ -1687,6 +1699,7 @@ App::get('/v1/account')
->desc('Get Account')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'get')
@@ -1707,6 +1720,7 @@ App::get('/v1/account/prefs')
->desc('Get Account Preferences')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'getPrefs')
@@ -1729,6 +1743,7 @@ App::get('/v1/account/sessions')
->desc('List Sessions')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'listSessions')
@@ -1767,6 +1782,7 @@ App::get('/v1/account/logs')
->desc('List Logs')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'listLogs')
@@ -1827,6 +1843,7 @@ App::get('/v1/account/sessions/:sessionId')
->desc('Get Session')
->groups(['api', 'account'])
->label('scope', 'account')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'getSession')
@@ -1874,6 +1891,7 @@ App::patch('/v1/account/name')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateName')
@@ -1908,6 +1926,7 @@ App::patch('/v1/account/password')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePassword')
@@ -1973,6 +1992,7 @@ App::patch('/v1/account/email')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateEmail')
@@ -2042,6 +2062,7 @@ App::patch('/v1/account/phone')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePhone')
@@ -2100,6 +2121,7 @@ App::patch('/v1/account/prefs')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePrefs')
@@ -2133,6 +2155,7 @@ App::patch('/v1/account/status')
->label('scope', 'account')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateStatus')
@@ -2176,6 +2199,7 @@ App::delete('/v1/account/sessions/:sessionId')
->label('event', 'users.[userId].sessions.[sessionId].delete')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{user.$id}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'deleteSession')
@@ -2252,6 +2276,7 @@ App::patch('/v1/account/sessions/:sessionId')
->label('audits.event', 'session.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'sessions.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateSession')
@@ -2336,6 +2361,7 @@ App::delete('/v1/account/sessions')
->label('event', 'users.[userId].sessions.[sessionId].delete')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{user.$id}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'deleteSessions')
@@ -2397,6 +2423,7 @@ App::post('/v1/account/recovery')
->label('audits.event', 'recovery.create')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createRecovery')
@@ -2537,6 +2564,7 @@ App::put('/v1/account/recovery')
->label('audits.event', 'recovery.update')
->label('audits.resource', 'user/{response.userId}')
->label('audits.userId', '{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateRecovery')
@@ -2623,6 +2651,7 @@ App::post('/v1/account/verification')
->label('event', 'users.[userId].verification.[tokenId].create')
->label('audits.event', 'verification.create')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createVerification')
@@ -2742,6 +2771,7 @@ App::put('/v1/account/verification')
->label('event', 'users.[userId].verification.[tokenId].update')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updateVerification')
@@ -2802,6 +2832,7 @@ App::post('/v1/account/verification/phone')
->label('event', 'users.[userId].verification.[tokenId].create')
->label('audits.event', 'verification.create')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'createPhoneVerification')
@@ -2897,6 +2928,7 @@ App::put('/v1/account/verification/phone')
->label('event', 'users.[userId].verification.[tokenId].update')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'account')
->label('sdk.method', 'updatePhoneVerification')
app/controllers/api/databases.php
+451 -290
View File
File diff suppressed because it is too large Load Diff
app/controllers/api/functions.php
+213 -145
View File
@@ -356,7 +356,9 @@ App::get('/v1/functions')
}
// Get cursor document if there was a cursor query
$cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
$cursor = \array_filter($queries, function ($query) {
return \in_array($query->getMethod(), [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
});
$cursor = reset($cursor);
if ($cursor) {
/** @var Query $cursor */
@@ -451,66 +453,92 @@ App::get('/v1/functions/:functionId/usage')
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS),
str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS),
str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE),
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
}
}
$response->dynamic(new Document([
'range' => $range,
'deploymentsTotal' => $usage[$metrics[0]],
'deploymentsStorage' => $usage[$metrics[1]],
'buildsTotal' => $usage[$metrics[2]],
'buildsStorage' => $usage[$metrics[3]],
'buildsTime' => $usage[$metrics[4]],
'executionsTotal' => $usage[$metrics[5]],
'executionsTime' => $usage[$metrics[6]],
]), Response::MODEL_USAGE_FUNCTION);
$metrics = [
"executions.$functionId.compute.total",
"executions.$functionId.compute.success",
"executions.$functionId.compute.failure",
"executions.$functionId.compute.time",
"builds.$functionId.compute.total",
"builds.$functionId.compute.success",
"builds.$functionId.compute.failure",
"builds.$functionId.compute.time",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'executionsTotal' => $stats["executions.$functionId.compute.total"] ?? [],
'executionsFailure' => $stats["executions.$functionId.compute.failure"] ?? [],
'executionsSuccesse' => $stats["executions.$functionId.compute.success"] ?? [],
'executionsTime' => $stats["executions.$functionId.compute.time"] ?? [],
'buildsTotal' => $stats["builds.$functionId.compute.total"] ?? [],
'buildsFailure' => $stats["builds.$functionId.compute.failure"] ?? [],
'buildsSuccess' => $stats["builds.$functionId.compute.success"] ?? [],
'buildsTime' => $stats["builds.$functionId.compute.time" ?? []]
]);
}
$response->dynamic($usage, Response::MODEL_USAGE_FUNCTION);
});
App::get('/v1/functions/usage')
@@ -528,67 +556,92 @@ App::get('/v1/functions/usage')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_FUNCTIONS,
METRIC_DEPLOYMENTS,
METRIC_DEPLOYMENTS_STORAGE,
METRIC_BUILDS,
METRIC_BUILDS_STORAGE,
METRIC_BUILDS_COMPUTE,
METRIC_EXECUTIONS,
METRIC_EXECUTIONS_COMPUTE,
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$metrics = [
'executions.$all.compute.total',
'executions.$all.compute.failure',
'executions.$all.compute.success',
'executions.$all.compute.time',
'builds.$all.compute.total',
'builds.$all.compute.failure',
'builds.$all.compute.success',
'builds.$all.compute.time',
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'executionsTotal' => $stats[$metrics[0]] ?? [],
'executionsFailure' => $stats[$metrics[1]] ?? [],
'executionsSuccess' => $stats[$metrics[2]] ?? [],
'executionsTime' => $stats[$metrics[3]] ?? [],
'buildsTotal' => $stats[$metrics[4]] ?? [],
'buildsFailure' => $stats[$metrics[5]] ?? [],
'buildsSuccess' => $stats[$metrics[6]] ?? [],
'buildsTime' => $stats[$metrics[7]] ?? [],
]);
}
}
$response->dynamic(new Document([
'range' => $range,
'functionsTotal' => $usage[$metrics[0]],
'deploymentsTotal' => $usage[$metrics[1]],
'deploymentsStorage' => $usage[$metrics[2]],
'buildsTotal' => $usage[$metrics[3]],
'buildsStorage' => $usage[$metrics[4]],
'buildsTime' => $usage[$metrics[5]],
'executionsTotal' => $usage[$metrics[6]],
'executionsTime' => $usage[$metrics[7]],
]), Response::MODEL_USAGE_FUNCTIONS);
$response->dynamic($usage, Response::MODEL_USAGE_FUNCTIONS);
});
App::put('/v1/functions/:functionId')
@@ -976,6 +1029,8 @@ App::post('/v1/functions/:functionId/deployments')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_DEPLOYMENT)
->param('functionId', '', new UID(), 'Function ID.')
->param('entrypoint', null, new Text(1028), 'Entrypoint File.', true)
->param('commands', null, new Text(8192, 0), 'Build Commands.', true)
->param('code', [], new File(), 'Gzip file with your code package. When used with the Appwrite CLI, pass the path to your code directory, and the CLI will automatically package your code. Use a path that is within the current directory.', skipValidation: true)
->param('activate', false, new Boolean(true), 'Automatically activate the deployment when it is finished building.')
->inject('request')
@@ -985,7 +1040,7 @@ App::post('/v1/functions/:functionId/deployments')
->inject('project')
->inject('deviceFunctions')
->inject('deviceLocal')
->action(function (string $functionId, mixed $code, mixed $activate, Request $request, Response $response, Database $dbForProject, Event $events, Document $project, Device $deviceFunctions, Device $deviceLocal) {
->action(function (string $functionId, ?string $entrypoint, ?string $commands, mixed $code, mixed $activate, Request $request, Response $response, Database $dbForProject, Event $events, Document $project, Device $deviceFunctions, Device $deviceLocal) {
$activate = filter_var($activate, FILTER_VALIDATE_BOOLEAN);
$function = $dbForProject->getDocument('functions', $functionId);
@@ -994,7 +1049,14 @@ App::post('/v1/functions/:functionId/deployments')
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$entrypoint = $function->getAttribute('entrypoint', '');
if ($entrypoint === null) {
$entrypoint = $function->getAttribute('entrypoint', '');
}
if ($commands === null) {
$commands = $function->getAttribute('entrypoint', '');
}
$commands = $function->getAttribute('commands', '');
if (empty($entrypoint)) {
@@ -1206,7 +1268,9 @@ App::get('/v1/functions/:functionId/deployments')
$queries[] = Query::equal('resourceType', ['functions']);
// Get cursor document if there was a cursor query
$cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
$cursor = \array_filter($queries, function ($query) {
return \in_array($query->getMethod(), [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
});
$cursor = reset($cursor);
if ($cursor) {
/** @var Query $cursor */
@@ -1438,18 +1502,19 @@ App::post('/v1/functions/:functionId/executions')
->inject('dbForProject')
->inject('user')
->inject('events')
->inject('usage')
->inject('mode')
->inject('queueForFunctions')
->inject('geodb')
->inject('queueForUsage')
->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, string $mode, Func $queueForFunctions, Reader $geodb, Usage $queueForUsage) {
->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $events, Stats $usage, string $mode, Func $queueForFunctions, Reader $geodb) {
$function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId));
if ($function->isEmpty() || !$function->getAttribute('enabled')) {
if (!($mode === APP_MODE_ADMIN && Auth::isPrivilegedUser(Authorization::getRoles()))) {
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
if ($function->isEmpty() || (!$function->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) {
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$runtimes = Config::getParam('runtimes', []);
@@ -1645,13 +1710,6 @@ App::post('/v1/functions/:functionId/executions')
$execution->setAttribute('logs', $executionResponse['logs']);
$execution->setAttribute('errors', $executionResponse['errors']);
$execution->setAttribute('duration', $executionResponse['duration']);
/**
* Sync execution compute usage from
*/
$queueForUsage
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($executionResponse['duration'] * 1000)) // per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($executionResponse['duration'] * 1000)) // per function
;
} catch (\Throwable $th) {
$durationEnd = \microtime(true);
@@ -1668,6 +1726,14 @@ App::post('/v1/functions/:functionId/executions')
$execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution));
}
// TODO revise this later using route label
$usage
->setParam('functionId', $function->getId())
->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', ''))
->setParam('executionTime', $execution->getAttribute('duration')); // ms
$roles = Authorization::getRoles();
$isPrivilegedUser = Auth::isPrivilegedUser($roles);
$isAppUser = Auth::isAppUser($roles);
@@ -1708,13 +1774,13 @@ App::get('/v1/functions/:functionId/executions')
->inject('dbForProject')
->inject('mode')
->action(function (string $functionId, array $queries, string $search, Response $response, Database $dbForProject, string $mode) {
$function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId));
if ($function->isEmpty() || !$function->getAttribute('enabled')) {
if (!($mode === APP_MODE_ADMIN && Auth::isPrivilegedUser(Authorization::getRoles()))) {
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
if ($function->isEmpty() || (!$function->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) {
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$queries = Query::parseQueries($queries);
@@ -1727,7 +1793,9 @@ App::get('/v1/functions/:functionId/executions')
$queries[] = Query::equal('functionId', [$function->getId()]);
// Get cursor document if there was a cursor query
$cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
$cursor = \array_filter($queries, function ($query) {
return \in_array($query->getMethod(), [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
});
$cursor = reset($cursor);
if ($cursor) {
/** @var Query $cursor */
@@ -1780,13 +1848,13 @@ App::get('/v1/functions/:functionId/executions/:executionId')
->inject('dbForProject')
->inject('mode')
->action(function (string $functionId, string $executionId, Response $response, Database $dbForProject, string $mode) {
$function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId));
if ($function->isEmpty() || !$function->getAttribute('enabled')) {
if (!($mode === APP_MODE_ADMIN && Auth::isPrivilegedUser(Authorization::getRoles()))) {
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
if ($function->isEmpty() || (!$function->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) {
throw new Exception(Exception::FUNCTION_NOT_FOUND);
}
$execution = $dbForProject->getDocument('executions', $executionId);
app/controllers/api/migrations.php
+3 -1
View File
@@ -381,7 +381,9 @@ App::get('/v1/migrations')
}
// Get cursor document if there was a cursor query
$cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
$cursor = \array_filter($queries, function ($query) {
return \in_array($query->getMethod(), [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
});
$cursor = reset($cursor);
if ($cursor) {
/** @var Query $cursor */
app/controllers/api/project.php
+85 -67
View File
@@ -3,7 +3,6 @@
use Appwrite\Extend\Exception;
use Appwrite\Utopia\Response;
use Utopia\App;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate as DuplicateException;
@@ -15,10 +14,11 @@ use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\UID;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
use Utopia\Database\DateTime;
App::get('/v1/project/usage')
->desc('Get usage stats for a project')
->groups(['api', 'usage'])
->groups(['api'])
->label('scope', 'projects.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'project')
@@ -30,76 +30,94 @@ App::get('/v1/project/usage')
->inject('response')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_NETWORK_REQUESTS,
METRIC_NETWORK_INBOUND,
METRIC_NETWORK_OUTBOUND,
METRIC_EXECUTIONS,
METRIC_DOCUMENTS,
METRIC_DATABASES,
METRIC_USERS,
METRIC_BUCKETS,
METRIC_FILES_STORAGE
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$metrics = [
'project.$all.network.requests',
'project.$all.network.bandwidth',
'project.$all.storage.size',
'users.$all.count.total',
'databases.$all.count.total',
'documents.$all.count.total',
'executions.$all.compute.total',
'buckets.$all.count.total'
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'requests' => $stats[$metrics[0]] ?? [],
'network' => $stats[$metrics[1]] ?? [],
'storage' => $stats[$metrics[2]] ?? [],
'users' => $stats[$metrics[3]] ?? [],
'databases' => $stats[$metrics[4]] ?? [],
'documents' => $stats[$metrics[5]] ?? [],
'executions' => $stats[$metrics[6]] ?? [],
'buckets' => $stats[$metrics[7]] ?? [],
]);
}
}
$response->dynamic(new Document([
'range' => $range,
'requestsTotal' => ($usage[$metrics[0]]),
'network' => ($usage[$metrics[1]] + $usage[$metrics[2]]),
'executionsTotal' => $usage[$metrics[3]],
'documentsTotal' => $usage[$metrics[4]],
'databasesTotal' => $usage[$metrics[5]],
'usersTotal' => $usage[$metrics[6]],
'bucketsTotal' => $usage[$metrics[7]],
'filesStorage' => $usage[$metrics[8]],
]), Response::MODEL_USAGE_PROJECT);
$response->dynamic($usage, Response::MODEL_USAGE_PROJECT);
});
// Variables
App::post('/v1/project/variables')
app/controllers/api/projects.php
+119 -2
View File
@@ -95,6 +95,7 @@ App::post('/v1/projects')
$backups['database_db_fra1_03'] = ['from' => '10:30', 'to' => '11:15'];
$backups['database_db_fra1_04'] = ['from' => '13:30', 'to' => '14:15'];
$backups['database_db_fra1_05'] = ['from' => '4:30', 'to' => '5:15'];
$backups['database_db_fra1_06'] = ['from' => '16:30', 'to' => '17:15'];
$databases = Config::getParam('pools-database', []);
@@ -118,7 +119,7 @@ App::post('/v1/projects')
}
}
if ($index = array_search('database_db_fra1_05', $databases)) {
if ($index = array_search('database_db_fra1_06', $databases)) {
$database = $databases[$index];
} else {
$database = $databases[array_rand($databases)];
@@ -240,7 +241,9 @@ App::get('/v1/projects')
}
// Get cursor document if there was a cursor query
$cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
$cursor = \array_filter($queries, function ($query) {
return \in_array($query->getMethod(), [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
});
$cursor = reset($cursor);
if ($cursor) {
/** @var Query $cursor */
@@ -286,6 +289,120 @@ App::get('/v1/projects/:projectId')
$response->dynamic($project, Response::MODEL_PROJECT);
});
App::get('/v1/projects/:projectId/usage')
->desc('Get usage stats for a project')
->groups(['api', 'projects', 'usage'])
->label('scope', 'projects.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'projects')
->label('sdk.method', 'getUsage')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_PROJECT)
->param('projectId', '', new UID(), 'Project unique ID.')
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->inject('response')
->inject('dbForConsole')
->inject('dbForProject')
->inject('register')
->action(function (string $projectId, string $range, Response $response, Database $dbForConsole, Database $dbForProject, Registry $register) {
$project = $dbForConsole->getDocument('projects', $projectId);
if ($project->isEmpty()) {
throw new Exception(Exception::PROJECT_NOT_FOUND);
}
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
$dbForProject->setNamespace("_{$project->getInternalId()}");
$metrics = [
'project.$all.network.requests',
'project.$all.network.bandwidth',
'project.$all.storage.size',
'users.$all.count.total',
'databases.$all.count.total',
'documents.$all.count.total',
'executions.$all.compute.total',
'buckets.$all.count.total'
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'requests' => $stats[$metrics[0]] ?? [],
'network' => $stats[$metrics[1]] ?? [],
'storage' => $stats[$metrics[2]] ?? [],
'users' => $stats[$metrics[3]] ?? [],
'databases' => $stats[$metrics[4]] ?? [],
'documents' => $stats[$metrics[5]] ?? [],
'executions' => $stats[$metrics[6]] ?? [],
'buckets' => $stats[$metrics[7]] ?? [],
]);
}
$response->dynamic($usage, Response::MODEL_USAGE_PROJECT);
});
App::patch('/v1/projects/:projectId')
->desc('Update Project')
->groups(['api', 'projects'])
app/controllers/api/proxy.php
+1 -1
View File
@@ -43,7 +43,7 @@ App::post('/v1/proxy/rules')
->action(function (string $domain, string $resourceType, string $resourceId, Response $response, Document $project, Event $events, Database $dbForConsole, Database $dbForProject) {
$mainDomain = App::getEnv('_APP_DOMAIN', '');
if ($domain === $mainDomain || $domain === 'localhost' || $domain === APP_HOSTNAME_INTERNAL) {
throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'This domain name is not allowed for security reasons.');
throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'This domain name is not allowed. Please pick another one.');
}
$document = $dbForConsole->findOne('rules', [
app/controllers/api/storage.php
+225 -109
View File
@@ -51,6 +51,7 @@ App::post('/v1/storage/buckets')
->label('event', 'buckets.[bucketId].create')
->label('audits.event', 'bucket.create')
->label('audits.resource', 'bucket/{response.$id}')
->label('usage.metric', 'buckets.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'createBucket')
@@ -146,6 +147,7 @@ App::get('/v1/storage/buckets')
->desc('List buckets')
->groups(['api', 'storage'])
->label('scope', 'buckets.read')
->label('usage.metric', 'buckets.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'listBuckets')
@@ -166,7 +168,9 @@ App::get('/v1/storage/buckets')
}
// Get cursor document if there was a cursor query
$cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
$cursor = \array_filter($queries, function ($query) {
return \in_array($query->getMethod(), [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
});
$cursor = reset($cursor);
if ($cursor) {
/** @var Query $cursor */
@@ -192,6 +196,7 @@ App::get('/v1/storage/buckets/:bucketId')
->desc('Get Bucket')
->groups(['api', 'storage'])
->label('scope', 'buckets.read')
->label('usage.metric', 'buckets.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getBucket')
@@ -220,6 +225,7 @@ App::put('/v1/storage/buckets/:bucketId')
->label('event', 'buckets.[bucketId].update')
->label('audits.event', 'bucket.update')
->label('audits.resource', 'bucket/{response.$id}')
->label('usage.metric', 'buckets.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'updateBucket')
@@ -287,6 +293,7 @@ App::delete('/v1/storage/buckets/:bucketId')
->label('audits.event', 'bucket.delete')
->label('event', 'buckets.[bucketId].delete')
->label('audits.resource', 'bucket/{request.bucketId}')
->label('usage.metric', 'buckets.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'deleteBucket')
@@ -329,6 +336,8 @@ App::post('/v1/storage/buckets/:bucketId/files')
->label('audits.event', 'file.create')
->label('event', 'buckets.[bucketId].files.[fileId].create')
->label('audits.resource', 'file/{response.$id}')
->label('usage.metric', 'files.{scope}.requests.create')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -354,10 +363,12 @@ App::post('/v1/storage/buckets/:bucketId/files')
->inject('deviceFiles')
->inject('deviceLocal')
->action(function (string $bucketId, string $fileId, mixed $file, ?array $permissions, Request $request, Response $response, Database $dbForProject, Document $user, Event $events, string $mode, Device $deviceFiles, Device $deviceLocal) {
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
@@ -669,6 +680,8 @@ App::get('/v1/storage/buckets/:bucketId/files')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'listFiles')
->label('sdk.description', '/docs/references/storage/list-files.md')
@@ -682,10 +695,12 @@ App::get('/v1/storage/buckets/:bucketId/files')
->inject('dbForProject')
->inject('mode')
->action(function (string $bucketId, array $queries, string $search, Response $response, Database $dbForProject, string $mode) {
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
@@ -703,7 +718,9 @@ App::get('/v1/storage/buckets/:bucketId/files')
}
// Get cursor document if there was a cursor query
$cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
$cursor = \array_filter($queries, function ($query) {
return \in_array($query->getMethod(), [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
});
$cursor = reset($cursor);
if ($cursor) {
/** @var Query $cursor */
@@ -744,6 +761,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFile')
->label('sdk.description', '/docs/references/storage/get-file.md')
@@ -756,10 +775,12 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId')
->inject('dbForProject')
->inject('mode')
->action(function (string $bucketId, string $fileId, Response $response, Database $dbForProject, string $mode) {
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
@@ -791,6 +812,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
->label('cache', true)
->label('cache.resourceType', 'bucket/{request.bucketId}')
->label('cache.resource', 'file/{request.fileId}')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFilePreview')
@@ -826,7 +849,10 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
@@ -952,6 +978,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
->desc('Get File for Download')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFileDownload')
@@ -970,7 +998,10 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
@@ -1090,6 +1121,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
->desc('Get File for View')
->groups(['api', 'storage'])
->label('scope', 'files.read')
->label('usage.metric', 'files.{scope}.requests.read')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'storage')
->label('sdk.method', 'getFileView')
@@ -1105,10 +1138,12 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
->inject('mode')
->inject('deviceFiles')
->action(function (string $bucketId, string $fileId, Response $response, Request $request, Database $dbForProject, string $mode, Device $deviceFiles) {
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
@@ -1242,6 +1277,8 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
->label('event', 'buckets.[bucketId].files.[fileId].update')
->label('audits.event', 'file.update')
->label('audits.resource', 'file/{response.$id}')
->label('usage.metric', 'files.{scope}.requests.update')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -1262,10 +1299,12 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
->inject('mode')
->inject('events')
->action(function (string $bucketId, string $fileId, ?string $name, ?array $permissions, Response $response, Database $dbForProject, Document $user, string $mode, Event $events) {
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
@@ -1348,6 +1387,8 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
->label('event', 'buckets.[bucketId].files.[fileId].delete')
->label('audits.event', 'file.delete')
->label('audits.resource', 'file/{request.fileId}')
->label('usage.metric', 'files.{scope}.requests.delete')
->label('usage.params', ['bucketId:{request.bucketId}'])
->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -1368,7 +1409,10 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
->action(function (string $bucketId, string $fileId, Response $response, Database $dbForProject, Event $events, string $mode, Device $deviceFiles, Delete $deletes) {
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
@@ -1449,62 +1493,103 @@ App::get('/v1/storage/usage')
->inject('dbForProject')
->action(function (string $range, Response $response, Database $dbForProject) {
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_BUCKETS,
METRIC_FILES,
METRIC_FILES_STORAGE,
];
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
$metrics = [
'project.$all.storage.size',
'buckets.$all.count.total',
'buckets.$all.requests.create',
'buckets.$all.requests.read',
'buckets.$all.requests.update',
'buckets.$all.requests.delete',
'files.$all.storage.size',
'files.$all.count.total',
'files.$all.requests.create',
'files.$all.requests.read',
'files.$all.requests.update',
'files.$all.requests.delete',
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
}
}
});
});
$format = (match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
});
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
$usage = new Document([
'range' => $range,
'bucketsCount' => $stats['buckets.$all.count.total'],
'bucketsCreate' => $stats['buckets.$all.requests.create'],
'bucketsRead' => $stats['buckets.$all.requests.read'],
'bucketsUpdate' => $stats['buckets.$all.requests.update'],
'bucketsDelete' => $stats['buckets.$all.requests.delete'],
'storage' => $stats['project.$all.storage.size'],
'filesCount' => $stats['files.$all.count.total'],
'filesCreate' => $stats['files.$all.requests.create'],
'filesRead' => $stats['files.$all.requests.read'],
'filesUpdate' => $stats['files.$all.requests.update'],
'filesDelete' => $stats['files.$all.requests.delete'],
]);
}
$response->dynamic(new Document([
'range' => $range,
'bucketsTotal' => $usage[$metrics[0]],
'filesTotal' => $usage[$metrics[1]],
'filesStorage' => $usage[$metrics[2]],
]), Response::MODEL_USAGE_STORAGE);
$response->dynamic($usage, Response::MODEL_USAGE_STORAGE);
});
App::get('/v1/storage/:bucketId/usage')
->desc('Get usage stats for storage bucket')
->desc('Get usage stats for a storage bucket')
->groups(['api', 'storage', 'usage'])
->label('scope', 'files.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
@@ -1525,55 +1610,86 @@ App::get('/v1/storage/:bucketId/usage')
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES),
str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE),
];
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
$metrics = [
"files.{$bucketId}.count.total",
"files.{$bucketId}.storage.size",
"files.{$bucketId}.requests.create",
"files.{$bucketId}.requests.read",
"files.{$bucketId}.requests.update",
"files.{$bucketId}.requests.delete",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
}
}
});
});
$format = (match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
});
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
];
}
$usage = new Document([
'range' => $range,
'filesCount' => $stats[$metrics[0]],
'filesStorage' => $stats[$metrics[1]],
'filesCreate' => $stats[$metrics[2]],
'filesRead' => $stats[$metrics[3]],
'filesUpdate' => $stats[$metrics[4]],
'filesDelete' => $stats[$metrics[5]],
]);
}
$response->dynamic(new Document([
'range' => $range,
'filesTotal' => $usage[$metrics[0]],
'filesStorage' => $usage[$metrics[1]],
]), Response::MODEL_USAGE_BUCKETS);
$response->dynamic($usage, Response::MODEL_USAGE_BUCKETS);
});
app/controllers/api/teams.php
+6 -2
View File
@@ -153,7 +153,9 @@ App::get('/v1/teams')
}
// Get cursor document if there was a cursor query
$cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
$cursor = \array_filter($queries, function ($query) {
return \in_array($query->getMethod(), [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
});
$cursor = reset($cursor);
if ($cursor) {
/** @var Query $cursor */
@@ -649,7 +651,9 @@ App::get('/v1/teams/:teamId/memberships')
$queries[] = Query::equal('teamId', [$teamId]);
// Get cursor document if there was a cursor query
$cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
$cursor = \array_filter($queries, function ($query) {
return \in_array($query->getMethod(), [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
});
$cursor = reset($cursor);
if ($cursor) {
/** @var Query $cursor */
app/controllers/api/users.php
+117 -52
View File
@@ -114,6 +114,7 @@ App::post('/v1/users')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'create')
@@ -146,6 +147,7 @@ App::post('/v1/users/bcrypt')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createBcryptUser')
@@ -176,6 +178,7 @@ App::post('/v1/users/md5')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createMD5User')
@@ -206,6 +209,7 @@ App::post('/v1/users/argon2')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createArgon2User')
@@ -236,6 +240,7 @@ App::post('/v1/users/sha')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createSHAUser')
@@ -273,6 +278,7 @@ App::post('/v1/users/phpass')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createPHPassUser')
@@ -303,6 +309,7 @@ App::post('/v1/users/scrypt')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createScryptUser')
@@ -346,6 +353,7 @@ App::post('/v1/users/scrypt-modified')
->label('scope', 'users.write')
->label('audits.event', 'user.create')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'createScryptModifiedUser')
@@ -376,6 +384,7 @@ App::get('/v1/users')
->desc('List Users')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'list')
@@ -396,7 +405,9 @@ App::get('/v1/users')
}
// Get cursor document if there was a cursor query
$cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
$cursor = \array_filter($queries, function ($query) {
return \in_array($query->getMethod(), [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
});
$cursor = reset($cursor);
if ($cursor) {
/** @var Query $cursor */
@@ -422,6 +433,7 @@ App::get('/v1/users/:userId')
->desc('Get User')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'get')
@@ -447,6 +459,7 @@ App::get('/v1/users/:userId/prefs')
->desc('Get User Preferences')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'getPrefs')
@@ -474,6 +487,7 @@ App::get('/v1/users/:userId/sessions')
->desc('List User Sessions')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listSessions')
@@ -515,6 +529,7 @@ App::get('/v1/users/:userId/memberships')
->desc('List User Memberships')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listMemberships')
@@ -554,6 +569,7 @@ App::get('/v1/users/:userId/logs')
->desc('List User Logs')
->groups(['api', 'users'])
->label('scope', 'users.read')
->label('usage.metric', 'users.{scope}.requests.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'listLogs')
@@ -656,7 +672,9 @@ App::get('/v1/users/identities')
}
// Get cursor document if there was a cursor query
$cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
$cursor = \array_filter($queries, function ($query) {
return \in_array($query->getMethod(), [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
});
$cursor = reset($cursor);
if ($cursor) {
/** @var Query $cursor */
@@ -686,6 +704,7 @@ App::patch('/v1/users/:userId/status')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateStatus')
@@ -721,7 +740,6 @@ App::put('/v1/users/:userId/labels')
->label('scope', 'users.write')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
@@ -760,6 +778,7 @@ App::patch('/v1/users/:userId/verification/phone')
->label('scope', 'users.write')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePhoneVerification')
@@ -796,6 +815,7 @@ App::patch('/v1/users/:userId/name')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateName')
@@ -833,6 +853,7 @@ App::patch('/v1/users/:userId/password')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePassword')
@@ -897,6 +918,7 @@ App::patch('/v1/users/:userId/email')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('audits.userId', '{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateEmail')
@@ -952,6 +974,7 @@ App::patch('/v1/users/:userId/phone')
->label('scope', 'users.write')
->label('audits.event', 'user.update')
->label('audits.resource', 'user/{response.$id}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePhone')
@@ -996,6 +1019,7 @@ App::patch('/v1/users/:userId/verification')
->label('audits.event', 'verification.update')
->label('audits.resource', 'user/{request.userId}')
->label('audits.userId', '{request.userId}')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updateEmailVerification')
@@ -1028,6 +1052,7 @@ App::patch('/v1/users/:userId/prefs')
->groups(['api', 'users'])
->label('event', 'users.[userId].update.prefs')
->label('scope', 'users.write')
->label('usage.metric', 'users.{scope}.requests.update')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'updatePrefs')
@@ -1063,6 +1088,7 @@ App::delete('/v1/users/:userId/sessions/:sessionId')
->label('scope', 'users.write')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{request.userId}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'deleteSession')
@@ -1106,6 +1132,7 @@ App::delete('/v1/users/:userId/sessions')
->label('scope', 'users.write')
->label('audits.event', 'session.delete')
->label('audits.resource', 'user/{user.$id}')
->label('usage.metric', 'sessions.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'deleteSessions')
@@ -1148,6 +1175,7 @@ App::delete('/v1/users/:userId')
->label('scope', 'users.write')
->label('audits.event', 'user.delete')
->label('audits.resource', 'user/{request.userId}')
->label('usage.metric', 'users.{scope}.requests.delete')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'users')
->label('sdk.method', 'delete')
@@ -1226,59 +1254,96 @@ App::get('/v1/users/usage')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_USAGE_USERS)
->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(fn ($value) => "oauth-" . $value, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true)
->inject('response')
->inject('dbForProject')
->inject('register')
->action(function (string $range, Response $response, Database $dbForProject) {
->action(function (string $range, string $provider, Response $response, Database $dbForProject) {
$periods = Config::getParam('usage', []);
$stats = $usage = [];
$days = $periods[$range];
$metrics = [
METRIC_USERS,
METRIC_SESSIONS,
];
Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $days['limit'];
$period = $days['period'];
$results = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($results as $result) {
$stats[$metric][$result->getAttribute('time')] = [
'value' => $result->getAttribute('value'),
];
}
}
});
$format = match ($days['period']) {
'1h' => 'Y-m-d\TH:00:00.000P',
'1d' => 'Y-m-d\T00:00:00.000P',
};
foreach ($metrics as $metric) {
$usage[$metric] = [];
$leap = time() - ($days['limit'] * $days['factor']);
while ($leap < time()) {
$leap += $days['factor'];
$formatDate = date($format, $leap);
$usage[$metric][] = [
'value' => $stats[$metric][$formatDate]['value'] ?? 0,
'date' => $formatDate,
$usage = [];
if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
$periods = [
'24h' => [
'period' => '1h',
'limit' => 24,
],
'7d' => [
'period' => '1d',
'limit' => 7,
],
'30d' => [
'period' => '1d',
'limit' => 30,
],
'90d' => [
'period' => '1d',
'limit' => 90,
],
];
}
}
$response->dynamic(new Document([
'range' => $range,
'usersTotal' => $usage[$metrics[0]],
'sessionsTotal' => $usage[$metrics[1]],
]), Response::MODEL_USAGE_USERS);
$metrics = [
'users.$all.count.total',
'users.$all.requests.create',
'users.$all.requests.read',
'users.$all.requests.update',
'users.$all.requests.delete',
'sessions.$all.requests.create',
'sessions.$all.requests.delete',
"sessions.$provider.requests.create",
];
$stats = [];
Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
foreach ($metrics as $metric) {
$limit = $periods[$range]['limit'];
$period = $periods[$range]['period'];
$requestDocs = $dbForProject->find('stats', [
Query::equal('period', [$period]),
Query::equal('metric', [$metric]),
Query::limit($limit),
Query::orderDesc('time'),
]);
$stats[$metric] = [];
foreach ($requestDocs as $requestDoc) {
$stats[$metric][] = [
'value' => $requestDoc->getAttribute('value'),
'date' => $requestDoc->getAttribute('time'),
];
}
// backfill metrics with empty values for graphs
$backfill = $limit - \count($requestDocs);
while ($backfill > 0) {
$last = $limit - $backfill - 1; // array index of last added metric
$diff = match ($period) { // convert period to seconds for unix timestamp math
'1h' => 3600,
'1d' => 86400,
};
$stats[$metric][] = [
'value' => 0,
'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
];
$backfill--;
}
$stats[$metric] = array_reverse($stats[$metric]);
}
});
$usage = new Document([
'range' => $range,
'usersCount' => $stats['users.$all.count.total'] ?? [],
'usersCreate' => $stats['users.$all.requests.create'] ?? [],
'usersRead' => $stats['users.$all.requests.read'] ?? [],
'usersUpdate' => $stats['users.$all.requests.update'] ?? [],
'usersDelete' => $stats['users.$all.requests.delete'] ?? [],
'sessionsCreate' => $stats['sessions.$all.requests.create'] ?? [],
'sessionsProviderCreate' => $stats["sessions.$provider.requests.create"] ?? [],
'sessionsDelete' => $stats['sessions.$all.requests.delete' ?? []]
]);
}
$response->dynamic($usage, Response::MODEL_USAGE_USERS);
});
app/controllers/general.php
+3 -1
View File
@@ -107,7 +107,8 @@ function router(App $utopia, Database $dbForConsole, SwooleRequest $swooleReques
$headers = [
'Content-Type: application/json',
'Content-Length: ' . \strlen($body),
'X-Appwrite-Project: ' . $projectId
'X-Appwrite-Project: ' . $projectId,
'Host: ' . $host,
];
$ch = \curl_init();
@@ -759,6 +760,7 @@ include_once __DIR__ . '/shared/api/auth.php';
App::wildcard()
->groups(['api'])
->label('scope', 'public')
->action(function () {
throw new AppwriteException(AppwriteException::GENERAL_ROUTE_NOT_FOUND);
});
app/controllers/shared/api.php
+73 -143
View File
@@ -8,8 +8,8 @@ use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Mail;
use Appwrite\Extend\Exception;
use Appwrite\Event\Usage;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Usage\Stats;
use Appwrite\Utopia\Response;
use Appwrite\Utopia\Request;
use Utopia\App;
@@ -48,99 +48,43 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar
return $label;
};
$databaseListener = function (string $event, Document $document, Document $project, Usage $queueForUsage, Database $dbForProject) {
$value = 1;
$databaseListener = function (string $event, Document $document, Stats $usage) {
$multiplier = 1;
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$value = -1;
$multiplier = -1;
}
switch (true) {
case $document->getCollection() === 'teams':
$queueForUsage
->addMetric(METRIC_TEAMS, $value); // per project
$collection = $document->getCollection();
switch ($collection) {
case 'users':
$usage->setParam('users.{scope}.count.total', 1 * $multiplier);
break;
case $document->getCollection() === 'users':
$queueForUsage
->addMetric(METRIC_USERS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
case 'databases':
$usage->setParam('databases.{scope}.count.total', 1 * $multiplier);
break;
case $document->getCollection() === 'sessions': // sessions
$queueForUsage
->addMetric(METRIC_SESSIONS, $value); //per project
case 'buckets':
$usage->setParam('buckets.{scope}.count.total', 1 * $multiplier);
break;
case $document->getCollection() === 'databases': // databases
$queueForUsage
->addMetric(METRIC_DATABASES, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$queueForUsage
->addMetric(METRIC_COLLECTIONS, $value) // per project
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database
;
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$collectionInternalId = $parts[3] ?? 0;
$queueForUsage
->addMetric(METRIC_DOCUMENTS, $value) // per project
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection
break;
case $document->getCollection() === 'buckets': //buckets
$queueForUsage
->addMetric(METRIC_BUCKETS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'bucket_'): // files
$queueForUsage
->addMetric(METRIC_FILES, $value) // per project
->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project
->addMetric(str_replace('{bucketInternalId}', $document->getAttribute('bucketInternalId'), METRIC_BUCKET_ID_FILES), $value) // per bucket
->addMetric(str_replace('{bucketInternalId}', $document->getAttribute('bucketInternalId'), METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket
break;
case $document->getCollection() === 'functions':
$queueForUsage
->addMetric(METRIC_FUNCTIONS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
->addReduce($document);
}
break;
case $document->getCollection() === 'deployments':
$queueForUsage
->addMetric(METRIC_DEPLOYMENTS, $value) // per project
->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value)// per function
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), $document->getAttribute('size') * $value);// per function
break;
case $document->getCollection() === 'executions':
$queueForUsage
->addMetric(METRIC_EXECUTIONS, $value) // per project
->addMetric(str_replace('{functionInternalId}', $document->getAttribute('functionInternalId'), METRIC_FUNCTION_ID_EXECUTIONS), $value);// per function
case 'deployments':
$usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size') * $multiplier);
break;
default:
if (strpos($collection, 'bucket_') === 0) {
$usage
->setParam('bucketId', $document->getAttribute('bucketId'))
->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal') * $multiplier)
->setParam('files.{scope}.count.total', 1 * $multiplier);
} elseif (strpos($collection, 'database_') === 0) {
$usage
->setParam('databaseId', $document->getAttribute('databaseId'));
if (strpos($collection, '_collection_') !== false) {
$usage
->setParam('collectionId', $document->getAttribute('$collectionId'))
->setParam('documents.{scope}.count.total', 1 * $multiplier);
} else {
$usage->setParam('collections.{scope}.count.total', 1 * $multiplier);
}
}
break;
}
};
@@ -157,10 +101,10 @@ App::init()
->inject('deletes')
->inject('database')
->inject('dbForProject')
->inject('queueForUsage')
->inject('mode')
->inject('mails')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Delete $deletes, EventDatabase $database, Database $dbForProject, Usage $queueForUsage, string $mode, Mail $mails) use ($databaseListener) {
->inject('usage')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Delete $deletes, EventDatabase $database, Database $dbForProject, string $mode, Mail $mails, Stats $usage) use ($databaseListener) {
$route = $utopia->getRoute();
@@ -242,25 +186,19 @@ App::init()
->setProject($project)
->setUser($user);
$smtp = $project->getAttribute('smtp', []);
if (!empty($smtp) && ($smtp['enabled'] ?? false)) {
$mails
->setSmtpHost($smtp['host'] ?? '')
->setSmtpPort($smtp['port'] ?? 25)
->setSmtpUsername($smtp['username'] ?? '')
->setSmtpPassword($smtp['password'] ?? '')
->setSmtpSenderEmail($smtp['sender'] ?? '')
->setSmtpReplyTo($smtp['replyTo'] ?? '');
}
$usage
->setParam('projectInternalId', $project->getInternalId())
->setParam('projectId', $project->getId())
->setParam('project.{scope}.network.requests', 1)
->setParam('httpMethod', $request->getMethod())
->setParam('project.{scope}.network.inbound', 0)
->setParam('project.{scope}.network.outbound', 0);
$deletes->setProject($project);
$database->setProject($project);
$calculateUsage = fn ($event, Document $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject);
$dbForProject
->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', $calculateUsage)
->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', $calculateUsage);
$dbForProject->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage));
$dbForProject->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage));
$useCache = $route->getLabel('cache', false);
@@ -282,7 +220,10 @@ App::init()
$bucket = Authorization::skip(fn () => $dbForProject->getDocument('buckets', $bucketId));
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && $mode !== APP_MODE_ADMIN)) {
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAPIKey && !$isPrivilegedUser)) {
throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
}
@@ -423,14 +364,14 @@ App::shutdown()
->inject('user')
->inject('events')
->inject('audits')
->inject('usage')
->inject('deletes')
->inject('database')
->inject('dbForProject')
->inject('queueForFunctions')
->inject('queueForUsage')
->inject('mode')
->inject('dbForConsole')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Delete $deletes, EventDatabase $database, Database $dbForProject, Func $queueForFunctions, Usage $queueForUsage, string $mode, Database $dbForConsole) use ($parseLabel) {
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) {
$responsePayload = $response->getPayload();
@@ -583,46 +524,35 @@ App::shutdown()
}
}
if ($project->getId() !== 'console') {
if ($mode !== APP_MODE_ADMIN) {
$fileSize = 0;
$file = $request->getFiles('file');
if (!empty($file)) {
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
if (
App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled'
&& $project->getId()
&& !empty($route->getLabel('sdk.namespace', null))
) { // Don't calculate console usage on admin mode
$metric = $route->getLabel('usage.metric', '');
$usageParams = $route->getLabel('usage.params', []);
$queueForUsage
->addMetric(METRIC_NETWORK_REQUESTS, 1)
->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize)
->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize());
}
$queueForUsage
->setProject($project)
->trigger();
}
/**
* Update user last activity
*/
if (!$user->isEmpty()) {
$accessedAt = $user->getAttribute('accessedAt', '');
if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCCESS)) > $accessedAt) {
$user->setAttribute('accessedAt', DateTime::now());
if (APP_MODE_ADMIN !== $mode) {
$dbForProject->updateDocument('users', $user->getId(), $user);
} else {
$dbForConsole->updateDocument('users', $user->getId(), $user);
if (!empty($metric)) {
$usage->setParam($metric, 1);
foreach ($usageParams as $param) {
$param = $parseLabel($param, $responsePayload, $requestParams, $user);
$parts = explode(':', $param);
if (count($parts) != 2) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Usage params not properly set');
}
$usage->setParam($parts[0], $parts[1]);
}
}
}
});
App::init()
->groups(['usage'])
->action(function () {
if (App::getEnv('_APP_USAGE_STATS', 'enabled') !== 'enabled') {
throw new Exception(Exception::GENERAL_USAGE_DISABLED);
$fileSize = 0;
$file = $request->getFiles('file');
if (!empty($file)) {
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$usage
->setParam('project.{scope}.network.inbound', $request->getSize() + $fileSize)
->setParam('project.{scope}.network.outbound', $response->getSize())
->submit();
}
});
app/init.php
+31 -6
View File
@@ -32,6 +32,7 @@ use Appwrite\Network\Validator\Email;
use Appwrite\Network\Validator\Origin;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\URL\URL as AppwriteURL;
use Appwrite\Usage\Stats;
use Utopia\App;
use Utopia\Logger\Logger;
use Utopia\Cache\Adapter\Redis as RedisCache;
@@ -236,7 +237,6 @@ Config::load('providers', __DIR__ . '/config/providers.php');
Config::load('platforms', __DIR__ . '/config/platforms.php');
Config::load('collections', __DIR__ . '/config/collections.php');
Config::load('runtimes', __DIR__ . '/config/runtimes.php');
Config::load('usage', __DIR__ . '/config/usage.php');
Config::load('roles', __DIR__ . '/config/roles.php'); // User roles and scopes
Config::load('scopes', __DIR__ . '/config/scopes.php'); // User roles and scopes
Config::load('services', __DIR__ . '/config/services.php'); // List of services
@@ -293,7 +293,7 @@ Database::addFilter(
return $value;
},
function (mixed $value, Document $attribute) {
$formatOptions = json_decode($attribute->getAttribute('formatOptions', '[]'), true);
$formatOptions = \json_decode($attribute->getAttribute('formatOptions', '[]'), true);
if (isset($formatOptions['elements'])) {
$attribute->setAttribute('elements', $formatOptions['elements']);
}
@@ -363,7 +363,7 @@ Database::addFilter(
->find('indexes', [
Query::equal('collectionInternalId', [$document->getInternalId()]),
Query::equal('databaseInternalId', [$document->getAttribute('databaseInternalId')]),
Query::limit(64),
Query::limit($database->getLimitForIndexes()),
]);
}
);
@@ -770,6 +770,31 @@ $register->set('pools', function () {
return $group;
});
$register->set('influxdb', function () {
// Register DB connection
$host = App::getEnv('_APP_INFLUXDB_HOST', '');
$port = App::getEnv('_APP_INFLUXDB_PORT', '');
if (empty($host) || empty($port)) {
return;
}
$driver = new InfluxDB\Driver\Curl("http://{$host}:{$port}");
$client = new InfluxDB\Client($host, $port, '', '', false, false, 5);
$client->setDriver($driver);
return $client;
});
$register->set('statsd', function () {
// Register DB connection
$host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
$port = App::getEnv('_APP_STATSD_PORT', 8125);
$connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port);
$statsd = new \Domnikl\Statsd\Client($connection);
return $statsd;
});
$register->set('smtp', function () {
$mail = new PHPMailer(true);
@@ -873,9 +898,9 @@ App::setResource('queue', function (Group $pools) {
App::setResource('queueForFunctions', function (Connection $queue) {
return new Func($queue);
}, ['queue']);
App::setResource('queueForUsage', function (Connection $queue) {
return new Usage($queue);
}, ['queue']);
App::setResource('usage', function ($register) {
return new Stats($register->get('statsd'));
}, ['register']);
App::setResource('clients', function ($request, $console, $project) {
$console->setAttribute('platforms', [ // Always allow current host
'$collection' => ID::custom('platforms'),
app/views/install/compose.phtml
+50 -30
View File
@@ -80,6 +80,7 @@ services:
- mariadb
- redis
# - clamav
- influxdb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
@@ -115,6 +116,8 @@ services:
- _APP_SMTP_USERNAME
- _APP_SMTP_PASSWORD
- _APP_USAGE_STATS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_STORAGE_LIMIT
- _APP_STORAGE_PREVIEW_LIMIT
- _APP_STORAGE_ANTIVIRUS
@@ -131,6 +134,8 @@ services:
- _APP_EXECUTOR_HOST
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_STATSD_HOST
- _APP_STATSD_PORT
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
@@ -468,6 +473,35 @@ services:
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
appwrite-usage:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: usage
container_name: appwrite-usage
<<: *x-logging
restart: unless-stopped
networks:
- appwrite
depends_on:
- influxdb
- mariadb
environment:
- _APP_ENV
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-schedule:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: schedule
@@ -552,41 +586,26 @@ services:
# volumes:
# - appwrite-uploads:/storage/uploads
appwrite-worker-usage:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: worker-usage
influxdb:
image: appwrite/influxdb:1.5.0
container_name: appwrite-influxdb
<<: *x-logging
container_name: appwrite-worker-usage
restart: unless-stopped
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
- appwrite-influxdb:/var/lib/influxdb:rw
telegraf:
image: appwrite/telegraf:1.4.0
container_name: appwrite-telegraf
<<: *x-logging
restart: unless-stopped
networks:
- appwrite
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_USAGE_STATS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
networks:
gateway:
@@ -602,6 +621,7 @@ volumes:
appwrite-cache:
appwrite-uploads:
appwrite-certificates:
appwrite-influxdb:
appwrite-config:
appwrite-functions:
appwrite-builds:
app/worker.php
+6 -10
View File
@@ -4,10 +4,12 @@ require_once __DIR__ . '/init.php';
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Usage\Stats;
use Swoole\Runtime;
use Utopia\App;
use Utopia\Cache\Adapter\Sharding;
use Utopia\Cache\Cache;
use Utopia\CLI\CLI;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
@@ -86,22 +88,16 @@ Server::setResource('queueForFunctions', function (Registry $register) {
);
}, ['register']);
Server::setResource('queueForUsage', function (Registry $register) {
$pools = $register->get('pools');
return new Usage(
$pools
->get('queue')
->pop()
->getResource()
);
}, ['register']);
Server::setResource('log', fn() => new Log());
Server::setResource('logger', function ($register) {
return $register->get('logger');
}, ['register']);
Server::setResource('statsd', function ($register) {
return $register->get('statsd');
}, ['register']);
Server::setResource('pools', function ($register) {
return $register->get('pools');
}, ['register']);
app/workers/builds.php
+18 -14
View File
@@ -213,7 +213,7 @@ class BuildsV1 extends Worker
Console::execute('cp -rfn ' . $tmpTemplateDirectory . '/' . $templateRootDirectory . '/* ' . $tmpDirectory . '/' . $rootDirectory, '', $stdout, $stderr);
// Commit and push
$exit = Console::execute('git config --global user.email "security@appwrite.io" && git config --global user.name "Appwrite" && cd ' . $tmpDirectory . ' && git add . && git commit -m "Create \'' . \escapeshellcmd($function->getAttribute('name', '')) . '\' function" && git push origin ' . \escapeshellcmd($branchName), '', $stdout, $stderr);
$exit = Console::execute('git config --global user.email "team@appwrite.io" && git config --global user.name "Appwrite" && cd ' . $tmpDirectory . ' && git add . && git commit -m "Create \'' . \escapeshellcmd($function->getAttribute('name', '')) . '\' function" && git push origin ' . \escapeshellcmd($branchName), '', $stdout, $stderr);
if ($exit !== 0) {
throw new \Exception('Unable to push code repository: ' . $stderr);
@@ -478,7 +478,7 @@ class BuildsV1 extends Worker
* Send realtime Event
*/
$target = Realtime::fromPayload(
// Pass first, most verbose event pattern
// Pass first, most verbose event pattern
event: $allEvents[0],
payload: $build,
project: $project
@@ -490,19 +490,23 @@ class BuildsV1 extends Worker
channels: $target['channels'],
roles: $target['roles']
);
}
/** Trigger usage queue */
$this
->getUsageQueue()
->setProject($project)
->addMetric(METRIC_BUILDS, 1) // per project
->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0))
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000)
->trigger();
/** Update usage stats */
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$statsd = $register->get('statsd');
$usage = new Stats($statsd);
$usage
->setParam('projectInternalId', $project->getInternalId())
->setParam('projectId', $project->getId())
->setParam('functionId', $function->getId())
->setParam('builds.{scope}.compute', 1)
->setParam('buildStatus', $build->getAttribute('status', ''))
->setParam('buildTime', $build->getAttribute('duration'))
->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0)
->submit();
}
}
}
protected function runGitAction(string $status, GitHub $github, string $providerCommitHash, string $owner, string $repositoryName, Document $project, Document $function, string $deploymentId, Database $dbForProject, Database $dbForConsole): void
app/workers/functions.php
+27 -26
View File
@@ -2,18 +2,18 @@
require_once __DIR__ . '/../worker.php';
use Appwrite\Event\Usage;
use Domnikl\Statsd\Client;
use Utopia\Queue\Message;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Usage\Stats;
use Appwrite\Utopia\Response\Model\Execution;
use Executor\Executor;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
@@ -31,7 +31,7 @@ Server::setResource('execute', function () {
Log $log,
Func $queueForFunctions,
Database $dbForProject,
Usage $queueForUsage,
Client $statsd,
Document $project,
Document $function,
string $trigger,
@@ -47,7 +47,6 @@ Server::setResource('execute', function () {
) {
$user ??= new Document();
$functionId = $function->getId();
$functionInternalId = $function->getInternalId();
$deploymentId = $function->getAttribute('deployment', '');
$log->addTag('functionId', $functionId);
@@ -55,7 +54,6 @@ Server::setResource('execute', function () {
/** Check if deployment exists */
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
$deploymentInternalId = $deployment->getInternalId();
if ($deployment->getAttribute('resourceId') !== $functionId) {
throw new Exception('Deployment not found. Create deployment before trying to execute a function');
@@ -129,14 +127,6 @@ Server::setResource('execute', function () {
if ($execution->isEmpty()) {
throw new Exception('Failed to create or read execution');
}
/**
* Usage
*/
$queueForUsage
->addMetric(METRIC_EXECUTIONS, 1) // per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1); // per function
}
if ($execution->getAttribute('status') !== 'processing') {
@@ -186,13 +176,13 @@ Server::setResource('execute', function () {
$executionResponse = $executor->createExecution(
projectId: $project->getId(),
deploymentId: $deploymentId,
version: $function->getAttribute('version'),
body: \strlen($body) > 0 ? $body : null,
variables: $vars,
timeout: $function->getAttribute('timeout', 0),
image: $runtime['image'],
source: $build->getAttribute('path', ''),
entrypoint: $deployment->getAttribute('entrypoint', ''),
version: $function->getAttribute('version'),
path: $path,
method: $method,
headers: $headers,
@@ -275,13 +265,24 @@ Server::setResource('execute', function () {
roles: $target['roles']
);
/** Trigger usage queue */
$queueForUsage
->setProject($project)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
->trigger()
;
/** Update usage stats */
if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
$usage = new Stats($statsd);
$usage
->setParam('projectId', $project->getId())
->setParam('projectInternalId', $project->getInternalId())
->setParam('functionId', $function->getId()) // TODO: We should use functionInternalId in usage stats
->setParam('executions.{scope}.compute', 1)
->setParam('executionStatus', $execution->getAttribute('status', ''))
->setParam('executionTime', $execution->getAttribute('duration'))
->setParam('networkRequestSize', 0)
->setParam('networkResponseSize', 0)
->submit();
}
if (!empty($error)) {
throw new Exception($error, $errorCode);
}
};
});
@@ -289,10 +290,10 @@ $server->job()
->inject('message')
->inject('dbForProject')
->inject('queueForFunctions')
->inject('queueForUsage')
->inject('statsd')
->inject('execute')
->inject('log')
->action(function (Message $message, Database $dbForProject, Func $queueForFunctions, Usage $queueForUsage, callable $execute, Log $log) {
->action(function (Message $message, Database $dbForProject, Func $queueForFunctions, Client $statsd, callable $execute, Log $log) {
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
@@ -334,7 +335,7 @@ $server->job()
}
Console::success('Iterating function: ' . $function->getAttribute('name'));
$execute(
queueForUsage: $queueForUsage,
statsd: $statsd,
dbForProject: $dbForProject,
project: $project,
function: $function,
@@ -382,7 +383,7 @@ $server->job()
path: $payload['path'],
method: $payload['method'],
headers: $payload['headers'],
queueForUsage: $queueForUsage,
statsd: $statsd,
);
break;
case 'schedule':
@@ -402,7 +403,7 @@ $server->job()
path: $payload['path'],
method: $payload['method'],
headers: $payload['headers'],
queueForUsage: $queueForUsage,
statsd: $statsd,
);
break;
}
app/workers/usage.php
-288
View File
@@ -1,288 +0,0 @@
<?php
require_once __DIR__ . '/../worker.php';
use Swoole\Timer;
use Utopia\App;
use Utopia\Cache\Cache;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Validator\Authorization;
use Utopia\Queue\Message;
use Utopia\CLI\Console;
use Utopia\Queue\Server;
use Utopia\Registry\Registry;
Authorization::disable();
Authorization::setDefaultStatus(false);
$stats = [];
$periods['1h'] = 'Y-m-d H:00';
$periods['1d'] = 'Y-m-d 00:00';
//$periods['1m'] = 'Y-m-1 00:00';
$periods['inf'] = '0000-00-00 00:00';
const INFINITY_PERIOD = '_inf_';
/**
* On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files.
* When we remove a parent document we need to deduct his children aggregation from the project scope.
*/
Server::setResource('reduce', function (Cache $cache, Registry $register, $pools) {
return function ($database, $projectInternalId, Document $document, array &$metrics) use ($pools, $cache, $register): void {
try {
$dbForProject = new Database(
$pools
->get($database)
->pop()
->getResource(),
$cache
);
$dbForProject->setNamespace('_' . $projectInternalId);
switch (true) {
case $document->getCollection() === 'users': // users
$sessions = count($document->getAttribute(METRIC_SESSIONS, 0));
if (!empty($sessions)) {
$metrics[] = [
'key' => METRIC_SESSIONS,
'value' => ($sessions * -1),
];
}
break;
case $document->getCollection() === 'databases': // databases
$collections = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
$documents = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
if (!empty($collections['value'])) {
$metrics[] = [
'key' => METRIC_COLLECTIONS,
'value' => ($collections['value'] * -1),
];
}
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
}
break;
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$documents = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
$metrics[] = [
'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS),
'value' => ($documents['value'] * -1),
];
}
break;
case $document->getCollection() === 'buckets':
$files = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
$storage = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
if (!empty($files['value'])) {
$metrics[] = [
'key' => METRIC_FILES,
'value' => ($files['value'] * -1),
];
}
if (!empty($storage['value'])) {
$metrics[] = [
'key' => METRIC_FILES_STORAGE,
'value' => ($storage['value'] * -1),
];
}
break;
case $document->getCollection() === 'functions':
$deployments = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
$deploymentsStorage = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
$builds = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
$buildsStorage = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
$buildsCompute = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
$executions = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
$executionsCompute = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
if (!empty($deployments['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS,
'value' => ($deployments['value'] * -1),
];
}
if (!empty($deploymentsStorage['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS_STORAGE,
'value' => ($deploymentsStorage['value'] * -1),
];
}
if (!empty($builds['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS,
'value' => ($builds['value'] * -1),
];
}
if (!empty($buildsStorage['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_STORAGE,
'value' => ($buildsStorage['value'] * -1),
];
}
if (!empty($buildsCompute['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_COMPUTE,
'value' => ($buildsCompute['value'] * -1),
];
}
if (!empty($executions['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS,
'value' => ($executions['value'] * -1),
];
}
if (!empty($executionsCompute['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS_COMPUTE,
'value' => ($executionsCompute['value'] * -1),
];
}
break;
default:
break;
}
} catch (\Exception $e) {
console::error("[reducer] " . " {DateTime::now()} " . " {$projectInternalId} " . " {$e->getMessage()}");
} finally {
$pools->reclaim();
}
};
}, ['cache', 'register', 'pools']);
$server->job()
->inject('message')
->inject('reduce')
->action(function (Message $message, callable $reduce) use (&$stats) {
$payload = $message->getPayload() ?? [];
$project = new Document($payload['project'] ?? []);
$projectId = $project->getInternalId();
foreach ($payload['reduce'] ?? [] as $document) {
if (empty($document)) {
continue;
}
$reduce(
database: $project->getAttribute('database'),
projectInternalId: $project->getInternalId(),
document: new Document($document),
metrics: $payload['metrics'],
);
}
$stats[$projectId]['database'] = $project->getAttribute('database');
foreach ($payload['metrics'] ?? [] as $metric) {
if (!isset($stats[$projectId]['keys'][$metric['key']])) {
$stats[$projectId]['keys'][$metric['key']] = $metric['value'];
continue;
}
$stats[$projectId]['keys'][$metric['key']] += $metric['value'];
}
});
$server
->workerStart()
->inject('register')
->inject('cache')
->inject('pools')
->action(function ($register, $cache, $pools) use ($periods, &$stats) {
Timer::tick(30000, function () use ($register, $cache, $pools, $periods, &$stats) {
$offset = count($stats);
$projects = array_slice($stats, 0, $offset, true);
array_splice($stats, 0, $offset);
foreach ($projects as $projectInternalId => $project) {
try {
$dbForProject = new Database(
$pools
->get($project['database'])
->pop()
->getResource(),
$cache
);
$dbForProject->setNamespace('_' . $projectInternalId);
foreach ($project['keys'] ?? [] as $key => $value) {
if ($value == 0) {
continue;
}
foreach ($periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
try {
$dbForProject->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => App::getEnv('_APP_REGION', 'default'),
]));
} catch (Duplicate $th) {
if ($value < 0) {
$dbForProject->decreaseDocumentAttribute(
'stats',
$id,
'value',
abs($value)
);
} else {
$dbForProject->increaseDocumentAttribute(
'stats',
$id,
'value',
$value
);
}
}
}
}
if (!empty($project['keys'])) {
$dbForProject->createDocument('statsLogger', new Document([
'time' => DateTime::now(),
'metrics' => $project['keys'],
]));
}
} catch (\Exception $e) {
$now = DateTime::now();
console::error("[Error] " . " Time: {$now} " . " projectInternalId: {$projectInternalId}" . " File: {$e->getFile()}" . " Line: {$e->getLine()} " . " message: {$e->getMessage()}");
} finally {
$pools->reclaim();
}
}
});
});
$server->start();
bin/usage
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php usage $@
bin/worker-usage
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
QUEUE=v1-usage php /usr/src/code/app/workers/usage.php $@
composer.json
+2 -1
View File
@@ -58,7 +58,7 @@
"utopia-php/logger": "0.3.*",
"utopia-php/messaging": "0.1.*",
"utopia-php/orchestration": "0.9.*",
"utopia-php/platform": "0.4.1",
"utopia-php/platform": "0.4.*",
"utopia-php/pools": "0.4.*",
"utopia-php/preloader": "0.2.*",
"utopia-php/queue": "0.5.*",
@@ -70,6 +70,7 @@
"resque/php-resque": "1.3.6",
"matomo/device-detector": "6.1.*",
"dragonmantank/cron-expression": "3.3.2",
"influxdb/influxdb-php": "1.15.2",
"phpmailer/phpmailer": "6.8.0",
"chillerlan/php-qrcode": "4.3.4",
"adhocore/jwt": "1.1.2",
composer.lock
Generated
+679 -24
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "83427d18095b42c274d2bd6eb302e4aa",
"content-hash": "12d4da6b26d9f15f632cd681a837e026",
"packages": [
{
"name": "adhocore/jwt",
@@ -520,6 +520,398 @@
],
"time": "2022-09-10T18:51:20+00:00"
},
{
"name": "guzzlehttp/guzzle",
"version": "7.7.0",
"source": {
"type": "git",
"url": "https://github.com/guzzle/guzzle.git",
"reference": "fb7566caccf22d74d1ab270de3551f72a58399f5"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/guzzle/guzzle/zipball/fb7566caccf22d74d1ab270de3551f72a58399f5",
"reference": "fb7566caccf22d74d1ab270de3551f72a58399f5",
"shasum": ""
},
"require": {
"ext-json": "*",
"guzzlehttp/promises": "^1.5.3 || ^2.0",
"guzzlehttp/psr7": "^1.9.1 || ^2.4.5",
"php": "^7.2.5 || ^8.0",
"psr/http-client": "^1.0",
"symfony/deprecation-contracts": "^2.2 || ^3.0"
},
"provide": {
"psr/http-client-implementation": "1.0"
},
"require-dev": {
"bamarni/composer-bin-plugin": "^1.8.1",
"ext-curl": "*",
"php-http/client-integration-tests": "dev-master#2c025848417c1135031fdf9c728ee53d0a7ceaee as 3.0.999",
"php-http/message-factory": "^1.1",
"phpunit/phpunit": "^8.5.29 || ^9.5.23",
"psr/log": "^1.1 || ^2.0 || ^3.0"
},
"suggest": {
"ext-curl": "Required for CURL handler support",
"ext-intl": "Required for Internationalized Domain Name (IDN) support",
"psr/log": "Required for using the Log middleware"
},
"type": "library",
"extra": {
"bamarni-bin": {
"bin-links": true,
"forward-command": false
}
},
"autoload": {
"files": [
"src/functions_include.php"
],
"psr-4": {
"GuzzleHttp\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Graham Campbell",
"email": "hello@gjcampbell.co.uk",
"homepage": "https://github.com/GrahamCampbell"
},
{
"name": "Michael Dowling",
"email": "mtdowling@gmail.com",
"homepage": "https://github.com/mtdowling"
},
{
"name": "Jeremy Lindblom",
"email": "jeremeamia@gmail.com",
"homepage": "https://github.com/jeremeamia"
},
{
"name": "George Mponos",
"email": "gmponos@gmail.com",
"homepage": "https://github.com/gmponos"
},
{
"name": "Tobias Nyholm",
"email": "tobias.nyholm@gmail.com",
"homepage": "https://github.com/Nyholm"
},
{
"name": "Márk Sági-Kazár",
"email": "mark.sagikazar@gmail.com",
"homepage": "https://github.com/sagikazarmark"
},
{
"name": "Tobias Schultze",
"email": "webmaster@tubo-world.de",
"homepage": "https://github.com/Tobion"
}
],
"description": "Guzzle is a PHP HTTP client library",
"keywords": [
"client",
"curl",
"framework",
"http",
"http client",
"psr-18",
"psr-7",
"rest",
"web service"
],
"support": {
"issues": "https://github.com/guzzle/guzzle/issues",
"source": "https://github.com/guzzle/guzzle/tree/7.7.0"
},
"funding": [
{
"url": "https://github.com/GrahamCampbell",
"type": "github"
},
{
"url": "https://github.com/Nyholm",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/guzzlehttp/guzzle",
"type": "tidelift"
}
],
"time": "2023-05-21T14:04:53+00:00"
},
{
"name": "guzzlehttp/promises",
"version": "2.0.1",
"source": {
"type": "git",
"url": "https://github.com/guzzle/promises.git",
"reference": "111166291a0f8130081195ac4556a5587d7f1b5d"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/guzzle/promises/zipball/111166291a0f8130081195ac4556a5587d7f1b5d",
"reference": "111166291a0f8130081195ac4556a5587d7f1b5d",
"shasum": ""
},
"require": {
"php": "^7.2.5 || ^8.0"
},
"require-dev": {
"bamarni/composer-bin-plugin": "^1.8.1",
"phpunit/phpunit": "^8.5.29 || ^9.5.23"
},
"type": "library",
"extra": {
"bamarni-bin": {
"bin-links": true,
"forward-command": false
}
},
"autoload": {
"psr-4": {
"GuzzleHttp\\Promise\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Graham Campbell",
"email": "hello@gjcampbell.co.uk",
"homepage": "https://github.com/GrahamCampbell"
},
{
"name": "Michael Dowling",
"email": "mtdowling@gmail.com",
"homepage": "https://github.com/mtdowling"
},
{
"name": "Tobias Nyholm",
"email": "tobias.nyholm@gmail.com",
"homepage": "https://github.com/Nyholm"
},
{
"name": "Tobias Schultze",
"email": "webmaster@tubo-world.de",
"homepage": "https://github.com/Tobion"
}
],
"description": "Guzzle promises library",
"keywords": [
"promise"
],
"support": {
"issues": "https://github.com/guzzle/promises/issues",
"source": "https://github.com/guzzle/promises/tree/2.0.1"
},
"funding": [
{
"url": "https://github.com/GrahamCampbell",
"type": "github"
},
{
"url": "https://github.com/Nyholm",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/guzzlehttp/promises",
"type": "tidelift"
}
],
"time": "2023-08-03T15:11:55+00:00"
},
{
"name": "guzzlehttp/psr7",
"version": "2.6.0",
"source": {
"type": "git",
"url": "https://github.com/guzzle/psr7.git",
"reference": "8bd7c33a0734ae1c5d074360512beb716bef3f77"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/guzzle/psr7/zipball/8bd7c33a0734ae1c5d074360512beb716bef3f77",
"reference": "8bd7c33a0734ae1c5d074360512beb716bef3f77",
"shasum": ""
},
"require": {
"php": "^7.2.5 || ^8.0",
"psr/http-factory": "^1.0",
"psr/http-message": "^1.1 || ^2.0",
"ralouphie/getallheaders": "^3.0"
},
"provide": {
"psr/http-factory-implementation": "1.0",
"psr/http-message-implementation": "1.0"
},
"require-dev": {
"bamarni/composer-bin-plugin": "^1.8.1",
"http-interop/http-factory-tests": "^0.9",
"phpunit/phpunit": "^8.5.29 || ^9.5.23"
},
"suggest": {
"laminas/laminas-httphandlerrunner": "Emit PSR-7 responses"
},
"type": "library",
"extra": {
"bamarni-bin": {
"bin-links": true,
"forward-command": false
}
},
"autoload": {
"psr-4": {
"GuzzleHttp\\Psr7\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Graham Campbell",
"email": "hello@gjcampbell.co.uk",
"homepage": "https://github.com/GrahamCampbell"
},
{
"name": "Michael Dowling",
"email": "mtdowling@gmail.com",
"homepage": "https://github.com/mtdowling"
},
{
"name": "George Mponos",
"email": "gmponos@gmail.com",
"homepage": "https://github.com/gmponos"
},
{
"name": "Tobias Nyholm",
"email": "tobias.nyholm@gmail.com",
"homepage": "https://github.com/Nyholm"
},
{
"name": "Márk Sági-Kazár",
"email": "mark.sagikazar@gmail.com",
"homepage": "https://github.com/sagikazarmark"
},
{
"name": "Tobias Schultze",
"email": "webmaster@tubo-world.de",
"homepage": "https://github.com/Tobion"
},
{
"name": "Márk Sági-Kazár",
"email": "mark.sagikazar@gmail.com",
"homepage": "https://sagikazarmark.hu"
}
],
"description": "PSR-7 message implementation that also provides common utility methods",
"keywords": [
"http",
"message",
"psr-7",
"request",
"response",
"stream",
"uri",
"url"
],
"support": {
"issues": "https://github.com/guzzle/psr7/issues",
"source": "https://github.com/guzzle/psr7/tree/2.6.0"
},
"funding": [
{
"url": "https://github.com/GrahamCampbell",
"type": "github"
},
{
"url": "https://github.com/Nyholm",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/guzzlehttp/psr7",
"type": "tidelift"
}
],
"time": "2023-08-03T15:06:02+00:00"
},
{
"name": "influxdb/influxdb-php",
"version": "1.15.2",
"source": {
"type": "git",
"url": "https://github.com/influxdata/influxdb-php.git",
"reference": "d6e59f4f04ab9107574fda69c2cbe36671253d03"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/influxdata/influxdb-php/zipball/d6e59f4f04ab9107574fda69c2cbe36671253d03",
"reference": "d6e59f4f04ab9107574fda69c2cbe36671253d03",
"shasum": ""
},
"require": {
"guzzlehttp/guzzle": "^6.0|^7.0",
"php": "^5.5 || ^7.0 || ^8.0"
},
"require-dev": {
"dms/phpunit-arraysubset-asserts": "^0.2.1",
"phpunit/phpunit": "^9.5"
},
"suggest": {
"ext-curl": "Curl extension, needed for Curl driver",
"stefanotorresi/influxdb-php-async": "An asyncronous client for InfluxDB, implemented via ReactPHP."
},
"type": "library",
"autoload": {
"psr-4": {
"InfluxDB\\": "src/InfluxDB"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Stephen Hoogendijk",
"email": "stephen@tca0.nl"
},
{
"name": "Daniel Martinez",
"email": "danimartcas@hotmail.com"
},
{
"name": "Gianluca Arbezzano",
"email": "gianarb92@gmail.com"
}
],
"description": "InfluxDB client library for PHP",
"keywords": [
"client",
"influxdata",
"influxdb",
"influxdb class",
"influxdb client",
"influxdb library",
"time series"
],
"support": {
"issues": "https://github.com/influxdata/influxdb-php/issues",
"source": "https://github.com/influxdata/influxdb-php/tree/1.15.2"
},
"abandoned": true,
"time": "2020-12-26T17:45:17+00:00"
},
{
"name": "jean85/pretty-package-versions",
"version": "1.6.0",
@@ -992,6 +1384,166 @@
],
"time": "2023-03-06T14:43:22+00:00"
},
{
"name": "psr/http-client",
"version": "1.0.2",
"source": {
"type": "git",
"url": "https://github.com/php-fig/http-client.git",
"reference": "0955afe48220520692d2d09f7ab7e0f93ffd6a31"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/php-fig/http-client/zipball/0955afe48220520692d2d09f7ab7e0f93ffd6a31",
"reference": "0955afe48220520692d2d09f7ab7e0f93ffd6a31",
"shasum": ""
},
"require": {
"php": "^7.0 || ^8.0",
"psr/http-message": "^1.0 || ^2.0"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.0.x-dev"
}
},
"autoload": {
"psr-4": {
"Psr\\Http\\Client\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "PHP-FIG",
"homepage": "https://www.php-fig.org/"
}
],
"description": "Common interface for HTTP clients",
"homepage": "https://github.com/php-fig/http-client",
"keywords": [
"http",
"http-client",
"psr",
"psr-18"
],
"support": {
"source": "https://github.com/php-fig/http-client/tree/1.0.2"
},
"time": "2023-04-10T20:12:12+00:00"
},
{
"name": "psr/http-factory",
"version": "1.0.2",
"source": {
"type": "git",
"url": "https://github.com/php-fig/http-factory.git",
"reference": "e616d01114759c4c489f93b099585439f795fe35"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/php-fig/http-factory/zipball/e616d01114759c4c489f93b099585439f795fe35",
"reference": "e616d01114759c4c489f93b099585439f795fe35",
"shasum": ""
},
"require": {
"php": ">=7.0.0",
"psr/http-message": "^1.0 || ^2.0"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.0.x-dev"
}
},
"autoload": {
"psr-4": {
"Psr\\Http\\Message\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "PHP-FIG",
"homepage": "https://www.php-fig.org/"
}
],
"description": "Common interfaces for PSR-7 HTTP message factories",
"keywords": [
"factory",
"http",
"message",
"psr",
"psr-17",
"psr-7",
"request",
"response"
],
"support": {
"source": "https://github.com/php-fig/http-factory/tree/1.0.2"
},
"time": "2023-04-10T20:10:41+00:00"
},
{
"name": "psr/http-message",
"version": "2.0",
"source": {
"type": "git",
"url": "https://github.com/php-fig/http-message.git",
"reference": "402d35bcb92c70c026d1a6a9883f06b2ead23d71"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/php-fig/http-message/zipball/402d35bcb92c70c026d1a6a9883f06b2ead23d71",
"reference": "402d35bcb92c70c026d1a6a9883f06b2ead23d71",
"shasum": ""
},
"require": {
"php": "^7.2 || ^8.0"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "2.0.x-dev"
}
},
"autoload": {
"psr-4": {
"Psr\\Http\\Message\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "PHP-FIG",
"homepage": "https://www.php-fig.org/"
}
],
"description": "Common interface for HTTP messages",
"homepage": "https://github.com/php-fig/http-message",
"keywords": [
"http",
"http-message",
"psr",
"psr-7",
"request",
"response"
],
"support": {
"source": "https://github.com/php-fig/http-message/tree/2.0"
},
"time": "2023-04-04T09:54:51+00:00"
},
{
"name": "psr/log",
"version": "1.1.4",
@@ -1042,6 +1594,50 @@
},
"time": "2021-05-03T11:20:27+00:00"
},
{
"name": "ralouphie/getallheaders",
"version": "3.0.3",
"source": {
"type": "git",
"url": "https://github.com/ralouphie/getallheaders.git",
"reference": "120b605dfeb996808c31b6477290a714d356e822"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/ralouphie/getallheaders/zipball/120b605dfeb996808c31b6477290a714d356e822",
"reference": "120b605dfeb996808c31b6477290a714d356e822",
"shasum": ""
},
"require": {
"php": ">=5.6"
},
"require-dev": {
"php-coveralls/php-coveralls": "^2.1",
"phpunit/phpunit": "^5 || ^6.5"
},
"type": "library",
"autoload": {
"files": [
"src/getallheaders.php"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Ralph Khattar",
"email": "ralph.khattar@gmail.com"
}
],
"description": "A polyfill for getallheaders.",
"support": {
"issues": "https://github.com/ralouphie/getallheaders/issues",
"source": "https://github.com/ralouphie/getallheaders/tree/develop"
},
"time": "2019-03-08T08:55:37+00:00"
},
{
"name": "resque/php-resque",
"version": "v1.3.6",
@@ -1181,6 +1777,73 @@
},
"time": "2021-06-04T20:33:46+00:00"
},
{
"name": "symfony/deprecation-contracts",
"version": "v3.3.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/deprecation-contracts.git",
"reference": "7c3aff79d10325257a001fcf92d991f24fc967cf"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/7c3aff79d10325257a001fcf92d991f24fc967cf",
"reference": "7c3aff79d10325257a001fcf92d991f24fc967cf",
"shasum": ""
},
"require": {
"php": ">=8.1"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-main": "3.4-dev"
},
"thanks": {
"name": "symfony/contracts",
"url": "https://github.com/symfony/contracts"
}
},
"autoload": {
"files": [
"function.php"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Nicolas Grekas",
"email": "p@tchwork.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "A generic function and convention to trigger deprecation notices",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/deprecation-contracts/tree/v3.3.0"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2023-05-23T14:45:45+00:00"
},
{
"name": "symfony/polyfill-php80",
"version": "v1.27.0",
@@ -1557,19 +2220,20 @@
},
{
"name": "utopia-php/database",
"version": "0.42.2",
"version": "0.42.3",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/database.git",
"reference": "bc5ceb30c85fb685b0b5704d2f74886d813ebd41"
"reference": "ab0e2f8ad46884f69b354cd8ee84a1a75fee26d1"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/database/zipball/bc5ceb30c85fb685b0b5704d2f74886d813ebd41",
"reference": "bc5ceb30c85fb685b0b5704d2f74886d813ebd41",
"url": "https://api.github.com/repos/utopia-php/database/zipball/ab0e2f8ad46884f69b354cd8ee84a1a75fee26d1",
"reference": "ab0e2f8ad46884f69b354cd8ee84a1a75fee26d1",
"shasum": ""
},
"require": {
"ext-mbstring": "*",
"ext-pdo": "*",
"php": ">=8.0",
"utopia-php/cache": "0.8.*",
@@ -1607,9 +2271,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/database/issues",
"source": "https://github.com/utopia-php/database/tree/0.42.2"
"source": "https://github.com/utopia-php/database/tree/0.42.3"
},
"time": "2023-08-17T19:04:37+00:00"
"time": "2023-08-22T02:15:28+00:00"
},
{
"name": "utopia-php/domains",
@@ -2564,16 +3228,16 @@
},
{
"name": "utopia-php/vcs",
"version": "dev-feat-clone-commit-hash",
"version": "0.2.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/vcs.git",
"reference": "5ed5d1685b3b714d6f20f7922ab4f5f5188313ee"
"reference": "ec388e056fec8675b5fe1fc7a4f77e4dc3f328cf"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/vcs/zipball/5ed5d1685b3b714d6f20f7922ab4f5f5188313ee",
"reference": "5ed5d1685b3b714d6f20f7922ab4f5f5188313ee",
"url": "https://api.github.com/repos/utopia-php/vcs/zipball/ec388e056fec8675b5fe1fc7a4f77e4dc3f328cf",
"reference": "ec388e056fec8675b5fe1fc7a4f77e4dc3f328cf",
"shasum": ""
},
"require": {
@@ -2607,9 +3271,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/vcs/issues",
"source": "https://github.com/utopia-php/vcs/tree/feat-clone-commit-hash"
"source": "https://github.com/utopia-php/vcs/tree/0.2.0"
},
"time": "2023-08-21T09:02:54+00:00"
"time": "2023-08-21T10:59:48+00:00"
},
{
"name": "utopia-php/websocket",
@@ -5406,18 +6070,9 @@
"time": "2023-07-26T07:16:09+00:00"
}
],
"aliases": [
{
"package": "utopia-php/vcs",
"version": "dev-feat-clone-commit-hash",
"alias": "0.1.99",
"alias_normalized": "0.1.99.0"
}
],
"aliases": [],
"minimum-stability": "stable",
"stability-flags": {
"utopia-php/vcs": 20
},
"stability-flags": [],
"prefer-stable": false,
"prefer-lowest": false,
"platform": {
docker-compose.yml
+56 -6
View File
@@ -138,6 +138,8 @@ services:
- _APP_SMTP_PASSWORD
- _APP_USERS_STATS_RECIPIENTS
- _APP_USAGE_STATS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_STORAGE_LIMIT
- _APP_STORAGE_PREVIEW_LIMIT
- _APP_STORAGE_ANTIVIRUS
@@ -154,6 +156,8 @@ services:
- _APP_EXECUTOR_HOST
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_STATSD_HOST
- _APP_STATSD_PORT
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
@@ -689,18 +693,19 @@ services:
- _APP_MAINTENANCE_RETENTION_AUDIT
- _APP_MAINTENANCE_RETENTION_SCHEDULES
appwrite-worker-usage:
entrypoint: worker-usage
appwrite-usage:
entrypoint: usage
<<: *x-logging
container_name: appwrite-worker-usage
container_name: appwrite-usage
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./dev:/usr/local/dev
depends_on:
- redis
- influxdb
- mariadb
environment:
- _APP_ENV
@@ -713,6 +718,9 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
@@ -772,7 +780,7 @@ services:
hostname: executor
<<: *x-logging
stop_signal: SIGINT
image: openruntimes/executor:0.3.3
image: openruntimes/executor:0.3.4
networks:
- appwrite
- runtimes
@@ -869,6 +877,26 @@ services:
# - appwrite
# volumes:
# - appwrite-uploads:/storage/uploads
influxdb:
image: appwrite/influxdb:1.5.0
container_name: appwrite-influxdb
<<: *x-logging
networks:
- appwrite
volumes:
- appwrite-influxdb:/var/lib/influxdb:rw
telegraf:
image: appwrite/telegraf:1.4.0
container_name: appwrite-telegraf
<<: *x-logging
networks:
- appwrite
environment:
- _APP_INFLUXDB_HOST
- _APP_INFLUXDB_PORT
# Dev Tools Start ------------------------------------------------------------------------------------------
#
# The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack
@@ -879,6 +907,7 @@ services:
# RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks
# RedisCommander - A nice UI for exploring Redis data
# Resque - A nice UI for exploring Redis pub/sub, view the different queues workloads, pending and failed tasks
# Chronograf - A nice UI for exploring InfluxDB data
# Webgrind - A nice UI for exploring and debugging code-level stuff
maildev: # used mainly for dev tests
@@ -941,6 +970,25 @@ services:
# - RESQUE_WEB_PORT=6379
# - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user
# - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password
# chronograf:
# image: chronograf:1.6
# container_name: appwrite-chronograf
# networks:
# - appwrite
# volumes:
# - appwrite-chronograf:/var/lib/chronograf
# ports:
# - "8888:8888"
# environment:
# - INFLUXDB_URL=http://influxdb:8086
# - KAPACITOR_URL=http://kapacitor:9092
# - AUTH_DURATION=48h
# - TOKEN_SECRET=duperduper5674829!jwt
# - GH_CLIENT_ID=d86f7145a41eacfc52cc
# - GH_CLIENT_SECRET=9e0081062367a2134e7f2ea95ba1a32d08b6c8ab
# - GH_ORGS=appwrite
# webgrind:
# image: 'jokkedk/webgrind:latest'
# volumes:
@@ -976,6 +1024,8 @@ volumes:
appwrite-cache:
appwrite-uploads:
appwrite-certificates:
appwrite-influxdb:
appwrite-config:
appwrite-functions:
appwrite-builds:
appwrite-config:
# appwrite-chronograf:
src/Appwrite/Platform/Services/Tasks.php
+1
View File
@@ -30,6 +30,7 @@ class Tasks extends Service
$this->type = self::TYPE_CLI;
$this
->addAction(Version::getName(), new Version())
->addAction(Usage::getName(), new Usage())
->addAction(Vars::getName(), new Vars())
->addAction(SSL::getName(), new SSL())
->addAction(Hamster::getName(), new Hamster())
src/Appwrite/Platform/Tasks/Usage.php
+60
View File
@@ -0,0 +1,60 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Usage\Calculators\TimeSeries;
use InfluxDB\Database as InfluxDatabase;
use Utopia\App;
use Utopia\CLI\Console;
use Utopia\Database\Database as UtopiaDatabase;
use Throwable;
use Utopia\Platform\Action;
use Utopia\Registry\Registry;
class Usage extends Action
{
public static function getName(): string
{
return 'usage';
}
public function __construct()
{
$this
->desc('Schedules syncing data from influxdb to Appwrite console db')
->inject('dbForConsole')
->inject('influxdb')
->inject('register')
->inject('getProjectDB')
->inject('logError')
->callback(fn ($dbForConsole, $influxDB, $register, $getProjectDB, $logError) => $this->action($dbForConsole, $influxDB, $register, $getProjectDB, $logError));
}
protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void
{
}
public function action(UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register, callable $getProjectDB, callable $logError)
{
Console::title('Usage Aggregation V1');
Console::success(APP_NAME . ' usage aggregation process v1 has started');
$errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action);
$interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
$region = App::getEnv('region', 'default');
$usage = new TimeSeries($region, $dbForConsole, $influxDB, $getProjectDB, $register, $errorLogger);
Console::loop(function () use ($interval, $usage) {
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds");
$loopStart = microtime(true);
$usage->collect();
$loopTook = microtime(true) - $loopStart;
$now = date('d-m-Y H:i:s', time());
Console::info("[{$now}] Aggregation took {$loopTook} seconds");
}, $interval);
}
}
src/Appwrite/Usage/Calculator.php
+15
View File
@@ -0,0 +1,15 @@
<?php
namespace Appwrite\Usage;
abstract class Calculator
{
protected string $region;
public function __construct(string $region)
{
$this->region = $region;
}
abstract public function collect(): void;
}
src/Appwrite/Usage/Calculators/TimeSeries.php
+557
View File
@@ -0,0 +1,557 @@
<?php
namespace Appwrite\Usage\Calculators;
use Utopia\App;
use Appwrite\Usage\Calculator;
use Utopia\Database\Database;
use Utopia\Database\Document;
use InfluxDB\Database as InfluxDatabase;
use DateTime;
use Utopia\Registry\Registry;
class TimeSeries extends Calculator
{
/**
* InfluxDB
*
* @var InfluxDatabase
*/
protected InfluxDatabase $influxDB;
/**
* Utopia Database
*
* @var Database
*/
protected Database $database;
/**
* Error Handler Callback
*
* @var callable
*/
protected $errorHandler;
/**
* Callback to get project DB
*
* @var callable
*/
protected mixed $getProjectDB;
/**
* Registry
*
* @var Registry
*/
protected Registry $register;
/**
* Latest times for metric that was synced to the database
*
* @var array
*/
private array $latestTime = [];
/**
* Periods the metrics are collected for
* @var array
*/
protected array $periods = [
[
'key' => '1h',
'startTime' => '-24 hours'
],
[
'key' => '1d',
'startTime' => '-30 days'
]
];
/**
* All the metrics that we are collecting
*
* @var array
*/
protected array $metrics = [
'project.$all.network.requests' => [
'table' => 'appwrite_usage_project_{scope}_network_requests',
],
'project.$all.network.bandwidth' => [
'table' => 'appwrite_usage_project_{scope}_network_bandwidth',
],
'project.$all.network.inbound' => [
'table' => 'appwrite_usage_project_{scope}_network_inbound',
],
'project.$all.network.outbound' => [
'table' => 'appwrite_usage_project_{scope}_network_outbound',
],
/* Users service metrics */
'users.$all.requests.create' => [
'table' => 'appwrite_usage_users_{scope}_requests_create',
],
'users.$all.requests.read' => [
'table' => 'appwrite_usage_users_{scope}_requests_read',
],
'users.$all.requests.update' => [
'table' => 'appwrite_usage_users_{scope}_requests_update',
],
'users.$all.requests.delete' => [
'table' => 'appwrite_usage_users_{scope}_requests_delete',
],
'databases.$all.requests.create' => [
'table' => 'appwrite_usage_databases_{scope}_requests_create',
],
'databases.$all.requests.read' => [
'table' => 'appwrite_usage_databases_{scope}_requests_read',
],
'databases.$all.requests.update' => [
'table' => 'appwrite_usage_databases_{scope}_requests_update',
],
'databases.$all.requests.delete' => [
'table' => 'appwrite_usage_databases_{scope}_requests_delete',
],
'collections.$all.requests.create' => [
'table' => 'appwrite_usage_collections_{scope}_requests_create',
],
'collections.$all.requests.read' => [
'table' => 'appwrite_usage_collections_{scope}_requests_read',
],
'collections.$all.requests.update' => [
'table' => 'appwrite_usage_collections_{scope}_requests_update',
],
'collections.$all.requests.delete' => [
'table' => 'appwrite_usage_collections_{scope}_requests_delete',
],
'documents.$all.requests.create' => [
'table' => 'appwrite_usage_documents_{scope}_requests_create',
],
'documents.$all.requests.read' => [
'table' => 'appwrite_usage_documents_{scope}_requests_read',
],
'documents.$all.requests.update' => [
'table' => 'appwrite_usage_documents_{scope}_requests_update',
],
'documents.$all.requests.delete' => [
'table' => 'appwrite_usage_documents_{scope}_requests_delete',
],
'collections.databaseId.requests.create' => [
'table' => 'appwrite_usage_collections_{scope}_requests_create',
'groupBy' => ['databaseId'],
],
'collections.databaseId.requests.read' => [
'table' => 'appwrite_usage_collections_{scope}_requests_read',
'groupBy' => ['databaseId'],
],
'collections.databaseId.requests.update' => [
'table' => 'appwrite_usage_collections_{scope}_requests_update',
'groupBy' => ['databaseId'],
],
'collections.databaseId.requests.delete' => [
'table' => 'appwrite_usage_collections_{scope}_requests_delete',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.create' => [
'table' => 'appwrite_usage_documents_{scope}_requests_create',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.read' => [
'table' => 'appwrite_usage_documents_{scope}_requests_read',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.update' => [
'table' => 'appwrite_usage_documents_{scope}_requests_update',
'groupBy' => ['databaseId'],
],
'documents.databaseId.requests.delete' => [
'table' => 'appwrite_usage_documents_{scope}_requests_delete',
'groupBy' => ['databaseId'],
],
'documents.databaseId/collectionId.requests.create' => [
'table' => 'appwrite_usage_documents_{scope}_requests_create',
'groupBy' => ['databaseId', 'collectionId'],
],
'documents.databaseId/collectionId.requests.read' => [
'table' => 'appwrite_usage_documents_{scope}_requests_read',
'groupBy' => ['databaseId', 'collectionId'],
],
'documents.databaseId/collectionId.requests.update' => [
'table' => 'appwrite_usage_documents_{scope}_requests_update',
'groupBy' => ['databaseId', 'collectionId'],
],
'documents.databaseId/collectionId.requests.delete' => [
'table' => 'appwrite_usage_documents_{scope}_requests_delete',
'groupBy' => ['databaseId', 'collectionId'],
],
'buckets.$all.requests.create' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_create',
],
'buckets.$all.requests.read' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_read',
],
'buckets.$all.requests.update' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_update',
],
'buckets.$all.requests.delete' => [
'table' => 'appwrite_usage_buckets_{scope}_requests_delete',
],
'files.$all.requests.create' => [
'table' => 'appwrite_usage_files_{scope}_requests_create',
],
'files.$all.requests.read' => [
'table' => 'appwrite_usage_files_{scope}_requests_read',
],
'files.$all.requests.update' => [
'table' => 'appwrite_usage_files_{scope}_requests_update',
],
'files.$all.requests.delete' => [
'table' => 'appwrite_usage_files_{scope}_requests_delete',
],
'files.bucketId.requests.create' => [
'table' => 'appwrite_usage_files_{scope}_requests_create',
'groupBy' => ['bucketId'],
],
'files.bucketId.requests.read' => [
'table' => 'appwrite_usage_files_{scope}_requests_read',
'groupBy' => ['bucketId'],
],
'files.bucketId.requests.update' => [
'table' => 'appwrite_usage_files_{scope}_requests_update',
'groupBy' => ['bucketId'],
],
'files.bucketId.requests.delete' => [
'table' => 'appwrite_usage_files_{scope}_requests_delete',
'groupBy' => ['bucketId'],
],
'sessions.$all.requests.create' => [
'table' => 'appwrite_usage_sessions__{scope}_requests_create',
],
'sessions.provider.requests.create' => [
'table' => 'appwrite_usage_sessions_{scope}_requests_create',
'groupBy' => ['provider'],
],
'sessions.$all.requests.delete' => [
'table' => 'appwrite_usage_sessions_{scope}_requests_delete',
],
'executions.$all.compute.total' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
],
'builds.$all.compute.total' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
],
'executions.$all.compute.failure' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'filters' => [
'functionStatus' => 'failed',
],
],
'builds.$all.compute.failure' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'filters' => [
'functionStatus' => 'failed',
],
],
'executions.$all.compute.success' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'filters' => [
'functionStatus' => 'success',
],
],
'builds.$all.compute.success' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'filters' => [
'functionStatus' => 'success',
],
],
'executions.functionId.compute.total' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
],
'builds.functionId.compute.total' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
],
'executions.functionId.compute.failure' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionStatus' => 'failed',
],
],
'builds.functionId.compute.failure' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionBuildStatus' => 'failed',
],
],
'executions.functionId.compute.success' => [
'table' => 'appwrite_usage_executions_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionStatus' => 'success',
],
],
'builds.functionId.compute.success' => [
'table' => 'appwrite_usage_builds_{scope}_compute',
'groupBy' => ['functionId'],
'filters' => [
'functionBuildStatus' => 'success',
],
],
// counters
'users.$all.count.total' => [
'table' => 'appwrite_usage_users_{scope}_count_total',
],
'buckets.$all.count.total' => [
'table' => 'appwrite_usage_buckets_{scope}_count_total',
],
'files.$all.count.total' => [
'table' => 'appwrite_usage_files_{scope}_count_total',
],
'files.bucketId.count.total' => [
'table' => 'appwrite_usage_files_{scope}_count_total',
'groupBy' => ['bucketId']
],
'databases.$all.count.total' => [
'table' => 'appwrite_usage_databases_{scope}_count_total',
],
'collections.$all.count.total' => [
'table' => 'appwrite_usage_collections_{scope}_count_total',
],
'documents.$all.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
],
'collections.databaseId.count.total' => [
'table' => 'appwrite_usage_collections_{scope}_count_total',
'groupBy' => ['databaseId']
],
'documents.databaseId.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
'groupBy' => ['databaseId']
],
'documents.databaseId/collectionId.count.total' => [
'table' => 'appwrite_usage_documents_{scope}_count_total',
'groupBy' => ['databaseId', 'collectionId']
],
'deployments.$all.storage.size' => [
'table' => 'appwrite_usage_deployments_{scope}_storage_size',
],
'project.$all.storage.size' => [
'table' => 'appwrite_usage_project_{scope}_storage_size',
],
'files.$all.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
],
'files.$bucketId.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
'groupBy' => ['bucketId']
],
'builds.$all.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
],
'executions.$all.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
],
'executions.functionId.compute.time' => [
'table' => 'appwrite_usage_executions_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'builds.functionId.compute.time' => [
'table' => 'appwrite_usage_builds_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'project.$all.compute.time' => [ // Built time + execution time
'table' => 'appwrite_usage_project_{scope}_compute_time',
'groupBy' => ['functionId'],
],
'deployments.$all.storage.size' => [
'table' => 'appwrite_usage_deployments_{scope}_storage_size'
],
'project.$all.storage.size' => [
'table' => 'appwrite_usage_project_{scope}_storage_size'
],
'files.$all.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size'
],
'files.bucketId.storage.size' => [
'table' => 'appwrite_usage_files_{scope}_storage_size',
'groupBy' => ['bucketId']
]
];
public function __construct(string $region, Database $database, InfluxDatabase $influxDB, callable $getProjectDB, Registry $register, callable $errorHandler = null)
{
parent::__construct($region);
$this->database = $database;
$this->influxDB = $influxDB;
$this->getProjectDB = $getProjectDB;
$this->register = $register;
$this->errorHandler = $errorHandler;
}
/**
* Create or Update Mertic
* Create or update each metric in the stats collection for the given project
*
* @param string $projectId
* @param int $time
* @param string $period
* @param string $metric
* @param int $value
* @param int $type
*
* @return void
*/
private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void
{
$id = \md5("{$time}_{$period}_{$metric}");
$project = $this->database->getDocument('projects', $projectId);
$database = call_user_func($this->getProjectDB, $project);
try {
$document = $database->getDocument('stats', $id);
if ($document->isEmpty()) {
$database->createDocument('stats', new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $metric,
'value' => $value,
'type' => $type,
'region' => $this->region,
]));
} else {
$database->updateDocument(
'stats',
$document->getId(),
$document->setAttribute('value', $value)
);
}
} catch (\Exception $e) { // if projects are deleted this might fail
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}");
} else {
throw $e;
}
}
$this->register->get('pools')->reclaim();
}
/**
* Sync From InfluxDB
* Sync stats from influxDB to stats collection in the Appwrite database
*
* @param string $metric
* @param array $options
* @param array $period
*
* @return void
*/
private function syncFromInfluxDB(string $metric, array $options, array $period): void
{
$start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339);
if (!empty($this->latestTime[$metric][$period['key']])) {
$start = $this->latestTime[$metric][$period['key']];
}
$end = (new DateTime())->format(DateTime::RFC3339);
$table = $options['table']; //Which influxdb table to query for this metric
$groupBy = empty($options['groupBy']) ? '' : ', ' . implode(', ', array_map(fn($groupBy) => '"' . $groupBy . '" ', $options['groupBy'])); //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc
$filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status
if (!empty($filters)) {
$filters = ' AND ' . implode(' AND ', array_map(fn ($filter, $value) => "\"{$filter}\"='{$value}'", array_keys($filters), array_values($filters)));
} else {
$filters = '';
}
$query = "SELECT sum(value) AS \"value\" ";
$query .= "FROM \"{$table}\" ";
$query .= "WHERE \"time\" > '{$start}' ";
$query .= "AND \"time\" < '{$end}' ";
$query .= "AND \"metric_type\"='counter' {$filters} ";
$query .= "GROUP BY time({$period['key']}), \"projectId\" {$groupBy} ";
$query .= "FILL(null)";
try {
$result = $this->influxDB->query($query);
$points = $result->getPoints();
foreach ($points as $point) {
$projectId = $point['projectId'];
if (!empty($projectId) && $projectId !== 'console') {
$metricUpdated = $metric;
if (!empty($groupBy)) {
foreach ($options['groupBy'] as $groupBy) {
$groupedBy = $point[$groupBy] ?? '';
if (empty($groupedBy)) {
continue;
}
$metricUpdated = str_replace($groupBy, $groupedBy, $metricUpdated);
}
}
$value = (!empty($point['value'])) ? $point['value'] : 0;
$this->createOrUpdateMetric(
$point['projectId'],
$point['time'],
$period['key'],
$metricUpdated,
$value,
0
);
$this->latestTime[$metric][$period['key']] = $point['time'];
}
}
} catch (\Exception $e) { // if projects are deleted this might fail
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e, "sync_metric_{$metric}_influxdb");
} else {
throw $e;
}
}
}
/**
* Collect Stats
* Collect all the stats from Influd DB to Database
*
* @return void
*/
public function collect(): void
{
foreach ($this->periods as $period) {
foreach ($this->metrics as $metric => $options) { //for each metrics
try {
$this->syncFromInfluxDB($metric, $options, $period);
} catch (\Exception $e) {
if (is_callable($this->errorHandler)) {
call_user_func($this->errorHandler, $e);
} else {
throw $e;
}
}
}
}
}
}
src/Appwrite/Usage/Stats.php
+225
View File
@@ -0,0 +1,225 @@
<?php
namespace Appwrite\Usage;
use Utopia\App;
class Stats
{
/**
* @var array
*/
protected $params = [];
/**
* @var mixed
*/
protected $statsd;
/**
* @var string
*/
protected $namespace = 'appwrite.usage';
/**
* Event constructor.
*
* @param mixed $statsd
*/
public function __construct($statsd)
{
$this->statsd = $statsd;
}
/**
* @param string $key
* @param mixed $value
*
* @return $this
*/
public function setParam(string $key, $value): self
{
$this->params[$key] = $value;
return $this;
}
/**
* @param string $key
*
* @return mixed|null
*/
public function getParam(string $key)
{
return (isset($this->params[$key])) ? $this->params[$key] : null;
}
/**
* @param string $namespace
*
* @return $this
*/
public function setNamespace(string $namespace): self
{
$this->namespace = $namespace;
return $this;
}
/**
* @return string
*/
public function getNamespace()
{
return $this->namespace;
}
/**
* Submit data to StatsD.
* Send various metrics to StatsD based on the parameters that are set
* @return void
*/
public function submit(): void
{
$projectId = $this->params['projectId'] ?? '';
$projectInternalId = $this->params['projectInternalId'];
$tags = ",projectInternalId={$projectInternalId},projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN');
// the global namespace is prepended to every key (optional)
$this->statsd->setNamespace($this->namespace);
$httpRequest = $this->params['project.{scope}.network.requests'] ?? 0;
$httpMethod = $this->params['httpMethod'] ?? '';
if ($httpRequest >= 1) {
$this->statsd->increment('project.{scope}.network.requests' . $tags . ',method=' . \strtolower($httpMethod));
}
$inbound = $this->params['project.{scope}.network.inbound'] ?? 0;
$outbound = $this->params['project.{scope}.network.outbound'] ?? 0;
$this->statsd->count('project.{scope}.network.inbound' . $tags, $inbound);
$this->statsd->count('project.{scope}.network.outbound' . $tags, $outbound);
$this->statsd->count('project.{scope}.network.bandwidth' . $tags, $inbound + $outbound);
$usersMetrics = [
'users.{scope}.requests.create',
'users.{scope}.requests.read',
'users.{scope}.requests.update',
'users.{scope}.requests.delete',
'users.{scope}.count.total',
];
foreach ($usersMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value === 1 || $value === -1) {
$this->statsd->count($metric . $tags, $value);
}
}
$dbMetrics = [
'databases.{scope}.requests.create',
'databases.{scope}.requests.read',
'databases.{scope}.requests.update',
'databases.{scope}.requests.delete',
'collections.{scope}.requests.create',
'collections.{scope}.requests.read',
'collections.{scope}.requests.update',
'collections.{scope}.requests.delete',
'documents.{scope}.requests.create',
'documents.{scope}.requests.read',
'documents.{scope}.requests.update',
'documents.{scope}.requests.delete',
'databases.{scope}.count.total',
'collections.{scope}.count.total',
'documents.{scope}.count.total'
];
foreach ($dbMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value === 1 || $value === -1) {
$dbTags = $tags . ",collectionId=" . ($this->params['collectionId'] ?? '') . ",databaseId=" . ($this->params['databaseId'] ?? '');
$this->statsd->count($metric . $dbTags, $value);
}
}
$storageMertics = [
'buckets.{scope}.requests.create',
'buckets.{scope}.requests.read',
'buckets.{scope}.requests.update',
'buckets.{scope}.requests.delete',
'files.{scope}.requests.create',
'files.{scope}.requests.read',
'files.{scope}.requests.update',
'files.{scope}.requests.delete',
'buckets.{scope}.count.total',
'files.{scope}.count.total',
'files.{scope}.storage.size'
];
foreach ($storageMertics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value !== 0) {
$storageTags = $tags . ",bucketId=" . ($this->params['bucketId'] ?? '');
$this->statsd->count($metric . $storageTags, $value);
}
}
$sessionsMetrics = [
'sessions.{scope}.requests.create',
'sessions.{scope}.requests.update',
'sessions.{scope}.requests.delete',
];
foreach ($sessionsMetrics as $metric) {
$value = $this->params[$metric] ?? 0;
if ($value >= 1) {
$sessionTags = $tags . ",provider=" . ($this->params['provider'] ?? '');
$this->statsd->count($metric . $sessionTags, $value);
}
}
$functionId = $this->params['functionId'] ?? '';
$functionExecution = $this->params['executions.{scope}.compute'] ?? 0;
$functionExecutionTime = ($this->params['executionTime'] ?? 0) * 1000; // ms
$functionExecutionStatus = $this->params['executionStatus'] ?? '';
$functionBuild = $this->params['builds.{scope}.compute'] ?? 0;
$functionBuildTime = ($this->params['buildTime'] ?? 0) * 1000; // ms
$functionBuildStatus = $this->params['buildStatus'] ?? '';
$functionCompute = $functionExecutionTime + $functionBuildTime;
$functionTags = $tags . ',functionId=' . $functionId;
$deploymentSize = $this->params['deployment.{scope}.storage.size'] ?? 0;
$storageSize = $this->params['files.{scope}.storage.size'] ?? 0;
if ($deploymentSize + $storageSize > 0 || $deploymentSize + $storageSize <= -1) {
$this->statsd->count('project.{scope}.storage.size' . $tags, $deploymentSize + $storageSize);
}
if ($deploymentSize !== 0) {
$this->statsd->count('deployments.{scope}.storage.size' . $functionTags, $deploymentSize);
}
if ($functionExecution >= 1) {
$this->statsd->increment('executions.{scope}.compute' . $functionTags . ',functionStatus=' . $functionExecutionStatus);
if ($functionExecutionTime > 0) {
$this->statsd->count('executions.{scope}.compute.time' . $functionTags, $functionExecutionTime);
}
}
if ($functionBuild >= 1) {
$this->statsd->increment('builds.{scope}.compute' . $functionTags . ',functionBuildStatus=' . $functionBuildStatus);
$this->statsd->count('builds.{scope}.compute.time' . $functionTags, $functionBuildTime);
}
if ($functionBuild + $functionExecution >= 1) {
$this->statsd->count('project.{scope}.compute.time' . $functionTags, $functionCompute);
}
$this->reset();
}
public function reset(): self
{
$this->params = [];
$this->namespace = 'appwrite.usage';
return $this;
}
}
src/Appwrite/Utopia/Response/Model/UsageBuckets.php
+29 -1
View File
@@ -16,7 +16,7 @@ class UsageBuckets extends Model
'default' => '',
'example' => '30d',
])
->addRule('filesTotal', [
->addRule('filesCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of files in this bucket.',
'default' => [],
@@ -30,6 +30,34 @@ class UsageBuckets extends Model
'example' => [],
'array' => true
])
->addRule('filesCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files deleted.',
'default' => [],
'example' => [],
'array' => true
])
;
}
src/Appwrite/Utopia/Response/Model/UsageCollection.php
+29 -1
View File
@@ -16,13 +16,41 @@ class UsageCollection extends Model
'default' => '',
'example' => '30d',
])
->addRule('documentsTotal', [
->addRule('documentsCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => [],
'array' => true
])
;
}
src/Appwrite/Utopia/Response/Model/UsageDatabase.php
+59 -3
View File
@@ -16,16 +16,72 @@ class UsageDatabase extends Model
'default' => '',
'example' => '30d',
])
->addRule('collectionsTotal', [
->addRule('documentsCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsTotal', [
->addRule('documentsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections delete.',
'default' => [],
'example' => [],
'array' => true
src/Appwrite/Utopia/Response/Model/UsageDatabases.php
+88 -4
View File
@@ -16,23 +16,107 @@ class UsageDatabases extends Model
'default' => '',
'example' => '30d',
])
->addRule('databasesTotal', [
->addRule('databasesCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsTotal', [
->addRule('documentsCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsTotal', [
->addRule('databasesCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of documents.',
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of collections.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for documents deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('collectionsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for collections delete.',
'default' => [],
'example' => [],
'array' => true
src/Appwrite/Utopia/Response/Model/UsageFunction.php
+28 -22
View File
@@ -16,16 +16,30 @@ class UsageFunction extends Model
'default' => '',
'example' => '30d',
])
->addRule('deploymentsTotal', [
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function deployments.',
'description' => 'Aggregated stats for number of function executions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('deploymentsStorage', [
->addRule('executionsFailure', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function deployments storage.',
'description' => 'Aggregated stats for function execution failures.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsSuccess', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution successes.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution duration.',
'default' => [],
'example' => [],
'array' => true
@@ -37,31 +51,23 @@ class UsageFunction extends Model
'example' => [],
'array' => true
])
->addRule('buildsStorage', [
->addRule('buildsFailure', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for builds storage.',
'description' => 'Aggregated stats for function build failures.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsSuccess', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build successes.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build compute.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution compute.',
'description' => 'Aggregated stats for function build duration.',
'default' => [],
'example' => [],
'array' => true
src/Appwrite/Utopia/Response/Model/UsageFunctions.php
+23 -24
View File
@@ -16,23 +16,30 @@ class UsageFunctions extends Model
'default' => '',
'example' => '30d',
])
->addRule('functionsTotal', [
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of functions.',
'description' => 'Aggregated stats for number of function executions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('deploymentsTotal', [
->addRule('executionsFailure', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function deployments.',
'description' => 'Aggregated stats for function execution failures.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('deploymentsStorage', [
->addRule('executionsSuccess', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function deployments storage.',
'description' => 'Aggregated stats for function execution successes.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution duration.',
'default' => [],
'example' => [],
'array' => true
@@ -44,31 +51,23 @@ class UsageFunctions extends Model
'example' => [],
'array' => true
])
->addRule('buildsStorage', [
->addRule('buildsFailure', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for builds storage.',
'description' => 'Aggregated stats for function build failures.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsSuccess', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build successes.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('buildsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function build compute.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTotal', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of function executions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('executionsTime', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function execution compute.',
'description' => 'Aggregated stats for function build duration.',
'default' => [],
'example' => [],
'array' => true
src/Appwrite/Utopia/Response/Model/UsageProject.php
+7 -7
View File
@@ -16,7 +16,7 @@ class UsageProject extends Model
'default' => '',
'example' => '30d',
])
->addRule('requestsTotal', [
->addRule('requests', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of requests.',
'default' => [],
@@ -30,42 +30,42 @@ class UsageProject extends Model
'example' => [],
'array' => true
])
->addRule('executionsTotal', [
->addRule('executions', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for function executions.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('documentsTotal', [
->addRule('documents', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of documents.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('databasesTotal', [
->addRule('databases', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of databases.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersTotal', [
->addRule('users', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of users.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesStorage', [
->addRule('storage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsTotal', [
->addRule('buckets', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for number of buckets.',
'default' => [],
src/Appwrite/Utopia/Response/Model/UsageStorage.php
+61 -5
View File
@@ -16,23 +16,79 @@ class UsageStorage extends Model
'default' => '',
'example' => '30d',
])
->addRule('bucketsTotal', [
->addRule('storage', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of buckets.',
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesTotal', [
->addRule('filesCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of files.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesStorage', [
->addRule('bucketsCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for the occupied storage size (in bytes).',
'description' => 'Aggregated stats for total number of buckets.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('bucketsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for buckets deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('filesDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for files deleted.',
'default' => [],
'example' => [],
'array' => true
src/Appwrite/Utopia/Response/Model/UsageUsers.php
+44 -3
View File
@@ -16,21 +16,62 @@ class UsageUsers extends Model
'default' => '',
'example' => '30d',
])
->addRule('usersTotal', [
->addRule('usersCount', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for total number of users.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsTotal', [
->addRule('usersCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersRead', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users read.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersUpdate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users updated.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('usersDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for users deleted.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for sessions created.',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsProviderCreate', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for sessions created for a provider ( email, anonymous or oauth2 ).',
'default' => [],
'example' => [],
'array' => true
])
->addRule('sessionsDelete', [
'type' => Response::MODEL_METRIC,
'description' => 'Aggregated stats for sessions deleted.',
'default' => [],
'example' => [],
'array' => true
])
;
}
src/Appwrite/Vcs/Comment.php
+22 -7
View File
@@ -7,6 +7,12 @@ use Utopia\App;
class Comment
{
protected array $tips = [
'Appwrite has a Discord community with over 16 000 members. [Come join us!](https://appwrite.io/discord)',
'You can use [Avatars API](https://appwrite.io/docs/client/avatars?sdk=web-default#avatarsGetQR) to generate QR code for any text or URLs',
'[Cursor pagination](https://appwrite.io/docs/pagination#cursor-pagination) performs better than offset pagination when loading further pages',
];
protected string $statePrefix = '[appwrite]: #';
/**
@@ -60,7 +66,7 @@ class Comment
}
//TODO: Update link to documentation
$text .= "**Your function has automatically been deployed.** Learn more about Appwrite Function Deployments in our [documentation](https://appwrite.io/docs/functions).\n\n";
$text .= "**Your function has been automatically deployed.** Learn more about Appwrite [Function Deployments](https://appwrite.io/docs/functions).\n\n";
foreach ($projects as $projectId => $project) {
$text .= "**{$project['name']}** `{$projectId}`\n\n";
@@ -71,12 +77,20 @@ class Comment
$hostname = App::getEnv('_APP_DOMAIN');
foreach ($project['functions'] as $functionId => $function) {
$generateImage = function (string $status) use ($protocol, $hostname) {
$extention = $status === 'building' ? 'gif' : 'png';
$imagesUrl = $protocol . '://' . $hostname . '/images/vcs/';
$imageUrl = '<picture><source media="(prefers-color-scheme: dark)" srcset="' . $imagesUrl . 'status-' . $status . '-dark.' . $extention . '"><img alt="' . $status . '" height="25" align="center" src="' . $imagesUrl . 'status-' . $status . '-light.' . $extention . '"></picture>';
return $imageUrl;
};
$status = match ($function['status']) {
'waiting' => '<img src="' . $protocol . '://' . $hostname . '/images/vcs/status-waiting.png" alt="Waiting" height="25" align="center"> Waiting to build',
'processing' => '<img src="' . $protocol . '://' . $hostname . '/images/vcs/status-building.gif" alt="Processing" height="29" align="center"> Processing',
'building' => '<img src="' . $protocol . '://' . $hostname . '/images/vcs/status-building.gif" alt="Building" height="29" align="center"> Building',
'ready' => '<img src="' . $protocol . '://' . $hostname . '/images/vcs/status-success.png" alt="Ready" height="25" align="center"> Ready',
'failed' => '<img src="' . $protocol . '://' . $hostname . '/images/vcs/status-failed.png" alt="Failed" height="25" align="center"> Failed',
'waiting' => $generateImage('waiting') . ' Waiting to build',
'processing' => $generateImage('processing') . ' Processing',
'building' => $generateImage('building') . ' Building',
'ready' => $generateImage('ready') . ' Ready',
'failed' => $generateImage('failed') . ' Failed',
};
if ($function['action']['type'] === 'logs') {
@@ -91,7 +105,8 @@ class Comment
$text .= "\n\n";
}
//TODO: Update did you know section
$text .= "> **💡 Did you know?** \n Appwrite has a Discord community with over 16 000 members. [Come join us!](https://appwrite.io/discord)\n\n";
$tip = $this->tips[array_rand($this->tips)];
$text .= "> **💡 Did you know?** \n " . $tip . "\n\n";
return $text;
}
tests/e2e/General/UsageTest.php
+787 -730
View File
File diff suppressed because it is too large Load Diff
tests/e2e/Services/Databases/DatabasesBase.php
+62 -9
View File
@@ -344,6 +344,63 @@ trait DatabasesBase
$this->assertEquals(400, $response['headers']['status-code']);
}
public function testUpdateAttributeEnum(): void
{
$database = $this->client->call(Client::METHOD_POST, '/databases', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
], [
'databaseId' => ID::unique(),
'name' => 'Test Database 2'
]);
$players = $this->client->call(Client::METHOD_POST, '/databases/' . $database['body']['$id'] . '/collections', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'collectionId' => ID::unique(),
'name' => 'Players',
'documentSecurity' => true,
'permissions' => [
Permission::create(Role::user($this->getUser()['$id'])),
],
]);
// Create enum attribute
$attribute = $this->client->call(Client::METHOD_POST, '/databases/' . $database['body']['$id'] . '/collections/' . $players['body']['$id'] . '/attributes/enum', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]), [
'key' => 'position',
'elements' => ['goalkeeper', 'defender', 'midfielder', 'forward'],
'required' => true,
'array' => false,
]);
$this->assertEquals(202, $attribute['headers']['status-code']);
$this->assertEquals($attribute['body']['key'], 'position');
$this->assertEquals($attribute['body']['elements'], ['goalkeeper', 'defender', 'midfielder', 'forward']);
\sleep(2);
// Update enum attribute
$attribute = $this->client->call(Client::METHOD_PATCH, '/databases/' . $database['body']['$id'] . '/collections/' . $players['body']['$id'] . '/attributes/enum/' . $attribute['body']['key'], array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]), [
'elements' => ['goalkeeper', 'defender', 'midfielder', 'forward', 'coach'],
'required' => true,
'default' => null
]);
$this->assertEquals(200, $attribute['headers']['status-code']);
$this->assertEquals($attribute['body']['elements'], ['goalkeeper', 'defender', 'midfielder', 'forward', 'coach']);
}
/**
* @depends testCreateAttributes
*/
@@ -2862,8 +2919,6 @@ trait DatabasesBase
$collectionId = $collection['body']['$id'];
sleep(2);
$attribute = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
@@ -2877,8 +2932,7 @@ trait DatabasesBase
$this->assertEquals(202, $attribute['headers']['status-code'], 202);
$this->assertEquals('attribute', $attribute['body']['key']);
// wait for db to add attribute
sleep(2);
\sleep(2);
$index = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/indexes', array_merge([
'content-type' => 'application/json',
@@ -2893,8 +2947,7 @@ trait DatabasesBase
$this->assertEquals(202, $index['headers']['status-code']);
$this->assertEquals('key_attribute', $index['body']['key']);
// wait for db to add attribute
sleep(2);
\sleep(2);
$document1 = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', array_merge([
'content-type' => 'application/json',
@@ -2993,7 +3046,7 @@ trait DatabasesBase
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session2,
]);
// Current user has no collection permissions and document permissions are disabled
// other2 has no collection permissions and document permissions are disabled
$this->assertEquals(404, $document3GetWithDocumentRead['headers']['status-code']);
$documentsUser2 = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', [
@@ -3003,8 +3056,8 @@ trait DatabasesBase
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session2,
]);
// Current user has no collection permissions and document permissions are disabled
$this->assertEquals(404, $documentsUser2['headers']['status-code']);
// other2 has no collection permissions and document permissions are disabled
$this->assertEquals(401, $documentsUser2['headers']['status-code']);
// Enable document permissions
$collection = $this->client->call(CLient::METHOD_PUT, '/databases/' . $databaseId . '/collections/' . $collectionId, [
tests/e2e/Services/Databases/DatabasesConsoleClientTest.php
+31 -46
View File
@@ -29,6 +29,7 @@ class DatabasesConsoleClientTest extends Scope
$this->assertTrue($database['body']['enabled']);
$databaseId = $database['body']['$id'];
/**
* Test for SUCCESS
*/
@@ -51,7 +52,7 @@ class DatabasesConsoleClientTest extends Scope
$this->assertEquals($movies['body']['name'], 'Movies');
/**
* Test When database is disabled but can still create collections
* Test when database is disabled but can still create collections
*/
$database = $this->client->call(Client::METHOD_PUT, '/databases/' . $databaseId, array_merge([
'content-type' => 'application/json',
@@ -78,6 +79,17 @@ class DatabasesConsoleClientTest extends Scope
'documentSecurity' => true,
]);
/**
* Test when collection is disabled but can still modify collections
*/
$database = $this->client->call(Client::METHOD_PUT, '/databases/' . $databaseId . '/collections/' . $movies['body']['$id'], array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'name' => 'Movies',
'enabled' => false,
]);
$this->assertEquals(201, $tvShows['headers']['status-code']);
$this->assertEquals($tvShows['body']['name'], 'TvShows');
@@ -87,11 +99,12 @@ class DatabasesConsoleClientTest extends Scope
/**
* @depends testCreateCollection
* @param array $data
* @throws \Exception
*/
public function testListCollection(array $data)
{
/**
* Test When database is disabled but can still call list collections
* Test when database is disabled but can still call list collections
*/
$databaseId = $data['databaseId'];
@@ -108,6 +121,7 @@ class DatabasesConsoleClientTest extends Scope
/**
* @depends testCreateCollection
* @param array $data
* @throws \Exception
*/
public function testGetCollection(array $data)
{
@@ -115,7 +129,7 @@ class DatabasesConsoleClientTest extends Scope
$moviesCollectionId = $data['moviesId'];
/**
* Test When database is disabled but can still call get collection
* Test when database and collection are disabled but can still call get collection
*/
$collection = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $moviesCollectionId, array_merge([
'content-type' => 'application/json',
@@ -125,12 +139,14 @@ class DatabasesConsoleClientTest extends Scope
$this->assertEquals(200, $collection['headers']['status-code']);
$this->assertEquals('Movies', $collection['body']['name']);
$this->assertEquals($moviesCollectionId, $collection['body']['$id']);
$this->assertTrue($collection['body']['enabled']);
$this->assertFalse($collection['body']['enabled']);
}
/**
* @depends testCreateCollection
* @param array $data
* @throws \Exception
* @throws \Exception
*/
public function testUpdateCollection(array $data)
{
@@ -138,7 +154,7 @@ class DatabasesConsoleClientTest extends Scope
$moviesCollectionId = $data['moviesId'];
/**
* Test When database is disabled but can still call update collection
* Test When database and collection are disabled but can still call update collection
*/
$collection = $this->client->call(Client::METHOD_PUT, '/databases/' . $databaseId . '/collections/' . $moviesCollectionId, array_merge([
'content-type' => 'application/json',
@@ -157,6 +173,8 @@ class DatabasesConsoleClientTest extends Scope
/**
* @depends testCreateCollection
* @param array $data
* @throws \Exception
* @throws \Exception
*/
public function testDeleteCollection(array $data)
{
@@ -164,7 +182,7 @@ class DatabasesConsoleClientTest extends Scope
$tvShowsId = $data['tvShowsId'];
/**
* Test When database is disabled but can still call Delete collection
* Test when database and collection are disabled but can still call delete collection
*/
$response = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $tvShowsId, array_merge([
'content-type' => 'application/json',
@@ -175,44 +193,6 @@ class DatabasesConsoleClientTest extends Scope
$this->assertEquals($response['body'], "");
}
/**
* @depends testCreateCollection
*/
public function testGetDatabaseUsage(array $data)
{
$databaseId = $data['databaseId'];
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '32h'
]);
$this->assertEquals(400, $response['headers']['status-code']);
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(count($response['body']), 3);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['documentsTotal']);
$this->assertIsArray($response['body']['collectionsTotal']);
}
/**
* @depends testCreateCollection
*/
@@ -250,10 +230,15 @@ class DatabasesConsoleClientTest extends Scope
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(count($response['body']), 2);
$this->assertEquals(count($response['body']), 6);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['documentsTotal']);
$this->assertIsArray($response['body']['documentsCount']);
$this->assertIsArray($response['body']['documentsCreate']);
$this->assertIsArray($response['body']['documentsRead']);
$this->assertIsArray($response['body']['documentsUpdate']);
$this->assertIsArray($response['body']['documentsDelete']);
}
/**
tests/e2e/Services/Databases/DatabasesPermissionsTeamTest.php
+1 -1
View File
@@ -176,7 +176,7 @@ class DatabasesPermissionsTeamTest extends Scope
if ($success) {
$this->assertCount(1, $documents['body']['documents']);
} else {
$this->assertEquals(404, $documents['headers']['status-code']);
$this->assertEquals(401, $documents['headers']['status-code']);
}
}
tests/e2e/Services/Functions/FunctionsConsoleClientTest.php
+8 -7
View File
@@ -68,7 +68,7 @@ class FunctionsConsoleClientTest extends Scope
/**
* @depends testCreateFunction
*/
public function testGetFunctionUsage(array $data)
public function testGetCollectionUsage(array $data)
{
/**
* Test for FAILURE
@@ -104,15 +104,16 @@ class FunctionsConsoleClientTest extends Scope
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 8);
$this->assertEquals(count($response['body']), 9);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['deploymentsTotal']);
$this->assertIsArray($response['body']['deploymentsStorage']);
$this->assertIsArray($response['body']['buildsTotal']);
$this->assertIsArray($response['body']['buildsStorage']);
$this->assertIsArray($response['body']['buildsTime']);
$this->assertIsArray($response['body']['executionsTotal']);
$this->assertIsArray($response['body']['executionsFailure']);
$this->assertIsArray($response['body']['executionsSuccess']);
$this->assertIsArray($response['body']['executionsTime']);
$this->assertIsArray($response['body']['buildsTotal']);
$this->assertIsArray($response['body']['buildsFailure']);
$this->assertIsArray($response['body']['buildsSuccess']);
$this->assertIsArray($response['body']['buildsTime']);
}
/**
tests/e2e/Services/Projects/ProjectsConsoleClientTest.php
+8 -8
View File
@@ -446,7 +446,7 @@ class ProjectsConsoleClientTest extends Scope
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/project/usage', array_merge([
$response = $this->client->call(Client::METHOD_GET, '/projects/' . $id . '/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()));
@@ -455,14 +455,14 @@ class ProjectsConsoleClientTest extends Scope
$this->assertEquals(count($response['body']), 9);
$this->assertNotEmpty($response['body']);
$this->assertEquals('30d', $response['body']['range']);
$this->assertIsArray($response['body']['requestsTotal']);
$this->assertIsArray($response['body']['requests']);
$this->assertIsArray($response['body']['network']);
$this->assertIsArray($response['body']['executionsTotal']);
$this->assertIsArray($response['body']['documentsTotal']);
$this->assertIsArray($response['body']['databasesTotal']);
$this->assertIsArray($response['body']['bucketsTotal']);
$this->assertIsArray($response['body']['usersTotal']);
$this->assertIsArray($response['body']['filesStorage']);
$this->assertIsArray($response['body']['executions']);
$this->assertIsArray($response['body']['documents']);
$this->assertIsArray($response['body']['databases']);
$this->assertIsArray($response['body']['buckets']);
$this->assertIsArray($response['body']['users']);
$this->assertIsArray($response['body']['storage']);
/**
* Test for FAILURE
tests/e2e/Services/Storage/StorageConsoleClientTest.php
+9 -6
View File
@@ -39,11 +39,10 @@ class StorageConsoleClientTest extends Scope
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(4, count($response['body']));
$this->assertEquals(12, count($response['body']));
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['bucketsTotal']);
$this->assertIsArray($response['body']['filesTotal']);
$this->assertIsArray($response['body']['filesStorage']);
$this->assertIsArray($response['body']['storage']);
$this->assertIsArray($response['body']['filesCount']);
}
public function testGetStorageBucketUsage()
@@ -95,9 +94,13 @@ class StorageConsoleClientTest extends Scope
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 3);
$this->assertEquals(count($response['body']), 7);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['filesTotal']);
$this->assertIsArray($response['body']['filesCount']);
$this->assertIsArray($response['body']['filesCreate']);
$this->assertIsArray($response['body']['filesRead']);
$this->assertIsArray($response['body']['filesUpdate']);
$this->assertIsArray($response['body']['filesDelete']);
$this->assertIsArray($response['body']['filesStorage']);
}
}
tests/e2e/Services/Users/UsersConsoleClientTest.php
+40 -3
View File
@@ -23,6 +23,17 @@ class UsersConsoleClientTest extends Scope
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '32h',
'provider' => 'email'
]);
$this->assertEquals($response['headers']['status-code'], 400);
$response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h',
'provider' => 'some-random-provider'
]);
$this->assertEquals($response['headers']['status-code'], 400);
@@ -35,12 +46,38 @@ class UsersConsoleClientTest extends Scope
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h',
'provider' => 'email'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 3);
$this->assertEquals(count($response['body']), 9);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['usersTotal']);
$this->assertIsArray($response['body']['sessionsTotal']);
$this->assertIsArray($response['body']['usersCount']);
$this->assertIsArray($response['body']['usersCreate']);
$this->assertIsArray($response['body']['usersRead']);
$this->assertIsArray($response['body']['usersUpdate']);
$this->assertIsArray($response['body']['usersDelete']);
$this->assertIsArray($response['body']['sessionsCreate']);
$this->assertIsArray($response['body']['sessionsProviderCreate']);
$this->assertIsArray($response['body']['sessionsDelete']);
$response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()), [
'range' => '24h'
]);
$this->assertEquals($response['headers']['status-code'], 200);
$this->assertEquals(count($response['body']), 9);
$this->assertEquals($response['body']['range'], '24h');
$this->assertIsArray($response['body']['usersCount']);
$this->assertIsArray($response['body']['usersCreate']);
$this->assertIsArray($response['body']['usersRead']);
$this->assertIsArray($response['body']['usersUpdate']);
$this->assertIsArray($response['body']['usersDelete']);
$this->assertIsArray($response['body']['sessionsCreate']);
$this->assertIsArray($response['body']['sessionsProviderCreate']);
$this->assertIsArray($response['body']['sessionsDelete']);
}
}
tests/unit/Usage/StatsTest.php
+48 -31
View File
@@ -2,53 +2,70 @@
namespace Tests\Unit\Usage;
use Appwrite\URL\URL as AppwriteURL;
use Appwrite\Usage\Stats;
use PHPUnit\Framework\TestCase;
use Utopia\App;
use Utopia\DSN\DSN;
use Utopia\Queue;
use Utopia\Queue\Client;
use Utopia\Queue\Connection;
class StatsTest extends TestCase
{
protected ?Connection $connection = null;
protected ?Client $client = null;
protected const QUEUE_NAME = 'usage-test-q';
/**
* @var Stats
*/
protected $object = null;
public function setUp(): void
{
$env = App::getEnv('_APP_CONNECTIONS_QUEUE', AppwriteURL::unparse([
'scheme' => 'redis',
'host' => App::getEnv('_APP_REDIS_HOST', 'redis'),
'port' => App::getEnv('_APP_REDIS_PORT', '6379'),
'user' => App::getEnv('_APP_REDIS_USER', ''),
'pass' => App::getEnv('_APP_REDIS_PASS', ''),
]));
$host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
$port = App::getEnv('_APP_STATSD_PORT', 8125);
$dsn = explode('=', $env);
$dsn = $dsn[1] ?? '';
$dsn = new DSN($dsn);
$this->connection = new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort());
$this->client = new Client(self::QUEUE_NAME, $this->connection);
$connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port);
$statsd = new \Domnikl\Statsd\Client($connection);
$this->object = new Stats($statsd);
}
public function tearDown(): void
{
}
public function testSamePayload(): void
public function testNamespace(): void
{
$inToQueue = [
'key_1' => 'value_1',
'key_2' => 'value_2',
];
$this->object->setNamespace('appwritetest.usage');
$this->assertEquals('appwritetest.usage', $this->object->getNamespace());
}
$result = $this->client->enqueue($inToQueue);
$this->assertTrue($result);
$outFromQueue = $this->connection->leftPopArray('utopia-queue.queue.' . self::QUEUE_NAME, 0)['payload'];
$this->assertNotEmpty($outFromQueue);
$this->assertSame($inToQueue, $outFromQueue);
public function testParams(): void
{
$this->object
->setParam('projectId', 'appwrite_test')
->setParam('projectInternalId', 1)
->setParam('networkRequestSize', 100)
;
$this->assertEquals('appwrite_test', $this->object->getParam('projectId'));
$this->assertEquals(1, $this->object->getParam('projectInternalId'));
$this->assertEquals(100, $this->object->getParam('networkRequestSize'));
$this->object->submit();
$this->assertEquals(null, $this->object->getParam('projectId'));
$this->assertEquals(null, $this->object->getParam('networkRequestSize'));
}
public function testReset(): void
{
$this->object
->setParam('projectId', 'appwrite_test')
->setParam('networkRequestSize', 100)
;
$this->assertEquals('appwrite_test', $this->object->getParam('projectId'));
$this->assertEquals(100, $this->object->getParam('networkRequestSize'));
$this->object->reset();
$this->assertEquals(null, $this->object->getParam('projectId'));
$this->assertEquals(null, $this->object->getParam('networkRequestSize'));
$this->assertEquals('appwrite.usage', $this->object->getNamespace());
}
}