Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '1.6.x' into pla-2428

Browse Source
This commit is contained in:
Chirag Aggarwal
2025-02-14 09:04:30 +05:30
committed by GitHub
parent 20c01d0e73 7143bef111
commit 9ebb4b9626
71 changed files with 4531 additions and 2311 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+2
View File
@@ -87,6 +87,7 @@ _APP_MAINTENANCE_RETENTION_EXECUTION=1209600
_APP_MAINTENANCE_RETENTION_ABUSE=86400
_APP_MAINTENANCE_RETENTION_AUDIT=1209600
_APP_USAGE_AGGREGATION_INTERVAL=30
_APP_STATS_RESOURCES_INTERVAL=3600
_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
_APP_MAINTENANCE_RETENTION_SCHEDULES=86400
_APP_USAGE_STATS=enabled
@@ -111,3 +112,4 @@ _APP_MESSAGE_PUSH_TEST_DSN=
_APP_WEBHOOK_MAX_FAILED_ATTEMPTS=10
_APP_PROJECT_REGIONS=default
_APP_FUNCTIONS_CREATION_ABUSE_LIMIT=5000
_APP_STATS_USAGE_DUAL_WRITING_DBS=database_db_main
.github/workflows/tests.yml
+83 -8
View File
@@ -8,9 +8,33 @@ env:
IMAGE: appwrite-dev
CACHE_KEY: appwrite-dev-${{ github.event.pull_request.head.sha }}
on: [pull_request]
on: [ pull_request ]
jobs:
check_database_changes:
name: Check if utopia-php/database changed
runs-on: ubuntu-latest
outputs:
database_changed: ${{ steps.check.outputs.database_changed }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Fetch base branch
run: git fetch origin ${{ github.event.pull_request.base.ref }}
- name: Check for utopia-php/database changes
id: check
run: |
if git diff origin/${{ github.event.pull_request.base.ref }} HEAD -- composer.lock | grep -q '"name": "utopia-php/database"'; then
echo "Database version changed, going to run all mode tests."
echo "database_changed=true" >> "$GITHUB_ENV"
echo "database_changed=true" >> "$GITHUB_OUTPUT"
else
echo "database_changed=false" >> "$GITHUB_ENV"
echo "database_changed=false" >> "$GITHUB_OUTPUT"
fi
setup:
name: Setup & Build Appwrite Image
runs-on: ubuntu-latest
@@ -103,6 +127,62 @@ jobs:
name: E2E Service Test
runs-on: ubuntu-latest
needs: setup
strategy:
fail-fast: false
matrix:
service: [
Account,
Avatars,
Console,
Databases,
Functions,
FunctionsSchedule,
GraphQL,
Health,
Locale,
Projects,
Realtime,
Storage,
Teams,
Users,
Webhooks,
VCS,
Messaging,
Migrations
]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Load Cache
uses: actions/cache@v4
with:
key: ${{ env.CACHE_KEY }}
path: /tmp/${{ env.IMAGE }}.tar
fail-on-cache-miss: true
- name: Load and Start Appwrite
run: |
docker load --input /tmp/${{ env.IMAGE }}.tar
docker compose up -d
sleep 30
- name: Run ${{ matrix.service }} tests with Project table mode
run: |
echo "Using project tables"
export _APP_DATABASE_SHARED_TABLES=
export _APP_DATABASE_SHARED_TABLES_V1=
docker compose exec -T \
-e _APP_DATABASE_SHARED_TABLES \
-e _APP_DATABASE_SHARED_TABLES_V1 \
appwrite test /usr/src/code/tests/e2e/Services/${{ matrix.service }} --debug
e2e_shared_mode_test:
name: E2E Shared Mode Service Test
runs-on: ubuntu-latest
needs: [ setup, check_database_changes ]
if: needs.check_database_changes.outputs.database_changed == 'true'
strategy:
fail-fast: false
matrix:
@@ -128,7 +208,6 @@ jobs:
Migrations
]
tables-mode: [
'Project',
'Shared V1',
'Shared V2',
]
@@ -160,12 +239,8 @@ jobs:
echo "Using shared tables V2"
export _APP_DATABASE_SHARED_TABLES=database_db_main
export _APP_DATABASE_SHARED_TABLES_V1=
else
echo "Using project tables"
export _APP_DATABASE_SHARED_TABLES=
export _APP_DATABASE_SHARED_TABLES_V1=
fi
docker compose exec -T \
-e _APP_DATABASE_SHARED_TABLES \
-e _APP_DATABASE_SHARED_TABLES_V1 \
@@ -251,4 +326,4 @@ jobs:
comment-id: ${{ steps.fc.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body-path: benchmark.txt
edit-mode: replace
edit-mode: replace
CONTRIBUTING.md
+6 -6
View File
@@ -367,7 +367,7 @@ In file `app/controllers/shared/api.php` On the database listener, add to an exi
```php
case $document->getCollection() === 'teams':
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_TEAMS, $value); // per project
break;
```
@@ -379,10 +379,10 @@ In that case you need also to handle children removal using addReduce() method c
```php
case $document->getCollection() === 'buckets': //buckets
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_BUCKETS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
$queueForStatsUsage
->addReduce($document);
}
break;
@@ -428,16 +428,16 @@ public function __construct()
->inject('dbForProject')
->inject('queueForFunctions')
->inject('queueForEvents')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('log')
->callback(fn (Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForUsage, $log));
->callback(fn (Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $log));
}
```
and then trigger the queue with the new metric like so:
```php
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_BUILDS, 1)
->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
Dockerfile
+4
View File
@@ -85,6 +85,10 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/worker-messaging && \
chmod +x /usr/local/bin/worker-migrations && \
chmod +x /usr/local/bin/worker-webhooks && \
chmod +x /usr/local/bin/worker-stats-usage && \
chmod +x /usr/local/bin/worker-stats-usage-dump && \
chmod +x /usr/local/bin/stats-resources && \
chmod +x /usr/local/bin/worker-stats-resources && \
chmod +x /usr/local/bin/worker-usage && \
chmod +x /usr/local/bin/worker-usage-dump
app/cli.php
+41
View File
@@ -5,6 +5,8 @@ require_once __DIR__ . '/init.php';
use Appwrite\Event\Certificate;
use Appwrite\Event\Delete;
use Appwrite\Event\Func;
use Appwrite\Event\StatsResources;
use Appwrite\Event\StatsUsage;
use Appwrite\Platform\Appwrite;
use Appwrite\Runtimes\Runtimes;
use Utopia\Cache\Adapter\Sharding;
@@ -160,6 +162,45 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform
};
}, ['pools', 'dbForPlatform', 'cache']);
CLI::setResource('getLogsDB', function (Group $pools, Cache $cache) {
$database = null;
return function (?Document $project = null) use ($pools, $cache, $database) {
if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') {
$database->setTenant($project->getInternalId());
return $database;
}
$dbAdapter = $pools
->get('logs')
->pop()
->getResource();
$database = new Database(
$dbAdapter,
$cache
);
$database
->setSharedTables(true)
->setNamespace('logsV1')
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
// set tenant
if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') {
$database->setTenant($project->getInternalId());
}
return $database;
};
}, ['pools', 'cache']);
CLI::setResource('queueForStatsUsage', function (Connection $publisher) {
return new StatsUsage($publisher);
}, ['publisher']);
CLI::setResource('queueForStatsResources', function (Publisher $publisher) {
return new StatsResources($publisher);
}, ['publisher']);
CLI::setResource('publisher', function (Group $pools) {
return $pools->get('publisher')->pop()->getResource();
}, ['pools']);
app/config/collections.php
+2
View File
@@ -5,6 +5,7 @@ $common = include __DIR__ . '/collections/common.php';
$projects = include __DIR__ . '/collections/projects.php';
$databases = include __DIR__ . '/collections/databases.php';
$platform = include __DIR__ . '/collections/platform.php';
$logs = include __DIR__ . '/collections/logs.php';
// see - http.php#245
// $collections['buckets']['files'];
@@ -27,6 +28,7 @@ $collections = [
'databases' => $databases,
'projects' => array_merge($projects, $common),
'console' => array_merge($platform, $common),
'logs' => $logs,
];
return $collections;
app/config/collections/logs.php
+94
View File
@@ -0,0 +1,94 @@
<?php
use Utopia\Database\Database;
use Utopia\Database\Helpers\ID;
$logsCollection = [];
$logsCollection['stats'] = [
'$collection' => ID::custom(Database::METADATA),
'$id' => ID::custom('stats'),
'name' => 'stats',
'attributes' => [
[
'$id' => ID::custom('metric'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('region'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 255,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('value'),
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 8,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('time'),
'type' => Database::VAR_DATETIME,
'format' => '',
'size' => 0,
'signed' => false,
'required' => false,
'default' => null,
'array' => false,
'filters' => ['datetime'],
],
[
'$id' => ID::custom('period'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 4,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
],
'indexes' => [
[
'$id' => ID::custom('_key_time'),
'type' => Database::INDEX_KEY,
'attributes' => ['time'],
'lengths' => [],
'orders' => [Database::ORDER_DESC],
],
[
'$id' => ID::custom('_key_period_time'),
'type' => Database::INDEX_KEY,
'attributes' => ['period', 'time'],
'lengths' => [],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_key_metric_period_time'),
'type' => Database::INDEX_UNIQUE,
'attributes' => ['metric', 'period', 'time'],
'lengths' => [],
'orders' => [Database::ORDER_DESC],
],
],
];
return $logsCollection;
app/config/specs/open-api3-1.6.x-client.json
+28 -28
View File
@@ -4761,7 +4761,7 @@
},
"x-appwrite": {
"method": "listExecutions",
"weight": 305,
"weight": 306,
"cookies": false,
"type": "",
"deprecated": false,
@@ -4846,7 +4846,7 @@
},
"x-appwrite": {
"method": "createExecution",
"weight": 304,
"weight": 305,
"cookies": false,
"type": "",
"deprecated": false,
@@ -4960,7 +4960,7 @@
},
"x-appwrite": {
"method": "getExecution",
"weight": 306,
"weight": 307,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5033,7 +5033,7 @@
},
"x-appwrite": {
"method": "query",
"weight": 330,
"weight": 331,
"cookies": false,
"type": "graphql",
"deprecated": false,
@@ -5084,7 +5084,7 @@
},
"x-appwrite": {
"method": "mutation",
"weight": 329,
"weight": 330,
"cookies": false,
"type": "graphql",
"deprecated": false,
@@ -5543,7 +5543,7 @@
},
"x-appwrite": {
"method": "createSubscriber",
"weight": 375,
"weight": 376,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5625,7 +5625,7 @@
},
"x-appwrite": {
"method": "deleteSubscriber",
"weight": 379,
"weight": 380,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5699,7 +5699,7 @@
},
"x-appwrite": {
"method": "listFiles",
"weight": 207,
"weight": 208,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5784,7 +5784,7 @@
},
"x-appwrite": {
"method": "createFile",
"weight": 206,
"weight": 207,
"cookies": false,
"type": "upload",
"deprecated": false,
@@ -5881,7 +5881,7 @@
},
"x-appwrite": {
"method": "getFile",
"weight": 208,
"weight": 209,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5952,7 +5952,7 @@
},
"x-appwrite": {
"method": "updateFile",
"weight": 213,
"weight": 214,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6040,7 +6040,7 @@
},
"x-appwrite": {
"method": "deleteFile",
"weight": 214,
"weight": 215,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6106,7 +6106,7 @@
},
"x-appwrite": {
"method": "getFileDownload",
"weight": 210,
"weight": 211,
"cookies": false,
"type": "location",
"deprecated": false,
@@ -6172,7 +6172,7 @@
},
"x-appwrite": {
"method": "getFilePreview",
"weight": 209,
"weight": 210,
"cookies": false,
"type": "location",
"deprecated": false,
@@ -6388,7 +6388,7 @@
},
"x-appwrite": {
"method": "getFileView",
"weight": 211,
"weight": 212,
"cookies": false,
"type": "location",
"deprecated": false,
@@ -6461,7 +6461,7 @@
},
"x-appwrite": {
"method": "list",
"weight": 218,
"weight": 219,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6536,7 +6536,7 @@
},
"x-appwrite": {
"method": "create",
"weight": 217,
"weight": 218,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6620,7 +6620,7 @@
},
"x-appwrite": {
"method": "get",
"weight": 219,
"weight": 220,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6681,7 +6681,7 @@
},
"x-appwrite": {
"method": "updateName",
"weight": 221,
"weight": 222,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6754,7 +6754,7 @@
},
"x-appwrite": {
"method": "delete",
"weight": 223,
"weight": 224,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6817,7 +6817,7 @@
},
"x-appwrite": {
"method": "listMemberships",
"weight": 225,
"weight": 226,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6902,7 +6902,7 @@
},
"x-appwrite": {
"method": "createMembership",
"weight": 224,
"weight": 225,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7012,7 +7012,7 @@
},
"x-appwrite": {
"method": "getMembership",
"weight": 226,
"weight": 227,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7083,7 +7083,7 @@
},
"x-appwrite": {
"method": "updateMembership",
"weight": 227,
"weight": 228,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7169,7 +7169,7 @@
},
"x-appwrite": {
"method": "deleteMembership",
"weight": 229,
"weight": 230,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7242,7 +7242,7 @@
},
"x-appwrite": {
"method": "updateMembershipStatus",
"weight": 228,
"weight": 229,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7339,7 +7339,7 @@
},
"x-appwrite": {
"method": "getPrefs",
"weight": 220,
"weight": 221,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7399,7 +7399,7 @@
},
"x-appwrite": {
"method": "updatePrefs",
"weight": 222,
"weight": 223,
"cookies": false,
"type": "",
"deprecated": false,
app/config/specs/open-api3-1.6.x-console.json
+297 -249
View File
File diff suppressed because it is too large Load Diff
app/config/specs/open-api3-1.6.x-server.json
+208 -159
View File
File diff suppressed because it is too large Load Diff
app/config/specs/open-api3-latest-client.json
+28 -28
View File
@@ -4761,7 +4761,7 @@
},
"x-appwrite": {
"method": "listExecutions",
"weight": 305,
"weight": 306,
"cookies": false,
"type": "",
"deprecated": false,
@@ -4846,7 +4846,7 @@
},
"x-appwrite": {
"method": "createExecution",
"weight": 304,
"weight": 305,
"cookies": false,
"type": "",
"deprecated": false,
@@ -4960,7 +4960,7 @@
},
"x-appwrite": {
"method": "getExecution",
"weight": 306,
"weight": 307,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5033,7 +5033,7 @@
},
"x-appwrite": {
"method": "query",
"weight": 330,
"weight": 331,
"cookies": false,
"type": "graphql",
"deprecated": false,
@@ -5084,7 +5084,7 @@
},
"x-appwrite": {
"method": "mutation",
"weight": 329,
"weight": 330,
"cookies": false,
"type": "graphql",
"deprecated": false,
@@ -5543,7 +5543,7 @@
},
"x-appwrite": {
"method": "createSubscriber",
"weight": 375,
"weight": 376,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5625,7 +5625,7 @@
},
"x-appwrite": {
"method": "deleteSubscriber",
"weight": 379,
"weight": 380,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5699,7 +5699,7 @@
},
"x-appwrite": {
"method": "listFiles",
"weight": 207,
"weight": 208,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5784,7 +5784,7 @@
},
"x-appwrite": {
"method": "createFile",
"weight": 206,
"weight": 207,
"cookies": false,
"type": "upload",
"deprecated": false,
@@ -5881,7 +5881,7 @@
},
"x-appwrite": {
"method": "getFile",
"weight": 208,
"weight": 209,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5952,7 +5952,7 @@
},
"x-appwrite": {
"method": "updateFile",
"weight": 213,
"weight": 214,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6040,7 +6040,7 @@
},
"x-appwrite": {
"method": "deleteFile",
"weight": 214,
"weight": 215,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6106,7 +6106,7 @@
},
"x-appwrite": {
"method": "getFileDownload",
"weight": 210,
"weight": 211,
"cookies": false,
"type": "location",
"deprecated": false,
@@ -6172,7 +6172,7 @@
},
"x-appwrite": {
"method": "getFilePreview",
"weight": 209,
"weight": 210,
"cookies": false,
"type": "location",
"deprecated": false,
@@ -6388,7 +6388,7 @@
},
"x-appwrite": {
"method": "getFileView",
"weight": 211,
"weight": 212,
"cookies": false,
"type": "location",
"deprecated": false,
@@ -6461,7 +6461,7 @@
},
"x-appwrite": {
"method": "list",
"weight": 218,
"weight": 219,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6536,7 +6536,7 @@
},
"x-appwrite": {
"method": "create",
"weight": 217,
"weight": 218,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6620,7 +6620,7 @@
},
"x-appwrite": {
"method": "get",
"weight": 219,
"weight": 220,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6681,7 +6681,7 @@
},
"x-appwrite": {
"method": "updateName",
"weight": 221,
"weight": 222,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6754,7 +6754,7 @@
},
"x-appwrite": {
"method": "delete",
"weight": 223,
"weight": 224,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6817,7 +6817,7 @@
},
"x-appwrite": {
"method": "listMemberships",
"weight": 225,
"weight": 226,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6902,7 +6902,7 @@
},
"x-appwrite": {
"method": "createMembership",
"weight": 224,
"weight": 225,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7012,7 +7012,7 @@
},
"x-appwrite": {
"method": "getMembership",
"weight": 226,
"weight": 227,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7083,7 +7083,7 @@
},
"x-appwrite": {
"method": "updateMembership",
"weight": 227,
"weight": 228,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7169,7 +7169,7 @@
},
"x-appwrite": {
"method": "deleteMembership",
"weight": 229,
"weight": 230,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7242,7 +7242,7 @@
},
"x-appwrite": {
"method": "updateMembershipStatus",
"weight": 228,
"weight": 229,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7339,7 +7339,7 @@
},
"x-appwrite": {
"method": "getPrefs",
"weight": 220,
"weight": 221,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7399,7 +7399,7 @@
},
"x-appwrite": {
"method": "updatePrefs",
"weight": 222,
"weight": 223,
"cookies": false,
"type": "",
"deprecated": false,
app/config/specs/open-api3-latest-console.json
+297 -249
View File
File diff suppressed because it is too large Load Diff
app/config/specs/open-api3-latest-server.json
+208 -159
View File
File diff suppressed because it is too large Load Diff
app/config/specs/swagger2-1.6.x-client.json
+28 -28
View File
@@ -4927,7 +4927,7 @@
},
"x-appwrite": {
"method": "listExecutions",
"weight": 305,
"weight": 306,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5009,7 +5009,7 @@
},
"x-appwrite": {
"method": "createExecution",
"weight": 304,
"weight": 305,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5127,7 +5127,7 @@
},
"x-appwrite": {
"method": "getExecution",
"weight": 306,
"weight": 307,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5198,7 +5198,7 @@
},
"x-appwrite": {
"method": "query",
"weight": 330,
"weight": 331,
"cookies": false,
"type": "graphql",
"deprecated": false,
@@ -5271,7 +5271,7 @@
},
"x-appwrite": {
"method": "mutation",
"weight": 329,
"weight": 330,
"cookies": false,
"type": "graphql",
"deprecated": false,
@@ -5768,7 +5768,7 @@
},
"x-appwrite": {
"method": "createSubscriber",
"weight": 375,
"weight": 376,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5852,7 +5852,7 @@
},
"x-appwrite": {
"method": "deleteSubscriber",
"weight": 379,
"weight": 380,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5924,7 +5924,7 @@
},
"x-appwrite": {
"method": "listFiles",
"weight": 207,
"weight": 208,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6006,7 +6006,7 @@
},
"x-appwrite": {
"method": "createFile",
"weight": 206,
"weight": 207,
"cookies": false,
"type": "upload",
"deprecated": false,
@@ -6097,7 +6097,7 @@
},
"x-appwrite": {
"method": "getFile",
"weight": 208,
"weight": 209,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6166,7 +6166,7 @@
},
"x-appwrite": {
"method": "updateFile",
"weight": 213,
"weight": 214,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6254,7 +6254,7 @@
},
"x-appwrite": {
"method": "deleteFile",
"weight": 214,
"weight": 215,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6325,7 +6325,7 @@
},
"x-appwrite": {
"method": "getFileDownload",
"weight": 210,
"weight": 211,
"cookies": false,
"type": "location",
"deprecated": false,
@@ -6396,7 +6396,7 @@
},
"x-appwrite": {
"method": "getFilePreview",
"weight": 209,
"weight": 210,
"cookies": false,
"type": "location",
"deprecated": false,
@@ -6595,7 +6595,7 @@
},
"x-appwrite": {
"method": "getFileView",
"weight": 211,
"weight": 212,
"cookies": false,
"type": "location",
"deprecated": false,
@@ -6666,7 +6666,7 @@
},
"x-appwrite": {
"method": "list",
"weight": 218,
"weight": 219,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6740,7 +6740,7 @@
},
"x-appwrite": {
"method": "create",
"weight": 217,
"weight": 218,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6831,7 +6831,7 @@
},
"x-appwrite": {
"method": "get",
"weight": 219,
"weight": 220,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6892,7 +6892,7 @@
},
"x-appwrite": {
"method": "updateName",
"weight": 221,
"weight": 222,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6966,7 +6966,7 @@
},
"x-appwrite": {
"method": "delete",
"weight": 223,
"weight": 224,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7029,7 +7029,7 @@
},
"x-appwrite": {
"method": "listMemberships",
"weight": 225,
"weight": 226,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7111,7 +7111,7 @@
},
"x-appwrite": {
"method": "createMembership",
"weight": 224,
"weight": 225,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7225,7 +7225,7 @@
},
"x-appwrite": {
"method": "getMembership",
"weight": 226,
"weight": 227,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7294,7 +7294,7 @@
},
"x-appwrite": {
"method": "updateMembership",
"weight": 227,
"weight": 228,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7379,7 +7379,7 @@
},
"x-appwrite": {
"method": "deleteMembership",
"weight": 229,
"weight": 230,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7450,7 +7450,7 @@
},
"x-appwrite": {
"method": "updateMembershipStatus",
"weight": 228,
"weight": 229,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7545,7 +7545,7 @@
},
"x-appwrite": {
"method": "getPrefs",
"weight": 220,
"weight": 221,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7605,7 +7605,7 @@
},
"x-appwrite": {
"method": "updatePrefs",
"weight": 222,
"weight": 223,
"cookies": false,
"type": "",
"deprecated": false,
app/config/specs/swagger2-1.6.x-console.json
+299 -249
View File
File diff suppressed because it is too large Load Diff
app/config/specs/swagger2-1.6.x-server.json
+210 -159
View File
File diff suppressed because it is too large Load Diff
app/config/specs/swagger2-latest-client.json
+28 -28
View File
@@ -4927,7 +4927,7 @@
},
"x-appwrite": {
"method": "listExecutions",
"weight": 305,
"weight": 306,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5009,7 +5009,7 @@
},
"x-appwrite": {
"method": "createExecution",
"weight": 304,
"weight": 305,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5127,7 +5127,7 @@
},
"x-appwrite": {
"method": "getExecution",
"weight": 306,
"weight": 307,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5198,7 +5198,7 @@
},
"x-appwrite": {
"method": "query",
"weight": 330,
"weight": 331,
"cookies": false,
"type": "graphql",
"deprecated": false,
@@ -5271,7 +5271,7 @@
},
"x-appwrite": {
"method": "mutation",
"weight": 329,
"weight": 330,
"cookies": false,
"type": "graphql",
"deprecated": false,
@@ -5768,7 +5768,7 @@
},
"x-appwrite": {
"method": "createSubscriber",
"weight": 375,
"weight": 376,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5852,7 +5852,7 @@
},
"x-appwrite": {
"method": "deleteSubscriber",
"weight": 379,
"weight": 380,
"cookies": false,
"type": "",
"deprecated": false,
@@ -5924,7 +5924,7 @@
},
"x-appwrite": {
"method": "listFiles",
"weight": 207,
"weight": 208,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6006,7 +6006,7 @@
},
"x-appwrite": {
"method": "createFile",
"weight": 206,
"weight": 207,
"cookies": false,
"type": "upload",
"deprecated": false,
@@ -6097,7 +6097,7 @@
},
"x-appwrite": {
"method": "getFile",
"weight": 208,
"weight": 209,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6166,7 +6166,7 @@
},
"x-appwrite": {
"method": "updateFile",
"weight": 213,
"weight": 214,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6254,7 +6254,7 @@
},
"x-appwrite": {
"method": "deleteFile",
"weight": 214,
"weight": 215,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6325,7 +6325,7 @@
},
"x-appwrite": {
"method": "getFileDownload",
"weight": 210,
"weight": 211,
"cookies": false,
"type": "location",
"deprecated": false,
@@ -6396,7 +6396,7 @@
},
"x-appwrite": {
"method": "getFilePreview",
"weight": 209,
"weight": 210,
"cookies": false,
"type": "location",
"deprecated": false,
@@ -6595,7 +6595,7 @@
},
"x-appwrite": {
"method": "getFileView",
"weight": 211,
"weight": 212,
"cookies": false,
"type": "location",
"deprecated": false,
@@ -6666,7 +6666,7 @@
},
"x-appwrite": {
"method": "list",
"weight": 218,
"weight": 219,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6740,7 +6740,7 @@
},
"x-appwrite": {
"method": "create",
"weight": 217,
"weight": 218,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6831,7 +6831,7 @@
},
"x-appwrite": {
"method": "get",
"weight": 219,
"weight": 220,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6892,7 +6892,7 @@
},
"x-appwrite": {
"method": "updateName",
"weight": 221,
"weight": 222,
"cookies": false,
"type": "",
"deprecated": false,
@@ -6966,7 +6966,7 @@
},
"x-appwrite": {
"method": "delete",
"weight": 223,
"weight": 224,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7029,7 +7029,7 @@
},
"x-appwrite": {
"method": "listMemberships",
"weight": 225,
"weight": 226,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7111,7 +7111,7 @@
},
"x-appwrite": {
"method": "createMembership",
"weight": 224,
"weight": 225,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7225,7 +7225,7 @@
},
"x-appwrite": {
"method": "getMembership",
"weight": 226,
"weight": 227,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7294,7 +7294,7 @@
},
"x-appwrite": {
"method": "updateMembership",
"weight": 227,
"weight": 228,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7379,7 +7379,7 @@
},
"x-appwrite": {
"method": "deleteMembership",
"weight": 229,
"weight": 230,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7450,7 +7450,7 @@
},
"x-appwrite": {
"method": "updateMembershipStatus",
"weight": 228,
"weight": 229,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7545,7 +7545,7 @@
},
"x-appwrite": {
"method": "getPrefs",
"weight": 220,
"weight": 221,
"cookies": false,
"type": "",
"deprecated": false,
@@ -7605,7 +7605,7 @@
},
"x-appwrite": {
"method": "updatePrefs",
"weight": 222,
"weight": 223,
"cookies": false,
"type": "",
"deprecated": false,
app/config/specs/swagger2-latest-console.json
+299 -249
View File
File diff suppressed because it is too large Load Diff
app/config/specs/swagger2-latest-server.json
+210 -159
View File
File diff suppressed because it is too large Load Diff
app/config/storage/inputs.php
+1
View File
@@ -7,4 +7,5 @@ return [
"gif" => "image/gif",
"png" => "image/png",
"heic" => "image/heic",
"webp" => "image/webp",
];
app/controllers/api/account.php
+21 -21
View File
@@ -17,11 +17,10 @@ use Appwrite\Event\Delete;
use Appwrite\Event\Event;
use Appwrite\Event\Mail;
use Appwrite\Event\Messaging;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Extend\Exception;
use Appwrite\Hooks\Hooks;
use Appwrite\Network\Validator\Email;
use Appwrite\Network\Validator\Redirect;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\ContentType;
@@ -60,6 +59,7 @@ use Utopia\System\System;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Assoc;
use Utopia\Validator\Boolean;
use Utopia\Validator\Host;
use Utopia\Validator\Text;
use Utopia\Validator\URL;
use Utopia\Validator\WhiteList;
@@ -1188,8 +1188,8 @@ App::get('/v1/account/sessions/oauth2/:provider')
->label('abuse-limit', 50)
->label('abuse-key', 'ip:{ip}')
->param('provider', '', new WhiteList(\array_keys(Config::getParam('oAuthProviders')), true), 'OAuth2 Provider. Currently, supported providers are: ' . \implode(', ', \array_keys(\array_filter(Config::getParam('oAuthProviders'), fn ($node) => (!$node['mock'])))) . '.')
->param('success', '', fn ($hostnames, $schemes) => new Redirect($hostnames, $schemes), 'URL to redirect back to your app after a successful login attempt. Only URLs from hostnames in your project\'s platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['hostnames', 'schemes'])
->param('failure', '', fn ($hostnames, $schemes) => new Redirect($hostnames, $schemes), 'URL to redirect back to your app after a failed login attempt. Only URLs from hostnames in your project\'s platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['hostnames', 'schemes'])
->param('success', '', fn ($clients) => new Host($clients), 'URL to redirect back to your app after a successful login attempt. Only URLs from hostnames in your project\'s platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients'])
->param('failure', '', fn ($clients) => new Host($clients), 'URL to redirect back to your app after a failed login attempt. Only URLs from hostnames in your project\'s platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients'])
->param('scopes', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'A list of custom OAuth2 scopes. Check each provider internal docs for a list of supported scopes. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' scopes are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true)
->inject('request')
->inject('response')
@@ -1784,8 +1784,8 @@ App::get('/v1/account/tokens/oauth2/:provider')
->label('abuse-limit', 50)
->label('abuse-key', 'ip:{ip}')
->param('provider', '', new WhiteList(\array_keys(Config::getParam('oAuthProviders')), true), 'OAuth2 Provider. Currently, supported providers are: ' . \implode(', ', \array_keys(\array_filter(Config::getParam('oAuthProviders'), fn ($node) => (!$node['mock'])))) . '.')
->param('success', '', fn ($hostnames, $schemes) => new Redirect($hostnames, $schemes), 'URL to redirect back to your app after a successful login attempt. Only URLs from hostnames in your project\'s platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['hostnames', 'schemes'])
->param('failure', '', fn ($hostnames, $schemes) => new Redirect($hostnames, $schemes), 'URL to redirect back to your app after a failed login attempt. Only URLs from hostnames in your project\'s platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['hostnames', 'schemes'])
->param('success', '', fn ($clients) => new Host($clients), 'URL to redirect back to your app after a successful login attempt. Only URLs from hostnames in your project\'s platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients'])
->param('failure', '', fn ($clients) => new Host($clients), 'URL to redirect back to your app after a failed login attempt. Only URLs from hostnames in your project\'s platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients'])
->param('scopes', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'A list of custom OAuth2 scopes. Check each provider internal docs for a list of supported scopes. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' scopes are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true)
->inject('request')
->inject('response')
@@ -1864,7 +1864,7 @@ App::post('/v1/account/tokens/magic-url')
->label('abuse-key', ['url:{url},email:{param-email}', 'url:{url},ip:{ip}'])
->param('userId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.')
->param('email', '', new Email(), 'User email.')
->param('url', '', fn ($hostnames, $schemes) => new Redirect($hostnames, $schemes), 'URL to redirect the user back to your app from the magic URL login. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['hostnames', 'schemes'])
->param('url', '', fn ($clients) => new Host($clients), 'URL to redirect the user back to your app from the magic URL login. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients'])
->param('phrase', false, new Boolean(), 'Toggle for security phrase. If enabled, email will be send with a randomly generated phrase and the phrase will also be included in the response. Confirming phrases match increases the security of your authentication flow.', true)
->inject('request')
->inject('response')
@@ -2432,9 +2432,9 @@ App::post('/v1/account/tokens/phone')
->inject('queueForMessaging')
->inject('locale')
->inject('timelimit')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('plan')
->action(function (string $userId, string $phone, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale, callable $timelimit, Usage $queueForUsage, array $plan) {
->action(function (string $userId, string $phone, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan) {
if (empty(System::getEnv('_APP_SMS_PROVIDER'))) {
throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured');
}
@@ -2583,11 +2583,11 @@ App::post('/v1/account/tokens/phone')
$countryCode = $helper->parse($phone)->getCountryCode();
if (!empty($countryCode)) {
$queueForUsage
$queueForStatsUsage
->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1);
}
}
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_AUTH_METHOD_PHONE, 1)
->setProject($project)
->trigger();
@@ -3157,7 +3157,7 @@ App::post('/v1/account/recovery')
->label('abuse-limit', 10)
->label('abuse-key', ['url:{url},email:{param-email}', 'url:{url},ip:{ip}'])
->param('email', '', new Email(), 'User email.')
->param('url', '', fn ($hostnames, $schemes) => new Redirect($hostnames, $schemes), 'URL to redirect the user back to your app from the recovery email. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', false, ['hostnames', 'schemes'])
->param('url', '', fn ($clients) => new Host($clients), 'URL to redirect the user back to your app from the recovery email. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', false, ['clients'])
->inject('request')
->inject('response')
->inject('user')
@@ -3432,7 +3432,7 @@ App::post('/v1/account/verification')
))
->label('abuse-limit', 10)
->label('abuse-key', 'url:{url},userId:{userId}')
->param('url', '', fn ($hostnames, $schemes) => new Redirect($hostnames, $schemes), 'URL to redirect the user back to your app from the verification email. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', false, ['hostnames', 'schemes']) // TODO add built-in confirm page
->param('url', '', fn ($clients) => new Host($clients), 'URL to redirect the user back to your app from the verification email. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', false, ['clients']) // TODO add built-in confirm page
->inject('request')
->inject('response')
->inject('project')
@@ -3678,9 +3678,9 @@ App::post('/v1/account/verification/phone')
->inject('project')
->inject('locale')
->inject('timelimit')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('plan')
->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Document $project, Locale $locale, callable $timelimit, Usage $queueForUsage, array $plan) {
->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Document $project, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan) {
if (empty(System::getEnv('_APP_SMS_PROVIDER'))) {
throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured');
}
@@ -3775,11 +3775,11 @@ App::post('/v1/account/verification/phone')
$countryCode = $helper->parse($phone)->getCountryCode();
if (!empty($countryCode)) {
$queueForUsage
$queueForStatsUsage
->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1);
}
}
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_AUTH_METHOD_PHONE, 1)
->setProject($project)
->trigger();
@@ -4310,9 +4310,9 @@ App::post('/v1/account/mfa/challenge')
->inject('queueForMessaging')
->inject('queueForMails')
->inject('timelimit')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('plan')
->action(function (string $factor, Response $response, Database $dbForProject, Document $user, Locale $locale, Document $project, Request $request, Event $queueForEvents, Messaging $queueForMessaging, Mail $queueForMails, callable $timelimit, Usage $queueForUsage, array $plan) {
->action(function (string $factor, Response $response, Database $dbForProject, Document $user, Locale $locale, Document $project, Request $request, Event $queueForEvents, Messaging $queueForMessaging, Mail $queueForMails, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan) {
$expire = DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_CONFIRM);
$code = Auth::codeGenerator();
@@ -4383,11 +4383,11 @@ App::post('/v1/account/mfa/challenge')
$countryCode = $helper->parse($phone)->getCountryCode();
if (!empty($countryCode)) {
$queueForUsage
$queueForStatsUsage
->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1);
}
}
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_AUTH_METHOD_PHONE, 1)
->setProject($project)
->trigger();
app/controllers/api/databases.php
+28 -25
View File
@@ -4,7 +4,7 @@ use Appwrite\Auth\Auth;
use Appwrite\Detector\Detector;
use Appwrite\Event\Database as EventDatabase;
use Appwrite\Event\Event;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Extend\Exception;
use Appwrite\Network\Validator\Email;
use Appwrite\SDK\AuthType;
@@ -27,6 +27,7 @@ use Utopia\Database\Document;
use Utopia\Database\Exception\Authorization as AuthorizationException;
use Utopia\Database\Exception\Conflict as ConflictException;
use Utopia\Database\Exception\Duplicate as DuplicateException;
use Utopia\Database\Exception\Index as IndexException;
use Utopia\Database\Exception\Limit as LimitException;
use Utopia\Database\Exception\NotFound as NotFoundException;
use Utopia\Database\Exception\Query as QueryException;
@@ -393,6 +394,8 @@ function updateAttribute(
throw new Exception(Exception::ATTRIBUTE_NOT_FOUND);
} catch (LimitException) {
throw new Exception(Exception::ATTRIBUTE_LIMIT_EXCEEDED);
} catch (IndexException $e) {
throw new Exception(Exception::INDEX_INVALID, $e->getMessage());
}
}
@@ -476,8 +479,8 @@ App::post('/v1/databases')
->inject('response')
->inject('dbForProject')
->inject('queueForEvents')
->inject('queueForUsage')
->action(function (string $databaseId, string $name, bool $enabled, Response $response, Database $dbForProject, Event $queueForEvents, Usage $queueForUsage) {
->inject('queueForStatsUsage')
->action(function (string $databaseId, string $name, bool $enabled, Response $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage) {
$databaseId = $databaseId == 'unique()' ? ID::unique() : $databaseId;
@@ -527,7 +530,7 @@ App::post('/v1/databases')
}
$queueForEvents->setParam('databaseId', $database->getId());
$queueForUsage->addMetric(str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_STORAGE), 1); // per database
$queueForStatsUsage->addMetric(str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_STORAGE), 1); // per database
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -797,8 +800,8 @@ App::delete('/v1/databases/:databaseId')
->inject('dbForProject')
->inject('queueForDatabase')
->inject('queueForEvents')
->inject('queueForUsage')
->action(function (string $databaseId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, Usage $queueForUsage) {
->inject('queueForStatsUsage')
->action(function (string $databaseId, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, StatsUsage $queueForStatsUsage) {
$database = $dbForProject->getDocument('databases', $databaseId);
@@ -821,7 +824,7 @@ App::delete('/v1/databases/:databaseId')
->setParam('databaseId', $database->getId())
->setPayload($response->output($database, Response::MODEL_DATABASE));
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_DATABASES_STORAGE, 1); // Global, deletion forces full recalculation
$response->noContent();
@@ -2618,8 +2621,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key
->inject('dbForProject')
->inject('queueForDatabase')
->inject('queueForEvents')
->inject('queueForUsage')
->action(function (string $databaseId, string $collectionId, string $key, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, Usage $queueForUsage) {
->inject('queueForStatsUsage')
->action(function (string $databaseId, string $collectionId, string $key, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents, StatsUsage $queueForStatsUsage) {
$db = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId));
@@ -2716,7 +2719,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key
->setContext('database', $db)
->setPayload($response->output($attribute, $model));
$queueForUsage
$queueForStatsUsage
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$db->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection
$response->noContent();
@@ -3134,9 +3137,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents')
->inject('dbForProject')
->inject('user')
->inject('queueForEvents')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('mode')
->action(function (string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, Usage $queueForUsage, string $mode) {
->action(function (string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, StatsUsage $queueForStatsUsage, string $mode) {
$data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array
@@ -3339,7 +3342,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents')
$processDocument($collection, $document);
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations)
->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations)
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection
@@ -3392,8 +3395,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents')
->inject('response')
->inject('dbForProject')
->inject('mode')
->inject('queueForUsage')
->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) {
->inject('queueForStatsUsage')
->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, StatsUsage $queueForStatsUsage) {
$database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId));
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
@@ -3505,7 +3508,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents')
$processDocument($collection, $document);
}
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations)
->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations)
;
@@ -3571,8 +3574,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
->inject('response')
->inject('dbForProject')
->inject('mode')
->inject('queueForUsage')
->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) {
->inject('queueForStatsUsage')
->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, StatsUsage $queueForStatsUsage) {
$database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId));
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
@@ -3648,7 +3651,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
$processDocument($collection, $document);
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations)
->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations)
;
@@ -3805,8 +3808,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum
->inject('dbForProject')
->inject('queueForEvents')
->inject('mode')
->inject('queueForUsage')
->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, string $mode, Usage $queueForUsage) {
->inject('queueForStatsUsage')
->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, string $mode, StatsUsage $queueForStatsUsage) {
$data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array
@@ -3946,7 +3949,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum
$setCollection($collection, $newDocument);
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations)
->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations)
;
@@ -4058,9 +4061,9 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
->inject('response')
->inject('dbForProject')
->inject('queueForEvents')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('mode')
->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, Usage $queueForUsage, string $mode) {
->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage, string $mode) {
$database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId));
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
@@ -4129,7 +4132,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
$processDocument($collection, $document);
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1)
->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1)
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection
app/controllers/api/functions.php
+4 -4
View File
@@ -6,7 +6,7 @@ use Appwrite\Event\Build;
use Appwrite\Event\Delete;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Event\Validator\FunctionEvent;
use Appwrite\Extend\Exception;
use Appwrite\Extend\Exception as AppwriteException;
@@ -1900,10 +1900,10 @@ App::post('/v1/functions/:functionId/executions')
->inject('dbForPlatform')
->inject('user')
->inject('queueForEvents')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('queueForFunctions')
->inject('geodb')
->action(function (string $functionId, string $body, mixed $async, string $path, string $method, mixed $headers, ?string $scheduledAt, Response $response, Request $request, Document $project, Database $dbForProject, Database $dbForPlatform, Document $user, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb) {
->action(function (string $functionId, string $body, mixed $async, string $path, string $method, mixed $headers, ?string $scheduledAt, Response $response, Request $request, Document $project, Database $dbForProject, Database $dbForPlatform, Document $user, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb) {
$async = \strval($async) === 'true' || \strval($async) === '1';
if (!$async && !is_null($scheduledAt)) {
@@ -2230,7 +2230,7 @@ App::post('/v1/functions/:functionId/executions')
throw $th;
}
} finally {
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project
app/controllers/api/health.php
+45 -12
View File
@@ -692,15 +692,15 @@ App::get('/v1/health/queue/functions')
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/queue/usage')
->desc('Get usage queue')
App::get('/v1/health/queue/stats-resources')
->desc('Get stats resources queue')
->groups(['api', 'health'])
->label('scope', 'health.read')
->label('sdk', new Method(
auth: [AuthType::KEY],
namespace: 'health',
name: 'getQueueUsage',
description: '/docs/references/health/get-queue-usage.md',
name: 'getQueueStatsResources',
description: '/docs/references/health/get-queue-stats-resources.md',
responses: [
new SDKResponse(
code: Response::STATUS_CODE_OK,
@@ -715,7 +715,7 @@ App::get('/v1/health/queue/usage')
->action(function (int|string $threshold, Publisher $publisher, Response $response) {
$threshold = \intval($threshold);
$size = $publisher->getQueueSize(new Queue(Event::USAGE_QUEUE_NAME));
$size = $publisher->getQueueSize(new Queue(Event::STATS_RESOURCES_QUEUE_NAME));
if ($size >= $threshold) {
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
@@ -724,15 +724,15 @@ App::get('/v1/health/queue/usage')
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
});
App::get('/v1/health/queue/usage-dump')
->desc('Get usage dump queue')
App::get('/v1/health/queue/stats-usage')
->desc('Get stats usage queue')
->groups(['api', 'health'])
->label('scope', 'health.read')
->label('sdk', new Method(
auth: [AuthType::KEY],
namespace: 'health',
name: 'getQueueUsageDump',
description: '/docs/references/health/get-queue-usage-dump.md',
name: 'getQueueUsage',
description: '/docs/references/health/get-queue-stats-usage.md',
responses: [
new SDKResponse(
code: Response::STATUS_CODE_OK,
@@ -747,7 +747,39 @@ App::get('/v1/health/queue/usage-dump')
->action(function (int|string $threshold, Publisher $publisher, Response $response) {
$threshold = \intval($threshold);
$size = $publisher->getQueueSize(new Queue(Event::USAGE_DUMP_QUEUE_NAME));
$size = $publisher->getQueueSize(new Queue(Event::STATS_USAGE_QUEUE_NAME));
if ($size >= $threshold) {
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
});
App::get('/v1/health/queue/stats-usage-dump')
->desc('Get usage dump queue')
->groups(['api', 'health'])
->label('scope', 'health.read')
->label('sdk', new Method(
auth: [AuthType::KEY],
namespace: 'health',
name: 'getQueueStatsUsageDump',
description: '/docs/references/health/get-queue-stats-usage-dump.md',
responses: [
new SDKResponse(
code: Response::STATUS_CODE_OK,
model: Response::MODEL_HEALTH_QUEUE,
)
],
contentType: ContentType::JSON
))
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->inject('publisher')
->inject('response')
->action(function (int|string $threshold, Publisher $publisher, Response $response) {
$threshold = \intval($threshold);
$size = $publisher->getQueueSize(new Queue(Event::STATS_USAGE_DUMP_QUEUE_NAME));
if ($size >= $threshold) {
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
@@ -920,8 +952,9 @@ App::get('/v1/health/queue/failed/:name')
Event::AUDITS_QUEUE_NAME,
Event::MAILS_QUEUE_NAME,
Event::FUNCTIONS_QUEUE_NAME,
Event::USAGE_QUEUE_NAME,
Event::USAGE_DUMP_QUEUE_NAME,
Event::STATS_RESOURCES_QUEUE_NAME,
Event::STATS_USAGE_QUEUE_NAME,
Event::STATS_USAGE_DUMP_QUEUE_NAME,
Event::WEBHOOK_QUEUE_NAME,
Event::CERTIFICATES_QUEUE_NAME,
Event::BUILDS_QUEUE_NAME,
app/controllers/api/projects.php
+1 -1
View File
@@ -1741,7 +1741,7 @@ App::post('/v1/projects/:projectId/platforms')
]
))
->param('projectId', '', new UID(), 'Project unique ID.')
->param('type', null, new WhiteList([Origin::CLIENT_TYPE_WEB, Origin::CLIENT_TYPE_FLUTTER_WEB, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_LINUX, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_APPLE_IOS, Origin::CLIENT_TYPE_APPLE_MACOS, Origin::CLIENT_TYPE_APPLE_WATCHOS, Origin::CLIENT_TYPE_APPLE_TVOS, Origin::CLIENT_TYPE_ANDROID, Origin::CLIENT_TYPE_UNITY, Origin::CLIENT_TYPE_REACT_NATIVE_IOS, Origin::CLIENT_TYPE_REACT_NATIVE_ANDROID], true), 'Platform type.')
->param('type', null, new WhiteList([Origin::CLIENT_TYPE_WEB, Origin::CLIENT_TYPE_FLUTTER_WEB, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_LINUX, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_APPLE_IOS, Origin::CLIENT_TYPE_APPLE_MACOS, Origin::CLIENT_TYPE_APPLE_WATCHOS, Origin::CLIENT_TYPE_APPLE_TVOS, Origin::CLIENT_TYPE_ANDROID, Origin::CLIENT_TYPE_UNITY, Origin::CLIENT_TYPE_REACT_NATIVE_IOS, Origin::CLIENT_TYPE_REACT_NATIVE_ANDROID], true), 'Platform type.')
->param('name', null, new Text(128), 'Platform name. Max length: 128 chars.')
->param('key', '', new Text(256), 'Package name for Android or bundle ID for iOS or macOS. Max length: 256 chars.', true)
->param('store', '', new Text(256), 'App store or Google Play store ID. Max length: 256 chars.', true)
app/controllers/api/storage.php
+4 -4
View File
@@ -6,7 +6,7 @@ use Appwrite\Auth\Auth;
use Appwrite\ClamAV\Network;
use Appwrite\Event\Delete;
use Appwrite\Event\Event;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Extend\Exception;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\SDK\AuthType;
@@ -942,8 +942,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
->inject('mode')
->inject('deviceForFiles')
->inject('deviceForLocal')
->inject('queueForUsage')
->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal, Usage $queueForUsage) {
->inject('queueForStatsUsage')
->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal, StatsUsage $queueForStatsUsage) {
if (!\extension_loaded('imagick')) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing');
@@ -1071,7 +1071,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
$contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg'];
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_FILES_TRANSFORMATIONS, 1)
->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1)
;
app/controllers/api/teams.php
+7 -7
View File
@@ -8,10 +8,9 @@ use Appwrite\Event\Delete;
use Appwrite\Event\Event;
use Appwrite\Event\Mail;
use Appwrite\Event\Messaging;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Extend\Exception;
use Appwrite\Network\Validator\Email;
use Appwrite\Network\Validator\Redirect;
use Appwrite\Platform\Workers\Deletes;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\ContentType;
@@ -50,6 +49,7 @@ use Utopia\Locale\Locale;
use Utopia\System\System;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Assoc;
use Utopia\Validator\Host;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
@@ -455,7 +455,7 @@ App::post('/v1/teams/:teamId/memberships')
}
return new ArrayList(new Key(), APP_LIMIT_ARRAY_PARAMS_SIZE);
}, 'Array of strings. Use this param to set the user roles in the team. A role can be any string. Learn more about [roles and permissions](https://appwrite.io/docs/permissions). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each 32 characters long.', false, ['project'])
->param('url', '', fn ($hostnames, $schemes) => new Redirect($hostnames, $schemes), 'URL to redirect the user back to your app from the invitation email. This parameter is not required when an API key is supplied. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['hostnames', 'schemes']) // TODO add our own built-in confirm page
->param('url', '', fn ($clients) => new Host($clients), 'URL to redirect the user back to your app from the invitation email. This parameter is not required when an API key is supplied. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients']) // TODO add our own built-in confirm page
->param('name', '', new Text(128), 'Name of the new team member. Max length: 128 chars.', true)
->inject('response')
->inject('project')
@@ -466,9 +466,9 @@ App::post('/v1/teams/:teamId/memberships')
->inject('queueForMessaging')
->inject('queueForEvents')
->inject('timelimit')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('plan')
->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, Messaging $queueForMessaging, Event $queueForEvents, callable $timelimit, Usage $queueForUsage, array $plan) {
->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, Messaging $queueForMessaging, Event $queueForEvents, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan) {
$isAppUser = Auth::isAppUser(Authorization::getRoles());
$isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
@@ -757,11 +757,11 @@ App::post('/v1/teams/:teamId/memberships')
$countryCode = $helper->parse($phone)->getCountryCode();
if (!empty($countryCode)) {
$queueForUsage
$queueForStatsUsage
->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1);
}
}
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_AUTH_METHOD_PHONE, 1)
->setProject($project)
->trigger();
app/controllers/api/vcs.php
+2 -2
View File
@@ -288,8 +288,8 @@ App::get('/v1/vcs/github/authorize')
type: MethodType::WEBAUTH,
hide: true,
))
->param('success', '', fn ($hostnames) => new Host($hostnames), 'URL to redirect back to console after a successful installation attempt.', true, ['hostnames'])
->param('failure', '', fn ($hostnames) => new Host($hostnames), 'URL to redirect back to console after a failed installation attempt.', true, ['hostnames'])
->param('success', '', fn ($clients) => new Host($clients), 'URL to redirect back to console after a successful installation attempt.', true, ['clients'])
->param('failure', '', fn ($clients) => new Host($clients), 'URL to redirect back to console after a failed installation attempt.', true, ['clients'])
->inject('request')
->inject('response')
->inject('project')
app/controllers/general.php
+22 -23
View File
@@ -7,7 +7,7 @@ use Appwrite\Auth\Auth;
use Appwrite\Event\Certificate;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Extend\Exception as AppwriteException;
use Appwrite\Network\Validator\Origin;
use Appwrite\SDK\AuthType;
@@ -50,7 +50,7 @@ Config::setParam('domainVerification', false);
Config::setParam('cookieDomain', 'localhost');
Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE);
function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname)
function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, SwooleRequest $swooleRequest, Request $request, Response $response, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname)
{
$utopia->getRoute()?->label('error', __DIR__ . '/../views/general/error.phtml');
@@ -434,7 +434,7 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_NETWORK_REQUESTS, 1)
->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize)
->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize())
@@ -497,16 +497,15 @@ App::init()
->inject('getProjectDB')
->inject('locale')
->inject('localeCodes')
->inject('hostnames')
->inject('clients')
->inject('geodb')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('queueForEvents')
->inject('queueForCertificates')
->inject('queueForFunctions')
->inject('isResourceBlocked')
->inject('previewHostname')
->inject('platforms')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, array $hostnames, Reader $geodb, Usage $queueForUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, string $previewHostname, array $platforms) {
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForPlatform, callable $getProjectDB, Locale $locale, array $localeCodes, array $clients, Reader $geodb, StatsUsage $queueForStatsUsage, Event $queueForEvents, Certificate $queueForCertificates, Func $queueForFunctions, callable $isResourceBlocked, string $previewHostname) {
/*
* Appwrite Router
*/
@@ -514,7 +513,7 @@ App::init()
$mainDomain = System::getEnv('_APP_DOMAIN', '');
// Only run Router when external domain
if ($host !== $mainDomain || !empty($previewHostname)) {
if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) {
if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) {
return;
}
}
@@ -622,7 +621,7 @@ App::init()
$port = \parse_url($request->getOrigin($referrer), PHP_URL_PORT);
$refDomainOrigin = 'localhost';
$validator = new Hostname($hostnames);
$validator = new Hostname($clients);
if ($validator->isValid($origin)) {
$refDomainOrigin = $origin;
}
@@ -713,7 +712,7 @@ App::init()
* Skip this check for non-web platforms which are not required to send an origin header
*/
$origin = $request->getOrigin($request->getReferer(''));
$originValidator = new Origin($platforms);
$originValidator = new Origin(\array_merge($project->getAttribute('platforms', []), $console->getAttribute('platforms', [])));
if (
!$originValidator->isValid($origin)
@@ -733,12 +732,12 @@ App::options()
->inject('dbForPlatform')
->inject('getProjectDB')
->inject('queueForEvents')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('queueForFunctions')
->inject('geodb')
->inject('isResourceBlocked')
->inject('previewHostname')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) {
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) {
/*
* Appwrite Router
*/
@@ -746,7 +745,7 @@ App::options()
$mainDomain = System::getEnv('_APP_DOMAIN', '');
// Only run Router when external domain
if ($host !== $mainDomain || !empty($previewHostname)) {
if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) {
if (router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname)) {
return;
}
}
@@ -771,8 +770,8 @@ App::error()
->inject('project')
->inject('logger')
->inject('log')
->inject('queueForUsage')
->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage) {
->inject('queueForStatsUsage')
->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, StatsUsage $queueForStatsUsage) {
$version = System::getEnv('_APP_VERSION', 'UNKNOWN');
$route = $utopia->getRoute();
$class = \get_class($error);
@@ -883,13 +882,13 @@ App::error()
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_NETWORK_REQUESTS, 1)
->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize)
->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize());
}
$queueForUsage
$queueForStatsUsage
->setProject($project)
->trigger();
}
@@ -1041,12 +1040,12 @@ App::get('/robots.txt')
->inject('dbForPlatform')
->inject('getProjectDB')
->inject('queueForEvents')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('queueForFunctions')
->inject('geodb')
->inject('isResourceBlocked')
->inject('previewHostname')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) {
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) {
$host = $request->getHostname() ?? '';
$mainDomain = System::getEnv('_APP_DOMAIN', '');
@@ -1054,7 +1053,7 @@ App::get('/robots.txt')
$template = new View(__DIR__ . '/../views/general/robots.phtml');
$response->text($template->render(false));
} else {
router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname);
router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname);
}
});
@@ -1069,12 +1068,12 @@ App::get('/humans.txt')
->inject('dbForPlatform')
->inject('getProjectDB')
->inject('queueForEvents')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('queueForFunctions')
->inject('geodb')
->inject('isResourceBlocked')
->inject('previewHostname')
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) {
->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForPlatform, callable $getProjectDB, Event $queueForEvents, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Reader $geodb, callable $isResourceBlocked, string $previewHostname) {
$host = $request->getHostname() ?? '';
$mainDomain = System::getEnv('_APP_DOMAIN', '');
@@ -1082,7 +1081,7 @@ App::get('/humans.txt')
$template = new View(__DIR__ . '/../views/general/humans.phtml');
$response->text($template->render(false));
} else {
router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname);
router($utopia, $dbForPlatform, $getProjectDB, $swooleRequest, $request, $response, $queueForEvents, $queueForStatsUsage, $queueForFunctions, $geodb, $isResourceBlocked, $previewHostname);
}
});
app/controllers/mock.php
+3 -3
View File
@@ -3,7 +3,6 @@
global $utopia, $request, $response;
use Appwrite\Extend\Exception;
use Appwrite\Network\Validator\Redirect;
use Appwrite\Utopia\Request;
use Appwrite\Utopia\Response;
use Utopia\App;
@@ -15,6 +14,7 @@ use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Validator\UID;
use Utopia\System\System;
use Utopia\Validator\Host;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
use Utopia\VCS\Adapter\Git\GitHub;
@@ -26,7 +26,7 @@ App::get('/v1/mock/tests/general/oauth2')
->label('docs', false)
->label('mock', true)
->param('client_id', '', new Text(100), 'OAuth2 Client ID.')
->param('redirect_uri', '', new Redirect(['localhost']), 'OAuth2 Redirect URI.') // Important to deny an open redirect attack
->param('redirect_uri', '', new Host(['localhost']), 'OAuth2 Redirect URI.') // Important to deny an open redirect attack
->param('scope', '', new Text(100), 'OAuth2 scope list.')
->param('state', '', new Text(1024), 'OAuth2 state.')
->inject('response')
@@ -44,7 +44,7 @@ App::get('/v1/mock/tests/general/oauth2/token')
->param('client_id', '', new Text(100), 'OAuth2 Client ID.')
->param('client_secret', '', new Text(100), 'OAuth2 scope list.')
->param('grant_type', 'authorization_code', new WhiteList(['refresh_token', 'authorization_code']), 'OAuth2 Grant Type.', true)
->param('redirect_uri', '', new Redirect(['localhost']), 'OAuth2 Redirect URI.', true)
->param('redirect_uri', '', new Host(['localhost']), 'OAuth2 Redirect URI.', true)
->param('code', '', new Text(100), 'OAuth2 state.', true)
->param('refresh_token', '', new Text(100), 'OAuth2 refresh token.', true)
->inject('response')
app/controllers/shared/api.php
+25 -25
View File
@@ -12,7 +12,7 @@ use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Messaging;
use Appwrite\Event\Realtime;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Event\Webhook;
use Appwrite\Extend\Exception;
use Appwrite\Extend\Exception as AppwriteException;
@@ -90,7 +90,7 @@ $eventDatabaseListener = function (Document $project, Document $document, Respon
}
};
$usageDatabaseListener = function (string $event, Document $document, Usage $queueForUsage) {
$usageDatabaseListener = function (string $event, Document $document, StatsUsage $queueForStatsUsage) {
$value = 1;
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$value = -1;
@@ -98,40 +98,40 @@ $usageDatabaseListener = function (string $event, Document $document, Usage $que
switch (true) {
case $document->getCollection() === 'teams':
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_TEAMS, $value); // per project
break;
case $document->getCollection() === 'users':
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_USERS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
$queueForStatsUsage
->addReduce($document);
}
break;
case $document->getCollection() === 'sessions': // sessions
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_SESSIONS, $value); //per project
break;
case $document->getCollection() === 'databases': // databases
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_DATABASES, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
$queueForStatsUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_COLLECTIONS, $value) // per project
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value)
;
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
$queueForStatsUsage
->addReduce($document);
}
break;
@@ -139,39 +139,39 @@ $usageDatabaseListener = function (string $event, Document $document, Usage $que
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$collectionInternalId = $parts[3] ?? 0;
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_DOCUMENTS, $value) // per project
->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection
break;
case $document->getCollection() === 'buckets': //buckets
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_BUCKETS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
$queueForStatsUsage
->addReduce($document);
}
break;
case str_starts_with($document->getCollection(), 'bucket_'): // files
$parts = explode('_', $document->getCollection());
$bucketInternalId = $parts[1];
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_FILES, $value) // per project
->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project
->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES), $value) // per bucket
->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket
break;
case $document->getCollection() === 'functions':
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_FUNCTIONS, $value); // per project
if ($event === Database::EVENT_DOCUMENT_DELETE) {
$queueForUsage
$queueForStatsUsage
->addReduce($document);
}
break;
case $document->getCollection() === 'deployments':
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_DEPLOYMENTS, $value) // per project
->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project
->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value) // per function
@@ -436,11 +436,11 @@ App::init()
->inject('queueForDeletes')
->inject('queueForDatabase')
->inject('queueForBuilds')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('dbForProject')
->inject('timelimit')
->inject('mode')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, callable $timelimit, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) {
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Publisher $publisher, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Database $dbForProject, callable $timelimit, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) {
$route = $utopia->getRoute();
@@ -550,8 +550,8 @@ App::init()
$queueForRealtime = new Realtime();
$dbForProject
->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForUsage))
->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForUsage))
->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage))
->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage))
->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener(
$project,
$document,
@@ -694,7 +694,7 @@ App::shutdown()
->inject('user')
->inject('queueForEvents')
->inject('queueForAudits')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('queueForDeletes')
->inject('queueForDatabase')
->inject('queueForBuilds')
@@ -703,7 +703,7 @@ App::shutdown()
->inject('queueForWebhooks')
->inject('queueForRealtime')
->inject('dbForProject')
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject) use ($parseLabel) {
->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, StatsUsage $queueForStatsUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject) use ($parseLabel) {
$responsePayload = $response->getPayload();
@@ -860,13 +860,13 @@ App::shutdown()
$fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
}
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_NETWORK_REQUESTS, 1)
->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize)
->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize());
}
$queueForUsage
$queueForStatsUsage
->setProject($project)
->trigger();
}
app/http.php
+129 -122
View File
@@ -17,7 +17,6 @@ use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception\Duplicate;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
@@ -156,6 +155,79 @@ function dispatch(Server $server, int $fd, int $type, $data = null): int
include __DIR__ . '/controllers/general.php';
function createDatabase(App $app, string $resourceKey, string $dbName, array $collections, mixed $pools, callable $extraSetup = null): void
{
$max = 10;
$sleep = 1;
$attempts = 0;
do {
try {
$attempts++;
$resource = $app->getResource($resourceKey);
/* @var $database Database */
$database = is_callable($resource) ? $resource() : $resource;
break; // exit loop on success
} catch (\Exception $e) {
Console::warning(" └── Database not ready. Retrying connection ({$attempts})...");
$pools->reclaim();
if ($attempts >= $max) {
throw new \Exception(' └── Failed to connect to database: ' . $e->getMessage());
}
sleep($sleep);
}
} while ($attempts < $max);
Console::success("[Setup] - $dbName database init started...");
// Attempt to create the database
try {
Console::info(" └── Creating database: $dbName...");
$database->create();
} catch (\Exception $e) {
Console::info(" └── Skip: metadata table already exists");
}
// Process collections
foreach ($collections as $key => $collection) {
if (($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
}
if (!$database->getCollection($key)->isEmpty()) {
continue;
}
Console::info(" └── Creating collection: {$collection['$id']}...");
$attributes = array_map(fn ($attr) => new Document([
'$id' => ID::custom($attr['$id']),
'type' => $attr['type'],
'size' => $attr['size'],
'required' => $attr['required'],
'signed' => $attr['signed'],
'array' => $attr['array'],
'filters' => $attr['filters'],
'default' => $attr['default'] ?? null,
'format' => $attr['format'] ?? ''
]), $collection['attributes']);
$indexes = array_map(fn ($index) => new Document([
'$id' => ID::custom($index['$id']),
'type' => $index['type'],
'attributes' => $index['attributes'],
'lengths' => $index['lengths'],
'orders' => $index['orders'],
]), $collection['indexes']);
$database->createCollection($key, $attributes, $indexes);
}
if ($extraSetup) {
$extraSetup($database);
}
}
$http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $register) {
$app = new App('UTC');
@@ -164,140 +236,75 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg
/** @var Group $pools */
App::setResource('pools', fn () => $pools);
// wait for database to be ready
$attempts = 0;
$max = 10;
$sleep = 1;
do {
try {
$attempts++;
$dbForPlatform = $app->getResource('dbForPlatform');
/** @var Utopia\Database\Database $dbForPlatform */
break; // leave the do-while if successful
} catch (\Throwable $e) {
Console::warning("Database not ready. Retrying connection ({$attempts})...");
if ($attempts >= $max) {
throw new \Exception('Failed to connect to database: ' . $e->getMessage());
}
sleep($sleep);
}
} while ($attempts < $max);
Console::success('[Setup] - Server database init started...');
try {
Console::success('[Setup] - Creating console database...');
$dbForPlatform->create();
} catch (Duplicate) {
Console::success('[Setup] - Skip: metadata table already exists');
}
if ($dbForPlatform->getCollection(Audit::COLLECTION)->isEmpty()) {
$audit = new Audit($dbForPlatform);
$audit->setup();
}
/** @var array $collections */
$collections = Config::getParam('collections', []);
$consoleCollections = $collections['console'];
foreach ($consoleCollections as $key => $collection) {
if (($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
}
if (!$dbForPlatform->getCollection($key)->isEmpty()) {
continue;
// create logs database first, `getLogsDB` is a callable.
createDatabase($app, 'getLogsDB', 'logs', $collections['logs'], $pools);
// create appwrite database, `dbForPlatform` is a direct access call.
createDatabase($app, 'dbForPlatform', 'appwrite', $collections['console'], $pools, function (Database $dbForPlatform) use ($collections) {
if ($dbForPlatform->getCollection(Audit::COLLECTION)->isEmpty()) {
$audit = new Audit($dbForPlatform);
$audit->setup();
}
Console::success('[Setup] - Creating console collection: ' . $collection['$id'] . '...');
if ($dbForPlatform->getDocument('buckets', 'default')->isEmpty() &&
!$dbForPlatform->exists($dbForPlatform->getDatabase(), 'bucket_1')) {
Console::info(" └── Creating default bucket...");
$dbForPlatform->createDocument('buckets', new Document([
'$id' => ID::custom('default'),
'$collection' => ID::custom('buckets'),
'name' => 'Default',
'maximumFileSize' => (int) System::getEnv('_APP_STORAGE_LIMIT', 0),
'allowedFileExtensions' => [],
'enabled' => true,
'compression' => 'gzip',
'encryption' => true,
'antivirus' => true,
'fileSecurity' => true,
'$permissions' => [
Permission::create(Role::any()),
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'search' => 'buckets Default',
]));
$attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']);
$indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']);
$bucket = $dbForPlatform->getDocument('buckets', 'default');
$dbForPlatform->createCollection($key, $attributes, $indexes);
}
if ($dbForPlatform->getDocument('buckets', 'default')->isEmpty() && !$dbForPlatform->exists($dbForPlatform->getDatabase(), 'bucket_1')) {
Console::success('[Setup] - Creating default bucket...');
$dbForPlatform->createDocument('buckets', new Document([
'$id' => ID::custom('default'),
'$collection' => ID::custom('buckets'),
'name' => 'Default',
'maximumFileSize' => (int) System::getEnv('_APP_STORAGE_LIMIT', 0), // 10MB
'allowedFileExtensions' => [],
'enabled' => true,
'compression' => 'gzip',
'encryption' => true,
'antivirus' => true,
'fileSecurity' => true,
'$permissions' => [
Permission::create(Role::any()),
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'search' => 'buckets Default',
]));
$bucket = $dbForPlatform->getDocument('buckets', 'default');
Console::success('[Setup] - Creating files collection for default bucket...');
$files = $collections['buckets']['files'] ?? [];
if (empty($files)) {
throw new Exception('Files collection is not configured.');
}
$attributes = \array_map(fn ($attribute) => new Document($attribute), $files['attributes']);
$indexes = \array_map(fn (array $index) => new Document($index), $files['indexes']);
$dbForPlatform->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes);
}
$projectCollections = $collections['projects'];
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
$sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', ''));
$sharedTablesV2 = \array_diff($sharedTables, $sharedTablesV1);
$cache = $app->getResource('cache');
foreach ($sharedTablesV2 as $hostname) {
$adapter = $pools
->get($hostname)
->pop()
->getResource();
$dbForProject = (new Database($adapter, $cache))
->setDatabase('appwrite')
->setSharedTables(true)
->setTenant(null)
->setNamespace(System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', ''));
try {
Console::success('[Setup] - Creating project database: ' . $hostname . '...');
$dbForProject->create();
} catch (Duplicate) {
Console::success('[Setup] - Skip: metadata table already exists');
}
foreach ($projectCollections as $key => $collection) {
if (($collection['$collection'] ?? '') !== Database::METADATA) {
continue;
}
if (!$dbForProject->getCollection($key)->isEmpty()) {
continue;
Console::info(" └── Creating files collection for default bucket...");
$files = $collections['buckets']['files'] ?? [];
if (empty($files)) {
throw new Exception('Files collection is not configured.');
}
$attributes = \array_map(fn ($attribute) => new Document($attribute), $collection['attributes']);
$indexes = \array_map(fn (array $index) => new Document($index), $collection['indexes']);
$attributes = array_map(fn ($attr) => new Document([
'$id' => ID::custom($attr['$id']),
'type' => $attr['type'],
'size' => $attr['size'],
'required' => $attr['required'],
'signed' => $attr['signed'],
'array' => $attr['array'],
'filters' => $attr['filters'],
'default' => $attr['default'] ?? null,
'format' => $attr['format'] ?? ''
]), $files['attributes']);
Console::success('[Setup] - Creating project collection: ' . $collection['$id'] . '...');
$indexes = array_map(fn ($index) => new Document([
'$id' => ID::custom($index['$id']),
'type' => $index['type'],
'attributes' => $index['attributes'],
'lengths' => $index['lengths'],
'orders' => $index['orders'],
]), $files['indexes']);
$dbForProject->createCollection($key, $attributes, $indexes);
$dbForPlatform->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes);
}
}
});
$pools->reclaim();
Console::success('[Setup] - Server database init completed...');
});
app/init.php
+97 -37
View File
@@ -32,7 +32,7 @@ use Appwrite\Event\Mail;
use Appwrite\Event\Messaging;
use Appwrite\Event\Migration;
use Appwrite\Event\Realtime;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Event\Webhook;
use Appwrite\Extend\Exception;
use Appwrite\Functions\Specification;
@@ -237,8 +237,6 @@ const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent';
const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed';
const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent';
const METRIC_WEBHOOK_ID_FAILED = '{webhookInternalId}.webhooks.events.failed';
const METRIC_AUTH_METHOD_PHONE = 'auth.method.phone';
const METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE = METRIC_AUTH_METHOD_PHONE . '.{countryCode}';
const METRIC_MESSAGES = 'messages';
@@ -269,6 +267,8 @@ const METRIC_FILES = 'files';
const METRIC_FILES_STORAGE = 'files.storage';
const METRIC_FILES_TRANSFORMATIONS = 'files.transformations';
const METRIC_BUCKET_ID_FILES_TRANSFORMATIONS = '{bucketInternalId}.files.transformations';
const METRIC_FILES_IMAGES_TRANSFORMED = 'files.imagesTransformed';
const METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED = '{bucketInternalId}.files.imagesTransformed';
const METRIC_BUCKET_ID_FILES = '{bucketInternalId}.files';
const METRIC_BUCKET_ID_FILES_STORAGE = '{bucketInternalId}.files.storage';
const METRIC_FUNCTIONS = 'functions';
@@ -301,6 +301,18 @@ const METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS = '{functionInternalId}.execution
const METRIC_NETWORK_REQUESTS = 'network.requests';
const METRIC_NETWORK_INBOUND = 'network.inbound';
const METRIC_NETWORK_OUTBOUND = 'network.outbound';
const METRIC_MAU = 'users.mau';
const METRIC_DAU = 'users.dau';
const METRIC_WAU = 'users.wau';
const METRIC_WEBHOOKS = 'webhooks';
const METRIC_PLATFORMS = 'platforms';
const METRIC_PROVIDERS = 'providers';
const METRIC_TOPICS = 'topics';
const METRIC_KEYS = 'keys';
const METRIC_RESOURCE_TYPE_ID_BUILDS = '{resourceType}.{resourceInternalId}.builds';
const METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE = '{resourceType}.{resourceInternalId}.builds.storage';
const METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments';
const METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE = '{resourceType}.{resourceInternalId}.deployments.storage';
// Resource types
@@ -1176,6 +1188,9 @@ App::setResource('queueForWebhooks', function (Queue\Publisher $publisher) {
App::setResource('queueForRealtime', function () {
return new Realtime();
}, []);
App::setResource('queueForStatsUsage', function (Queue\Publisher $publisher) {
return new StatsUsage($publisher);
}, ['publisher']);
App::setResource('queueForAudits', function (Queue\Publisher $publisher) {
return new Audit($publisher);
}, ['publisher']);
@@ -1191,46 +1206,58 @@ App::setResource('queueForCertificates', function (Queue\Publisher $publisher) {
App::setResource('queueForMigrations', function (Queue\Publisher $publisher) {
return new Migration($publisher);
}, ['publisher']);
App::setResource('platforms', function (Document $project, Document $console) {
return [
...$project->getAttribute('platforms', []),
...$console->getAttribute('platforms', []),
];
}, ['project', 'console']);
App::setResource('hostnames', function (array $platforms) {
// Allow environment configured hostnames
$hostnames = [];
App::setResource('clients', function ($request, $console, $project) {
$console->setAttribute('platforms', [ // Always allow current host
'$collection' => ID::custom('platforms'),
'name' => 'Current Host',
'type' => Origin::CLIENT_TYPE_WEB,
'hostname' => $request->getHostname(),
], Document::SET_TYPE_APPEND);
$hostnames = explode(',', System::getEnv('_APP_CONSOLE_HOSTNAMES', ''));
$validator = new Hostname();
foreach (explode(',', System::getEnv('_APP_CONSOLE_HOSTNAMES', '')) as $hostname) {
foreach ($hostnames as $hostname) {
$hostname = trim($hostname);
if ($validator->isValid($hostname)) {
$hostnames[] = $hostname;
if (!$validator->isValid($hostname)) {
continue;
}
$console->setAttribute('platforms', [
'$collection' => ID::custom('platforms'),
'type' => Origin::CLIENT_TYPE_WEB,
'name' => $hostname,
'hostname' => $hostname,
], Document::SET_TYPE_APPEND);
}
/**
* Get All verified client URLs for both console and current projects
* + Filter for duplicated entries
*/
$clientsConsole = \array_map(
fn ($node) => $node['hostname'],
\array_filter(
$console->getAttribute('platforms', []),
fn ($node) => (isset($node['type']) && ($node['type'] === Origin::CLIENT_TYPE_WEB) && !empty($node['hostname']))
)
);
$clients = $clientsConsole;
$platforms = $project->getAttribute('platforms', []);
foreach ($platforms as $node) {
if (
isset($node['type']) &&
($node['type'] === Origin::CLIENT_TYPE_WEB ||
$node['type'] === Origin::CLIENT_TYPE_FLUTTER_WEB) &&
!empty($node['hostname'])
) {
$clients[] = $node['hostname'];
}
}
// Add database configured hostnames
foreach ($platforms as $platform) {
if (!empty($platform['hostname']) && in_array($platform['type'], [
Origin::CLIENT_TYPE_WEB,
Origin::CLIENT_TYPE_FLUTTER_WEB,
])) {
$hostnames[] = $platform['hostname'];
}
}
return \array_unique($clients);
}, ['request', 'console', 'project']);
return \array_unique($hostnames);
}, ['platforms']);
App::setResource('schemes', function (array $platforms, Document $project) {
// Allow expo development scheme by default
$schemes = ['exp'];
// Allow `appwrite-callback-${projectId}` scheme by default
if (!empty($project) && $project->getId() !== 'console') {
$schemes[] = 'appwrite-callback-' . $project->getId();
}
return \array_unique($schemes);
}, ['platforms', 'project']);
App::setResource('user', function ($mode, $project, $console, $request, $response, $dbForProject, $dbForPlatform) {
/** @var Appwrite\Utopia\Request $request */
/** @var Appwrite\Utopia\Response $response */
@@ -1534,6 +1561,39 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform
};
}, ['pools', 'dbForPlatform', 'cache']);
App::setResource('getLogsDB', function (Group $pools, Cache $cache) {
$database = null;
return function (?Document $project = null) use ($pools, $cache, $database) {
if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') {
$database->setTenant($project->getInternalId());
return $database;
}
$dbAdapter = $pools
->get('logs')
->pop()
->getResource();
$database = new Database(
$dbAdapter,
$cache
);
$database
->setSharedTables(true)
->setNamespace('logsV1')
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
// set tenant
if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') {
$database->setTenant($project->getInternalId());
}
return $database;
};
}, ['pools', 'cache']);
App::setResource('cache', function (Group $pools) {
$list = Config::getParam('pools-cache', []);
$adapters = [];
app/views/install/compose.phtml
+65 -6
View File
@@ -653,10 +653,69 @@ $image = $this->getParam('image', '');
- _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
- _APP_MAINTENANCE_RETENTION_SCHEDULES
appwrite-worker-usage:
appwrite-task-stats-resources:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: worker-usage
container_name: appwrite-worker-usage
container_name: appwrite-task-stats-resources
entrypoint: stats-resources
<<: *x-logging
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_USAGE_STATS
- _APP_LOGGING_CONFIG
- _APP_DATABASE_SHARED_TABLES
- _APP_STATS_RESOURCES_INTERVAL
appwrite-worker-stats-resources:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: worker-stats-resources
container_name: appwrite-worker-stats-resources
<<: *x-logging
restart: unless-stopped
networks:
- appwrite
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_USAGE_STATS
- _APP_LOGGING_CONFIG
- _APP_STATS_RESOURCES_INTERVAL
appwrite-worker-stats-usage:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: worker-stats-usage
container_name: appwrite-worker-stats-usage
<<: *x-logging
restart: unless-stopped
networks:
@@ -681,11 +740,11 @@ $image = $this->getParam('image', '');
- _APP_LOGGING_CONFIG
- _APP_USAGE_AGGREGATION_INTERVAL
appwrite-worker-usage-dump:
appwrite-worker-stats-usage-dump:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: worker-usage-dump
entrypoint: worker-stats-usage-dump
<<: *x-logging
container_name: appwrite-worker-usage-dump
container_name: appwrite-worker-stats-usage-dump
restart: unless-stopped
networks:
- appwrite
app/worker.php
+45
View File
@@ -13,8 +13,12 @@ use Appwrite\Event\Func;
use Appwrite\Event\Mail;
use Appwrite\Event\Messaging;
use Appwrite\Event\Migration;
use Appwrite\Event\StatsUsage;
use Appwrite\Event\StatsUsageDump;
/** remove */
use Appwrite\Event\Usage;
use Appwrite\Event\UsageDump;
/** /remove */
use Appwrite\Platform\Appwrite;
use Swoole\Runtime;
use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis;
@@ -173,6 +177,39 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf
};
}, ['pools', 'dbForPlatform', 'cache']);
Server::setResource('getLogsDB', function (Group $pools, Cache $cache) {
$database = null;
return function (?Document $project = null) use ($pools, $cache, $database) {
if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') {
$database->setTenant($project->getInternalId());
return $database;
}
$dbAdapter = $pools
->get('logs')
->pop()
->getResource();
$database = new Database(
$dbAdapter,
$cache
);
$database
->setSharedTables(true)
->setNamespace('logsV1')
->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS)
->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES);
// set tenant
if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') {
$database->setTenant($project->getInternalId());
}
return $database;
};
}, ['pools', 'cache']);
Server::setResource('abuseRetention', function () {
return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400);
});
@@ -240,6 +277,14 @@ Server::setResource('queueForUsageDump', function (Publisher $publisher) {
return new UsageDump($publisher);
}, ['publisher']);
Server::setResource('queueForStatsUsage', function (Publisher $publisher) {
return new StatsUsage($publisher);
}, ['publisher']);
Server::setResource('queueForStatsUsageDump', function (Publisher $publisher) {
return new StatsUsageDump($publisher);
}, ['publisher']);
Server::setResource('queueForDatabase', function (Publisher $publisher) {
return new EventDatabase($publisher);
}, ['publisher']);
bin/stats-resources
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php stats-resources $@
bin/worker-stats-resources
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/worker.php stats-resources $@
bin/worker-stats-usage
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/worker.php stats-usage $@
bin/worker-stats-usage-dump
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/worker.php stats-usage-dump $@
composer.json
+3 -3
View File
@@ -45,13 +45,13 @@
"ext-sockets": "*",
"appwrite/php-runtimes": "0.16.*",
"appwrite/php-clamav": "2.0.*",
"utopia-php/abuse": "0.49.*",
"utopia-php/abuse": "0.50.*",
"utopia-php/analytics": "0.10.*",
"utopia-php/audit": "0.49.*",
"utopia-php/audit": "0.51.*",
"utopia-php/cache": "0.11.*",
"utopia-php/cli": "0.15.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.58.4",
"utopia-php/database": "0.59.0",
"utopia-php/domains": "0.5.*",
"utopia-php/dsn": "0.2.1",
"utopia-php/framework": "0.33.*",
composer.lock
Generated
+47 -47
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "232691925e05350c7a3831a4e43d79d1",
"content-hash": "ed36bf1392e79d1b1bb07fb2a81f03bf",
"packages": [
{
"name": "adhocore/jwt",
@@ -3377,16 +3377,16 @@
},
{
"name": "utopia-php/abuse",
"version": "0.49.0",
"version": "0.50.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/abuse.git",
"reference": "76612c274b895aa3d4d1fa27557a6402463eea99"
"reference": "3ff67819e9de61506c5ca070a70552f7ebe99f80"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/abuse/zipball/76612c274b895aa3d4d1fa27557a6402463eea99",
"reference": "76612c274b895aa3d4d1fa27557a6402463eea99",
"url": "https://api.github.com/repos/utopia-php/abuse/zipball/3ff67819e9de61506c5ca070a70552f7ebe99f80",
"reference": "3ff67819e9de61506c5ca070a70552f7ebe99f80",
"shasum": ""
},
"require": {
@@ -3394,7 +3394,7 @@
"ext-pdo": "*",
"ext-redis": "*",
"php": ">=8.0",
"utopia-php/database": "0.58.*"
"utopia-php/database": "0.59.*"
},
"require-dev": {
"laravel/pint": "1.*",
@@ -3422,9 +3422,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/abuse/issues",
"source": "https://github.com/utopia-php/abuse/tree/0.49.0"
"source": "https://github.com/utopia-php/abuse/tree/0.50.0"
},
"time": "2025-02-04T07:33:59+00:00"
"time": "2025-02-12T09:13:59+00:00"
},
{
"name": "utopia-php/analytics",
@@ -3474,21 +3474,21 @@
},
{
"name": "utopia-php/audit",
"version": "0.49.0",
"version": "0.51.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/audit.git",
"reference": "9d5c5e0cf0f6d9157b911fc3971da4331d71c96d"
"reference": "a5a4b73a57e27a0fac8025b1d6038e145a1ca04e"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/audit/zipball/9d5c5e0cf0f6d9157b911fc3971da4331d71c96d",
"reference": "9d5c5e0cf0f6d9157b911fc3971da4331d71c96d",
"url": "https://api.github.com/repos/utopia-php/audit/zipball/a5a4b73a57e27a0fac8025b1d6038e145a1ca04e",
"reference": "a5a4b73a57e27a0fac8025b1d6038e145a1ca04e",
"shasum": ""
},
"require": {
"php": ">=8.0",
"utopia-php/database": "0.58.*"
"utopia-php/database": "0.59.*"
},
"require-dev": {
"laravel/pint": "1.*",
@@ -3515,9 +3515,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/audit/issues",
"source": "https://github.com/utopia-php/audit/tree/0.49.0"
"source": "https://github.com/utopia-php/audit/tree/0.51.0"
},
"time": "2025-02-04T07:27:18+00:00"
"time": "2025-02-12T09:12:44+00:00"
},
{
"name": "utopia-php/cache",
@@ -3717,16 +3717,16 @@
},
{
"name": "utopia-php/database",
"version": "0.58.4",
"version": "0.59.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/database.git",
"reference": "ff3fd22e4fe757cc2a78f17169f6dcc45c96d0fe"
"reference": "0eed7f1ad3eb66ff4a7d73b68dd9d3e05089eb18"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/database/zipball/ff3fd22e4fe757cc2a78f17169f6dcc45c96d0fe",
"reference": "ff3fd22e4fe757cc2a78f17169f6dcc45c96d0fe",
"url": "https://api.github.com/repos/utopia-php/database/zipball/0eed7f1ad3eb66ff4a7d73b68dd9d3e05089eb18",
"reference": "0eed7f1ad3eb66ff4a7d73b68dd9d3e05089eb18",
"shasum": ""
},
"require": {
@@ -3767,9 +3767,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/database/issues",
"source": "https://github.com/utopia-php/database/tree/0.58.4"
"source": "https://github.com/utopia-php/database/tree/0.59.0"
},
"time": "2025-02-05T02:51:02+00:00"
"time": "2025-02-12T08:08:29+00:00"
},
{
"name": "utopia-php/domains",
@@ -4170,16 +4170,16 @@
},
{
"name": "utopia-php/migration",
"version": "0.6.17",
"version": "0.6.19",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/migration.git",
"reference": "677a5c4688d7f54d1631a91f76a35d51346cf96b"
"reference": "3c9497f7a54ef88b1077c48d8326893133ad78eb"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/677a5c4688d7f54d1631a91f76a35d51346cf96b",
"reference": "677a5c4688d7f54d1631a91f76a35d51346cf96b",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/3c9497f7a54ef88b1077c48d8326893133ad78eb",
"reference": "3c9497f7a54ef88b1077c48d8326893133ad78eb",
"shasum": ""
},
"require": {
@@ -4187,7 +4187,7 @@
"ext-curl": "*",
"ext-openssl": "*",
"php": ">=8.1",
"utopia-php/database": "0.58.*",
"utopia-php/database": "0.59.*",
"utopia-php/dsn": "0.2.*",
"utopia-php/framework": "0.33.*",
"utopia-php/storage": "0.18.*"
@@ -4220,9 +4220,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/migration/issues",
"source": "https://github.com/utopia-php/migration/tree/0.6.17"
"source": "https://github.com/utopia-php/migration/tree/0.6.19"
},
"time": "2025-02-05T05:27:29+00:00"
"time": "2025-02-13T07:50:21+00:00"
},
{
"name": "utopia-php/mongo",
@@ -4490,16 +4490,16 @@
},
{
"name": "utopia-php/queue",
"version": "0.8.2",
"version": "0.8.6",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/queue.git",
"reference": "a6ec26a787e8292ca2d7b8f5a0ad179b46b2c4d0"
"reference": "b713b997285c29d120bbcbe3d6e93762d850f87c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/queue/zipball/a6ec26a787e8292ca2d7b8f5a0ad179b46b2c4d0",
"reference": "a6ec26a787e8292ca2d7b8f5a0ad179b46b2c4d0",
"url": "https://api.github.com/repos/utopia-php/queue/zipball/b713b997285c29d120bbcbe3d6e93762d850f87c",
"reference": "b713b997285c29d120bbcbe3d6e93762d850f87c",
"shasum": ""
},
"require": {
@@ -4549,9 +4549,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/queue/issues",
"source": "https://github.com/utopia-php/queue/tree/0.8.2"
"source": "https://github.com/utopia-php/queue/tree/0.8.6"
},
"time": "2025-02-06T11:01:15+00:00"
"time": "2025-02-10T03:35:00+00:00"
},
{
"name": "utopia-php/registry",
@@ -4607,16 +4607,16 @@
},
{
"name": "utopia-php/storage",
"version": "0.18.8",
"version": "0.18.9",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/storage.git",
"reference": "84737afa634e6a833fc4f8b0c967553234d3f215"
"reference": "1cf455404e8700b3093fd73d74a38d41cdced90c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/storage/zipball/84737afa634e6a833fc4f8b0c967553234d3f215",
"reference": "84737afa634e6a833fc4f8b0c967553234d3f215",
"url": "https://api.github.com/repos/utopia-php/storage/zipball/1cf455404e8700b3093fd73d74a38d41cdced90c",
"reference": "1cf455404e8700b3093fd73d74a38d41cdced90c",
"shasum": ""
},
"require": {
@@ -4656,9 +4656,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/storage/issues",
"source": "https://github.com/utopia-php/storage/tree/0.18.8"
"source": "https://github.com/utopia-php/storage/tree/0.18.9"
},
"time": "2024-12-04T08:30:35+00:00"
"time": "2025-02-11T13:10:40+00:00"
},
{
"name": "utopia-php/swoole",
@@ -5560,16 +5560,16 @@
},
{
"name": "myclabs/deep-copy",
"version": "1.12.1",
"version": "1.13.0",
"source": {
"type": "git",
"url": "https://github.com/myclabs/DeepCopy.git",
"reference": "123267b2c49fbf30d78a7b2d333f6be754b94845"
"reference": "024473a478be9df5fdaca2c793f2232fe788e414"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/123267b2c49fbf30d78a7b2d333f6be754b94845",
"reference": "123267b2c49fbf30d78a7b2d333f6be754b94845",
"url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/024473a478be9df5fdaca2c793f2232fe788e414",
"reference": "024473a478be9df5fdaca2c793f2232fe788e414",
"shasum": ""
},
"require": {
@@ -5608,7 +5608,7 @@
],
"support": {
"issues": "https://github.com/myclabs/DeepCopy/issues",
"source": "https://github.com/myclabs/DeepCopy/tree/1.12.1"
"source": "https://github.com/myclabs/DeepCopy/tree/1.13.0"
},
"funding": [
{
@@ -5616,7 +5616,7 @@
"type": "tidelift"
}
],
"time": "2024-11-08T17:47:46+00:00"
"time": "2025-02-12T12:17:51+00:00"
},
{
"name": "nikic/php-parser",
@@ -8747,7 +8747,7 @@
],
"aliases": [],
"minimum-stability": "stable",
"stability-flags": [],
"stability-flags": {},
"prefer-stable": false,
"prefer-lowest": false,
"platform": {
docker-compose.yml
+70 -7
View File
@@ -726,10 +726,72 @@ services:
- _APP_MAINTENANCE_DELAY
- _APP_DATABASE_SHARED_TABLES
appwrite-worker-usage:
entrypoint: worker-usage
appwrite-task-stats-resources:
container_name: appwrite-task-stats-resources
entrypoint: stats-resources
<<: *x-logging
container_name: appwrite-worker-usage
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_USAGE_STATS
- _APP_LOGGING_CONFIG
- _APP_DATABASE_SHARED_TABLES
- _APP_STATS_RESOURCES_INTERVAL
appwrite-worker-stats-resources:
entrypoint: worker-stats-resources
<<: *x-logging
container_name: appwrite-worker-stats-resources
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_USAGE_STATS
- _APP_LOGGING_CONFIG
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_DATABASE_SHARED_TABLES
appwrite-worker-stats-usage:
entrypoint: worker-stats-usage
<<: *x-logging
container_name: appwrite-worker-stats-usage
image: appwrite-dev
networks:
- appwrite
@@ -757,10 +819,10 @@ services:
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_DATABASE_SHARED_TABLES
appwrite-worker-usage-dump:
entrypoint: worker-usage-dump
appwrite-worker-stats-usage-dump:
entrypoint: worker-stats-usage-dump
<<: *x-logging
container_name: appwrite-worker-usage-dump
container_name: appwrite-worker-stats-usage-dump
image: appwrite-dev
networks:
- appwrite
@@ -787,7 +849,8 @@ services:
- _APP_LOGGING_CONFIG
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_DATABASE_SHARED_TABLES
- _APP_STATS_USAGE_DUAL_WRITING_DBS
appwrite-task-scheduler-functions:
entrypoint: schedule-functions
<<: *x-logging
docs/references/health/get-queue-stats-resources.md
+1
View File
@@ -0,0 +1 @@
Get the number of metrics that are waiting to be processed in the Appwrite stats resources queue.
docs/references/health/get-queue-usage-dump.md → docs/references/health/get-queue-stats-usage-dump.md
View File
docs/references/health/get-queue-usage.md → docs/references/health/get-queue-stats-usage.md
View File
src/Appwrite/Event/Event.php
+11
View File
@@ -24,11 +24,22 @@ class Event
public const FUNCTIONS_QUEUE_NAME = 'v1-functions';
public const FUNCTIONS_CLASS_NAME = 'FunctionsV1';
/** remove */
public const USAGE_QUEUE_NAME = 'v1-usage';
public const USAGE_CLASS_NAME = 'UsageV1';
public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump';
public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1';
/** /remove */
public const STATS_RESOURCES_QUEUE_NAME = 'v1-stats-resources';
public const STATS_RESOURCES_CLASS_NAME = 'StatsResourcesV1';
public const STATS_USAGE_QUEUE_NAME = 'v1-stats-usage';
public const STATS_USAGE_CLASS_NAME = 'StatsUsageV1';
public const STATS_USAGE_DUMP_QUEUE_NAME = 'v1-stats-usage-dump';
public const STATS_USAGE_DUMP_CLASS_NAME = 'StatsUsageDumpV1';
public const WEBHOOK_QUEUE_NAME = 'v1-webhooks';
public const WEBHOOK_CLASS_NAME = 'WebhooksV1';
src/Appwrite/Event/StatsResources.php
+29
View File
@@ -0,0 +1,29 @@
<?php
namespace Appwrite\Event;
use Utopia\Queue\Publisher;
class StatsResources extends Event
{
public function __construct(protected Publisher $publisher)
{
parent::__construct($publisher);
$this
->setQueue(Event::STATS_RESOURCES_QUEUE_NAME)
->setClass(Event::STATS_RESOURCES_CLASS_NAME);
}
/**
* Prepare the payload for the usage event.
*
* @return array
*/
protected function preparePayload(): array
{
return [
'project' => $this->project
];
}
}
src/Appwrite/Event/StatsUsage.php
+65
View File
@@ -0,0 +1,65 @@
<?php
namespace Appwrite\Event;
use Utopia\Database\Document;
use Utopia\Queue\Publisher;
class StatsUsage extends Event
{
protected array $metrics = [];
protected array $reduce = [];
public function __construct(protected Publisher $publisher)
{
parent::__construct($publisher);
$this
->setQueue(Event::STATS_USAGE_QUEUE_NAME)
->setClass(Event::STATS_USAGE_CLASS_NAME);
}
/**
* Add reduce.
*
* @param Document $document
* @return self
*/
public function addReduce(Document $document): self
{
$this->reduce[] = $document;
return $this;
}
/**
* Add metric.
*
* @param string $key
* @param int $value
* @return self
*/
public function addMetric(string $key, int $value): self
{
$this->metrics[] = [
'key' => $key,
'value' => $value,
];
return $this;
}
/**
* Prepare the payload for the event
*
* @return array
*/
protected function preparePayload(): array
{
return [
'project' => $this->getProject(),
'reduce' => $this->reduce,
'metrics' => $this->metrics,
];
}
}
src/Appwrite/Event/StatsUsageDump.php
+44
View File
@@ -0,0 +1,44 @@
<?php
namespace Appwrite\Event;
use Utopia\Queue\Publisher;
class StatsUsageDump extends Event
{
protected array $stats;
public function __construct(protected Publisher $publisher)
{
parent::__construct($publisher);
$this
->setQueue(Event::STATS_USAGE_DUMP_QUEUE_NAME)
->setClass(Event::STATS_USAGE_DUMP_CLASS_NAME);
}
/**
* Add Stats.
*
* @param array $stats
* @return self
*/
public function setStats(array $stats): self
{
$this->stats = $stats;
return $this;
}
/**
* Prepare the payload for the usage dump event.
*
* @return array
*/
protected function preparePayload(): array
{
return [
'stats' => $this->stats,
];
}
}
src/Appwrite/GraphQL/Types/Mapper.php
-1
View File
@@ -266,7 +266,6 @@ class Mapper
case 'Appwrite\Utopia\Database\Validator\CustomId':
case 'Utopia\Validator\Domain':
case 'Appwrite\Network\Validator\Email':
case 'Appwrite\Network\Validator\Redirect':
case 'Appwrite\Event\Validator\Event':
case 'Appwrite\Event\Validator\FunctionEvent':
case 'Utopia\Validator\HexColor':
src/Appwrite/Network/Validator/Redirect.php
-86
View File
@@ -1,86 +0,0 @@
<?php
namespace Appwrite\Network\Validator;
use Utopia\Validator\Host;
/**
* Redirect
*
* Validate that URL has an allowed host for redirect
*
* @package Utopia\Validator
*/
class Redirect extends Host
{
protected array $schemes = [];
/**
* @param array<string> $hostnames Allow list of allowed hostnames
* @param array<string> $schemes Allow list of allowed schemes
*/
public function __construct(array $hostnames = [], array $schemes = [])
{
$this->schemes = $schemes;
parent::__construct($hostnames);
}
/**
* Get Description
*
* Returns validator description
*
* @return string
*/
public function getDescription(): string
{
$schemes = '';
if (!empty($this->schemes)) {
$schemes = "URL scheme must be one of the following: " . implode(", ", array_map(function ($scheme) {
return "`$scheme`://";
}, $this->schemes));
}
$hostnames = '';
if (!empty($this->hostnames)) {
$hostnames = "URL hostname must be one of the following: " . implode(", ", array_map(function ($hostname) {
return "http://`$hostname`";
}, $this->hostnames));
}
return $schemes . ($schemes && $hostnames ? " or " : "") . $hostnames;
}
/**
* Is valid
*
* Validation will pass when $value is a valid URL and the host is allowed
*
* @param mixed $value
* @return bool
*/
public function isValid($value): bool
{
if (empty($value) || !\is_string($value)) {
return false;
}
// Then check for scheme
$scheme = '';
if (preg_match('/^([a-z][a-z0-9+\.-]*):\/+/i', $value, $matches)) {
$scheme = strtolower($matches[1]);
}
// These are dangerous schemes, may expose XSS vulnerabilities
if (in_array($scheme, ["javascript", "data", "blob", "file"])) {
return false;
}
// When the scheme is in the allowed list, the URL is valid.
if (!empty($this->schemes) && in_array($scheme, $this->schemes)) {
return true;
}
return parent::isValid($value);
}
}
src/Appwrite/Platform/Action.php
+90
View File
@@ -0,0 +1,90 @@
<?php
namespace Appwrite\Platform;
use Swoole\Coroutine as Co;
use Utopia\Database\Database;
use Utopia\Database\Query;
use Utopia\Platform\Action as UtopiaAction;
class Action extends UtopiaAction
{
/**
* Log Error Callback
*
* @var callable
*/
protected mixed $logError;
/**
* Foreach Document
* Call provided callback for each document in the collection
*
* @param string $projectId
* @param string $collection
* @param array $queries
* @param callable $callback
*
* @return void
*/
protected function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null, int $limit = 1000, bool $concurrent = false): void
{
$results = [];
$sum = $limit;
$latestDocument = null;
while ($sum === $limit) {
$newQueries = $queries;
try {
if ($latestDocument !== null) {
array_unshift($newQueries, Query::cursorAfter($latestDocument));
}
$newQueries[] = Query::limit($limit);
$database->disableValidation();
$results = $database->find($collection, $newQueries);
$database->enableValidation();
} catch (\Exception $e) {
if (!empty($this->logError)) {
call_user_func_array($this->logError, [$e, "CLI", "fetch_documents_namespace_{$database->getNamespace()}_collection{$collection}"]);
}
}
if (empty($results)) {
return;
}
$sum = count($results);
if ($concurrent) {
$callables = [];
$errors = [];
foreach ($results as $document) {
if (is_callable($callback)) {
$callables[] = Co\go(function () use ($document, $callback, &$errors) {
try {
$callback($document);
} catch (\Throwable $error) {
$errors[] = $error;
}
});
}
}
Co::join($callables);
if (!empty($errors)) {
throw new \Error("Errors found in concurrent foreachDocument: " . \json_encode($errors));
}
} else {
foreach ($results as $document) {
if (is_callable($callback)) {
$callback($document);
}
}
}
$latestDocument = $results[array_key_last($results)];
}
}
}
src/Appwrite/Platform/Services/Tasks.php
+2
View File
@@ -13,6 +13,7 @@ use Appwrite\Platform\Tasks\ScheduleMessages;
use Appwrite\Platform\Tasks\SDKs;
use Appwrite\Platform\Tasks\Specs;
use Appwrite\Platform\Tasks\SSL;
use Appwrite\Platform\Tasks\StatsResources;
use Appwrite\Platform\Tasks\Upgrade;
use Appwrite\Platform\Tasks\Vars;
use Appwrite\Platform\Tasks\Version;
@@ -38,6 +39,7 @@ class Tasks extends Service
->addAction(Upgrade::getName(), new Upgrade())
->addAction(Vars::getName(), new Vars())
->addAction(Version::getName(), new Version())
->addAction(StatsResources::getName(), new StatsResources())
;
}
}
src/Appwrite/Platform/Services/Workers.php
+11 -2
View File
@@ -11,8 +11,13 @@ use Appwrite\Platform\Workers\Functions;
use Appwrite\Platform\Workers\Mails;
use Appwrite\Platform\Workers\Messaging;
use Appwrite\Platform\Workers\Migrations;
use Appwrite\Platform\Workers\StatsResources;
use Appwrite\Platform\Workers\StatsUsage;
use Appwrite\Platform\Workers\StatsUsageDump;
/** remove */
use Appwrite\Platform\Workers\Usage;
use Appwrite\Platform\Workers\UsageDump;
/** /remove */
use Appwrite\Platform\Workers\Webhooks;
use Utopia\Platform\Service;
@@ -31,10 +36,14 @@ class Workers extends Service
->addAction(Mails::getName(), new Mails())
->addAction(Messaging::getName(), new Messaging())
->addAction(Webhooks::getName(), new Webhooks())
->addAction(StatsUsageDump::getName(), new StatsUsageDump())
->addAction(StatsUsage::getName(), new StatsUsage())
->addAction(Migrations::getName(), new Migrations())
->addAction(StatsResources::getName(), new StatsResources())
/** Remove */
->addAction(UsageDump::getName(), new UsageDump())
->addAction(Usage::getName(), new Usage())
->addAction(Migrations::getName(), new Migrations())
/** /remove */
;
}
}
src/Appwrite/Platform/Tasks/StatsResources.php
+81
View File
@@ -0,0 +1,81 @@
<?php
namespace Appwrite\Platform\Tasks;
use Appwrite\Event\StatsResources as EventStatsResources;
use Appwrite\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\System\System;
/**
* Usage count
*
* Runs every hour, schedules project
* for aggregating resource count
*/
class StatsResources extends Action
{
/**
* Log Error Callback
*
* @var callable
*/
protected mixed $logError;
/**
* Console DB
*
* @var Database
*/
protected Database $dbForPlatform;
public static function getName()
{
return 'stats-resources';
}
public function __construct()
{
$this
->desc('Schedules projects for usage count')
->inject('dbForPlatform')
->inject('logError')
->inject('queueForStatsResources')
->callback([$this, 'action']);
}
public function action(Database $dbForPlatform, callable $logError, EventStatsResources $queue): void
{
$this->logError = $logError;
$this->dbForPlatform = $dbForPlatform;
Console::title("Stats resources V1");
Console::success('Stats resources: started');
$interval = (int) System::getEnv('_APP_STATS_RESOURCES_INTERVAL', '3600');
Console::loop(function () use ($queue) {
Authorization::disable();
Authorization::setDefaultStatus(false);
$last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours'));
/**
* For each project that were accessed in last 24 hours
*/
$this->foreachDocument($this->dbForPlatform, 'projects', [
Query::greaterThanEqual('accessedAt', DateTime::format($last24Hours))
], function ($project) use ($queue) {
$queue
->setProject($project)
->trigger();
Console::success('project: ' . $project->getId() . '(' . $project->getInternalId() . ')' . ' queued');
});
}, $interval);
Console::log("Stats resources: exited");
}
}
src/Appwrite/Platform/Workers/Audits.php
+59 -13
View File
@@ -6,15 +6,33 @@ use Appwrite\Auth\Auth;
use Exception;
use Throwable;
use Utopia\Audit\Audit;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception\Authorization;
use Utopia\Database\Exception\Structure;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\System\System;
class Audits extends Action
{
private const BATCH_SIZE_DEVELOPMENT = 1; // smaller batch size for development
private const BATCH_SIZE_PRODUCTION = 5_000;
private const BATCH_AGGREGATION_INTERVAL = 60; // in seconds
private int $lastTriggeredTime = 0;
private array $logs = [];
private function getBatchSize(): int
{
return System::getEnv('_APP_ENV', 'development') === 'development'
? self::BATCH_SIZE_DEVELOPMENT
: self::BATCH_SIZE_PRODUCTION;
}
public static function getName(): string
{
return 'audits';
@@ -30,6 +48,8 @@ class Audits extends Action
->inject('message')
->inject('dbForProject')
->callback(fn ($message, $dbForProject) => $this->action($message, $dbForProject));
$this->lastTriggeredTime = time();
}
@@ -44,13 +64,14 @@ class Audits extends Action
*/
public function action(Message $message, Database $dbForProject): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
Console::info('Aggregating audit logs');
$event = $payload['event'] ?? '';
$auditPayload = $payload['payload'] ?? '';
$mode = $payload['mode'] ?? '';
@@ -63,23 +84,48 @@ class Audits extends Action
$userEmail = $user->getAttribute('email', '');
$userType = $user->getAttribute('type', Auth::ACTIVITY_TYPE_USER);
$audit = new Audit($dbForProject);
$audit->log(
userId: $user->getInternalId(),
// Pass first, most verbose event pattern
event: $event,
resource: $resource,
userAgent: $userAgent,
ip: $ip,
location: '',
data: [
// Create event data
$eventData = [
'userId' => $user->getInternalId(),
'event' => $event,
'resource' => $resource,
'userAgent' => $userAgent,
'ip' => $ip,
'location' => '',
'data' => [
'userId' => $user->getId(),
'userName' => $userName,
'userEmail' => $userEmail,
'userType' => $userType,
'mode' => $mode,
'data' => $auditPayload,
]
);
],
'timestamp' => DateTime::formatTz(DateTime::now())
];
$this->logs[] = $eventData;
// Check if we should process the batch by checking both for the batch size and the elapsed time
$batchSize = $this->getBatchSize();
$shouldProcessBatch = count($this->logs) >= $batchSize;
if (!$shouldProcessBatch && count($this->logs) > 0) {
$shouldProcessBatch = (time() - $this->lastTriggeredTime) >= self::BATCH_AGGREGATION_INTERVAL;
}
if ($shouldProcessBatch) {
Console::log('Processing batch with ' . count($this->logs) . ' events');
$audit = new Audit($dbForProject);
try {
$audit->logBatch($this->logs);
Console::success('Audit logs processed successfully');
} catch (Throwable $e) {
Console::error('Error processing audit logs: ' . $e->getMessage());
} finally {
// Clear the pending events after successful batch processing
$this->logs = [];
$this->lastTriggeredTime = time();
}
}
}
}
src/Appwrite/Platform/Workers/Builds.php
+21 -15
View File
@@ -5,7 +5,7 @@ namespace Appwrite\Platform\Workers;
use Ahc\Jwt\JWT;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Utopia\Response\Model\Deployment;
use Appwrite\Vcs\Comment;
@@ -50,12 +50,14 @@ class Builds extends Action
->inject('dbForPlatform')
->inject('queueForEvents')
->inject('queueForFunctions')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('cache')
->inject('dbForProject')
->inject('deviceForFunctions')
->inject('isResourceBlocked')
->inject('log')
->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log));
->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log) =>
$this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $log));
}
/**
@@ -64,7 +66,7 @@ class Builds extends Action
* @param Database $dbForPlatform
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @param Usage $queueForUsage
* @param StatsUsage $queueForStatsUsage
* @param Cache $cache
* @param Database $dbForProject
* @param Device $deviceForFunctions
@@ -72,7 +74,7 @@ class Builds extends Action
* @return void
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void
public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Log $log): void
{
$payload = $message->getPayload() ?? [];
@@ -93,7 +95,7 @@ class Builds extends Action
case BUILD_TYPE_RETRY:
Console::info('Creating build for deployment: ' . $deployment->getId());
$github = new GitHub($cache);
$this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $log);
$this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log);
break;
default:
@@ -105,7 +107,7 @@ class Builds extends Action
* @param Device $deviceForFunctions
* @param Func $queueForFunctions
* @param Event $queueForEvents
* @param Usage $queueForUsage
* @param StatsUsage $queueForStatsUsage
* @param Database $dbForPlatform
* @param Database $dbForProject
* @param GitHub $github
@@ -118,7 +120,7 @@ class Builds extends Action
* @throws \Utopia\Database\Exception
* @throws Exception
*/
protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, Log $log): void
protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void
{
$executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST'));
@@ -127,7 +129,11 @@ class Builds extends Action
$function = $dbForProject->getDocument('functions', $functionId);
if ($function->isEmpty()) {
throw new \Exception('Function not found', 404);
throw new \Exception('Function not found');
}
if ($isResourceBlocked($project, RESOURCE_TYPE_FUNCTIONS, $functionId)) {
throw new \Exception('Function blocked');
}
$deploymentId = $deployment->getId();
@@ -135,11 +141,11 @@ class Builds extends Action
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
if ($deployment->isEmpty()) {
throw new \Exception('Deployment not found', 404);
throw new \Exception('Deployment not found');
}
if (empty($deployment->getAttribute('entrypoint', ''))) {
throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 500);
throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".');
}
$version = $function->getAttribute('version', 'v2');
@@ -571,7 +577,7 @@ class Builds extends Action
$build = $dbForProject->getDocument('builds', $build->getId());
if ($build->isEmpty()) {
throw new \Exception('Build not found', 404);
throw new \Exception('Build not found');
}
if ($build->getAttribute('status') === 'canceled') {
@@ -706,20 +712,20 @@ class Builds extends Action
/** Trigger usage queue */
if ($build->getAttribute('status') === 'ready') {
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project
->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS), 1) // per function
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$build->getAttribute('duration', 0) * 1000);
} elseif ($build->getAttribute('status') === 'failed') {
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_BUILDS_FAILED, 1) // per project
->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED), 1) // per function
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED), (int)$build->getAttribute('duration', 0) * 1000);
}
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_BUILDS, 1) // per project
->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
src/Appwrite/Platform/Workers/Deletes.php
+3 -1
View File
@@ -7,6 +7,7 @@ use Appwrite\Certificates\Adapter as CertificatesAdapter;
use Appwrite\Extend\Exception;
use Executor\Executor;
use Throwable;
use Utopia\Abuse\Adapters\TimeLimit\Database as AbuseDatabase;
use Utopia\Audit\Audit;
use Utopia\Cache\Adapter\Filesystem;
use Utopia\Cache\Cache;
@@ -505,7 +506,8 @@ class Deletes extends Action
$projectCollectionIds = [
...\array_keys(Config::getParam('collections', [])['projects']),
Audit::COLLECTION
Audit::COLLECTION,
AbuseDatabase::COLLECTION,
];
$limit = \count($projectCollectionIds) + 25;
src/Appwrite/Platform/Workers/Functions.php
+10 -10
View File
@@ -5,7 +5,7 @@ namespace Appwrite\Platform\Workers;
use Ahc\Jwt\JWT;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Utopia\Response\Model\Execution;
use Exception;
@@ -46,13 +46,13 @@ class Functions extends Action
->inject('dbForProject')
->inject('queueForFunctions')
->inject('queueForEvents')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('log')
->inject('isResourceBlocked')
->callback(fn (Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForUsage, $log, $isResourceBlocked));
->callback(fn (Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked) => $this->action($project, $message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $log, $isResourceBlocked));
}
public function action(Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log, callable $isResourceBlocked): void
public function action(Document $project, Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log, callable $isResourceBlocked): void
{
$payload = $message->getPayload() ?? [];
@@ -137,7 +137,7 @@ class Functions extends Action
log: $log,
dbForProject: $dbForProject,
queueForFunctions: $queueForFunctions,
queueForUsage: $queueForUsage,
queueForStatsUsage: $queueForStatsUsage,
queueForEvents: $queueForEvents,
project: $project,
function: $function,
@@ -177,7 +177,7 @@ class Functions extends Action
log: $log,
dbForProject: $dbForProject,
queueForFunctions: $queueForFunctions,
queueForUsage: $queueForUsage,
queueForStatsUsage: $queueForStatsUsage,
queueForEvents: $queueForEvents,
project: $project,
function: $function,
@@ -199,7 +199,7 @@ class Functions extends Action
log: $log,
dbForProject: $dbForProject,
queueForFunctions: $queueForFunctions,
queueForUsage: $queueForUsage,
queueForStatsUsage: $queueForStatsUsage,
queueForEvents: $queueForEvents,
project: $project,
function: $function,
@@ -284,7 +284,7 @@ class Functions extends Action
* @param Log $log
* @param Database $dbForProject
* @param Func $queueForFunctions
* @param Usage $queueForUsage
* @param StatsUsage $queueForStatsUsage
* @param Event $queueForEvents
* @param Document $project
* @param Document $function
@@ -308,7 +308,7 @@ class Functions extends Action
Log $log,
Database $dbForProject,
Func $queueForFunctions,
Usage $queueForUsage,
StatsUsage $queueForStatsUsage,
Event $queueForEvents,
Document $project,
Document $function,
@@ -552,7 +552,7 @@ class Functions extends Action
$errorCode = $th->getCode();
} finally {
/** Trigger usage queue */
$queueForUsage
$queueForStatsUsage
->setProject($project)
->addMetric(METRIC_EXECUTIONS, 1)
->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
src/Appwrite/Platform/Workers/Messaging.php
+12 -12
View File
@@ -2,7 +2,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Messaging\Status as MessageStatus;
use Swoole\Runtime;
use Utopia\CLI\Console;
@@ -70,8 +70,8 @@ class Messaging extends Action
->inject('log')
->inject('dbForProject')
->inject('deviceForFiles')
->inject('queueForUsage')
->callback(fn (Message $message, Document $project, Log $log, Database $dbForProject, Device $deviceForFiles, Usage $queueForUsage) => $this->action($message, $project, $log, $dbForProject, $deviceForFiles, $queueForUsage));
->inject('queueForStatsUsage')
->callback(fn (Message $message, Document $project, Log $log, Database $dbForProject, Device $deviceForFiles, StatsUsage $queueForStatsUsage) => $this->action($message, $project, $log, $dbForProject, $deviceForFiles, $queueForStatsUsage));
}
/**
@@ -80,7 +80,7 @@ class Messaging extends Action
* @param Log $log
* @param Database $dbForProject
* @param Device $deviceForFiles
* @param Usage $queueForUsage
* @param StatsUsage $queueForStatsUsage
* @return void
* @throws \Exception
*/
@@ -90,7 +90,7 @@ class Messaging extends Action
Log $log,
Database $dbForProject,
Device $deviceForFiles,
Usage $queueForUsage
StatsUsage $queueForStatsUsage
): void {
Runtime::setHookFlags(SWOOLE_HOOK_ALL ^ SWOOLE_HOOK_TCP);
$payload = $message->getPayload() ?? [];
@@ -111,7 +111,7 @@ class Messaging extends Action
case MESSAGE_SEND_TYPE_EXTERNAL:
$message = $dbForProject->getDocument('messages', $payload['messageId']);
$this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project, $queueForUsage);
$this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project, $queueForStatsUsage);
break;
default:
throw new \Exception('Unknown message type: ' . $type);
@@ -123,7 +123,7 @@ class Messaging extends Action
Document $message,
Device $deviceForFiles,
Document $project,
Usage $queueForUsage
StatsUsage $queueForStatsUsage
): void {
$topicIds = $message->getAttribute('topics', []);
$targetIds = $message->getAttribute('targets', []);
@@ -229,8 +229,8 @@ class Messaging extends Action
/**
* @var array<array> $results
*/
$results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
$results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) {
return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) {
if (\array_key_exists($providerId, $providers)) {
$provider = $providers[$providerId];
} else {
@@ -257,8 +257,8 @@ class Messaging extends Action
$adapter->getMaxMessagesPerRequest()
);
return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForUsage) {
return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) {
return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) {
$deliveredTotal = 0;
$deliveryErrors = [];
$messageData = clone $message;
@@ -298,7 +298,7 @@ class Messaging extends Action
$deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage();
} finally {
$errorTotal = \count($deliveryErrors);
$queueForUsage
$queueForStatsUsage
->setProject($project)
->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal))
->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal)
src/Appwrite/Platform/Workers/StatsResources.php
+383
View File
@@ -0,0 +1,383 @@
<?php
namespace Appwrite\Platform\Workers;
use Appwrite\Platform\Action;
use Exception;
use Throwable;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Queue\Message;
class StatsResources extends Action
{
/**
* Date format for different periods
*/
protected array $periods = [
'1h' => 'Y-m-d H:00',
'1d' => 'Y-m-d 00:00',
'inf' => '0000-00-00 00:00'
];
/**
* @var array $documents
*
* Array of documents to batch write
*
*/
protected array $documents = [];
public static function getName(): string
{
return 'stats-resources';
}
/**
* @throws Exception
*/
public function __construct()
{
$this
->desc('Stats resources worker')
->inject('message')
->inject('project')
->inject('getProjectDB')
->inject('getLogsDB')
->inject('dbForPlatform')
->inject('logError')
->callback([$this, 'action']);
}
/**
* @param Message $message
* @param Document $project
* @param callable $getProjectDB
* @return void
* @throws \Utopia\Database\Exception
* @throws Exception
*/
public function action(Message $message, Document $project, callable $getProjectDB, callable $getLogsDB, Database $dbForPlatform, callable $logError): void
{
$this->logError = $logError;
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
if (empty($project->getAttribute('database'))) {
var_dump($payload);
return;
}
// Reset documents for each job
$this->documents = [];
$this->countForProject($dbForPlatform, $getLogsDB, $getProjectDB, $project);
}
protected function countForProject(Database $dbForPlatform, callable $getLogsDB, callable $getProjectDB, Document $project): void
{
Console::info('Begining count for: ' . $project->getId());
$dbForLogs = null;
$dbForProject = null;
try {
/** @var \Utopia\Database\Database $dbForLogs */
$dbForLogs = call_user_func($getLogsDB, $project);
/** @var \Utopia\Database\Database $dbForProject */
$dbForProject = call_user_func($getProjectDB, $project);
} catch (Throwable $th) {
Console::error('Unable to get database');
Console::error($th->getMessage());
return;
}
try {
$region = $project->getAttribute('region');
$platforms = $dbForPlatform->count('platforms', [
Query::equal('projectInternalId', [$project->getInternalId()])
]);
$webhooks = $dbForPlatform->count('webhooks', [
Query::equal('projectInternalId', [$project->getInternalId()])
]);
$keys = $dbForPlatform->count('keys', [
Query::equal('projectInternalId', [$project->getInternalId()])
]);
$databases = $dbForProject->count('databases');
$buckets = $dbForProject->count('buckets');
$users = $dbForProject->count('users');
$last30Days = (new \DateTime())->sub(\DateInterval::createFromDateString('30 days'))->format('Y-m-d 00:00:00');
$usersMAU = $dbForProject->count('users', [
Query::greaterThanEqual('accessedAt', $last30Days)
]);
$last24Hours = (new \DateTime())->sub(\DateInterval::createFromDateString('24 hours'))->format('Y-m-d h:m:00');
$usersDAU = $dbForProject->count('users', [
Query::greaterThanEqual('accessedAt', $last24Hours)
]);
$last7Days = (new \DateTime())->sub(\DateInterval::createFromDateString('7 days'))->format('Y-m-d 00:00:00');
$usersWAU = $dbForProject->count('users', [
Query::greaterThanEqual('accessedAt', $last7Days)
]);
$teams = $dbForProject->count('teams');
$functions = $dbForProject->count('functions');
$messages = $dbForProject->count('messages');
$providers = $dbForProject->count('providers');
$topics = $dbForProject->count('topics');
$metrics = [
METRIC_DATABASES => $databases,
METRIC_BUCKETS => $buckets,
METRIC_USERS => $users,
METRIC_FUNCTIONS => $functions,
METRIC_TEAMS => $teams,
METRIC_MESSAGES => $messages,
METRIC_MAU => $usersMAU,
METRIC_DAU => $usersDAU,
METRIC_WAU => $usersWAU,
METRIC_WEBHOOKS => $webhooks,
METRIC_PLATFORMS => $platforms,
METRIC_PROVIDERS => $providers,
METRIC_TOPICS => $topics,
METRIC_KEYS => $keys,
];
foreach ($metrics as $metric => $value) {
$this->createStatsDocuments($region, $metric, $value);
}
try {
$this->countForBuckets($dbForProject, $dbForLogs, $region);
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]);
}
try {
$this->countImageTransformations($dbForProject, $dbForLogs, $region);
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_buckets_{$project->getId()}"]);
}
try {
$this->countForDatabase($dbForProject, $dbForLogs, $region);
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_database_{$project->getId()}"]);
}
try {
$this->countForFunctions($dbForProject, $dbForLogs, $region);
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_functions_{$project->getId()}"]);
}
$this->writeDocuments($dbForLogs, $project);
} catch (Throwable $th) {
call_user_func_array($this->logError, [$th, "StatsResources", "count_for_project_{$project->getId()}"]);
}
Console::info('End of count for: ' . $project->getId());
}
protected function countForBuckets(Database $dbForProject, Database $dbForLogs, string $region)
{
$totalFiles = 0;
$totalStorage = 0;
$this->foreachDocument($dbForProject, 'buckets', [], function ($bucket) use ($dbForProject, $dbForLogs, $region, &$totalFiles, &$totalStorage) {
$files = $dbForProject->count('bucket_' . $bucket->getInternalId());
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES);
$this->createStatsDocuments($region, $metric, $files);
$storage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeActual');
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE);
$this->createStatsDocuments($region, $metric, $storage);
$totalStorage += $storage;
$totalFiles += $files;
});
$this->createStatsDocuments($region, METRIC_FILES, $totalFiles);
$this->createStatsDocuments($region, METRIC_FILES_STORAGE, $totalStorage);
}
/**
* Need separate function to count per period data
*/
protected function countImageTransformations(Database $dbForProject, Database $dbForLogs, string $region)
{
$totalImageTransformations = 0;
$totalDailyImageTransformations = 0;
$totalHourlyImageTransformations = 0;
$this->foreachDocument($dbForProject, 'buckets', [], function ($bucket) use ($dbForProject, $dbForLogs, $region, &$totalDailyImageTransformations, &$totalHourlyImageTransformations, &$totalImageTransformations) {
$imageTransformations = $dbForProject->count('bucket_' . $bucket->getInternalId(), [
Query::isNotNull('transformedAt')
]);
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED);
$this->createStatsDocuments($region, $metric, $imageTransformations, 'inf');
$totalImageTransformations += $imageTransformations;
// hourly
$time = \date($this->periods['1h'], \time());
$start = $time;
$end = (new \DateTime($start))->format('Y-m-d H:59:59');
$hourlyImageTransformations = $dbForProject->count('bucket_' . $bucket->getInternalId(), [
Query::greaterThanEqual('transformedAt', $start),
Query::lessThanEqual('transformedAt', $end),
]);
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED);
$this->createStatsDocuments($region, $metric, $hourlyImageTransformations, '1h');
$totalHourlyImageTransformations += $hourlyImageTransformations;
// daily
$time = \date($this->periods['1d'], \time());
$start = $time;
$end = (new \DateTime($start))->format('Y-m-d 11:59:59');
$dailyImageTransformations = $dbForProject->count('bucket_' . $bucket->getInternalId(), [
Query::greaterThanEqual('transformedAt', $start),
Query::lessThanEqual('transformedAt', $end),
]);
$metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_IMAGES_TRANSFORMED);
$this->createStatsDocuments($region, $metric, $dailyImageTransformations, '1d');
$totalDailyImageTransformations += $dailyImageTransformations;
});
$this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalImageTransformations, 'inf');
$this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalDailyImageTransformations, '1d');
$this->createStatsDocuments($region, METRIC_FILES_IMAGES_TRANSFORMED, $totalHourlyImageTransformations, '1h');
}
protected function countForDatabase(Database $dbForProject, Database $dbForLogs, string $region)
{
$totalCollections = 0;
$totalDocuments = 0;
$this->foreachDocument($dbForProject, 'databases', [], function ($database) use ($dbForProject, $dbForLogs, $region, &$totalCollections, &$totalDocuments) {
$collections = $dbForProject->count('database_' . $database->getInternalId());
$metric = str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS);
$this->createStatsDocuments($region, $metric, $collections);
$documents = $this->countForCollections($dbForProject, $dbForLogs, $database, $region);
$totalDocuments += $documents;
$totalCollections += $collections;
});
$this->createStatsDocuments($region, METRIC_COLLECTIONS, $totalCollections);
$this->createStatsDocuments($region, METRIC_DOCUMENTS, $totalDocuments);
}
protected function countForCollections(Database $dbForProject, Database $dbForLogs, Document $database, string $region): int
{
$databaseDocuments = 0;
$this->foreachDocument($dbForProject, 'database_' . $database->getInternalId(), [], function ($collection) use ($dbForProject, $dbForLogs, $database, $region, &$totalCollections, &$databaseDocuments) {
$documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
$metric = str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS);
$this->createStatsDocuments($region, $metric, $documents);
$databaseDocuments += $documents;
});
$metric = str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_DOCUMENTS);
$this->createStatsDocuments($region, $metric, $databaseDocuments);
return $databaseDocuments;
}
protected function countForFunctions(Database $dbForProject, Database $dbForLogs, string $region)
{
$deploymentsStorage = $dbForProject->sum('deployments', 'size');
$buildsStorage = $dbForProject->sum('builds', 'size');
$this->createStatsDocuments($region, METRIC_DEPLOYMENTS_STORAGE, $deploymentsStorage);
$this->createStatsDocuments($region, METRIC_BUILDS_STORAGE, $buildsStorage);
$deployments = $dbForProject->count('deployments');
$builds = $dbForProject->count('builds');
$this->createStatsDocuments($region, METRIC_DEPLOYMENTS, $deployments);
$this->createStatsDocuments($region, METRIC_BUILDS, $builds);
$this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForLogs, $region) {
$functionDeploymentsStorage = $dbForProject->sum('deployments', 'size', [
Query::equal('resourceInternalId', [$function->getInternalId()]),
Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]),
]);
$this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS_STORAGE), $functionDeploymentsStorage);
$functionDeployments = $dbForProject->count('deployments', [
Query::equal('resourceInternalId', [$function->getInternalId()]),
Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]),
]);
$this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_DEPLOYMENTS), $functionDeployments);
/**
* As deployments and builds have 1-1 relationship,
* the count for one should match the other
*/
$this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS), $functionDeployments);
$functionBuildsStorage = 0;
$this->foreachDocument($dbForProject, 'deployments', [
Query::equal('resourceInternalId', [$function->getInternalId()]),
Query::equal('resourceType', [RESOURCE_TYPE_FUNCTIONS]),
], function (Document $deployment) use ($dbForProject, &$functionBuildsStorage): void {
$build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', ''));
$functionBuildsStorage += $build->getAttribute('size', 0);
});
$this->createStatsDocuments($region, str_replace(['{resourceType}','{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS,$function->getInternalId()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $functionBuildsStorage);
});
}
protected function createStatsDocuments(string $region, string $metric, int $value, ?string $period = null)
{
if ($period === null) {
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : \date($format, \time());
$id = \md5("{$time}_{$period}_{$metric}");
$this->documents[] = new Document([
'$id' => $id,
'metric' => $metric,
'period' => $period,
'region' => $region,
'value' => $value,
'time' => $time,
]);
}
} else {
$time = $period === 'inf' ? null : \date($this->periods[$period], \time());
$id = \md5("{$time}_{$period}_{$metric}");
$this->documents[] = new Document([
'$id' => $id,
'metric' => $metric,
'period' => $period,
'region' => $region,
'value' => $value,
'time' => $time,
]);
}
}
protected function writeDocuments(Database $dbForLogs, Document $project): void
{
$dbForLogs->createOrUpdateDocuments(
'stats',
$this->documents
);
$this->documents = [];
Console::success('Stats written to logs db for project: ' . $project->getId() . '(' . $project->getInternalId() . ')');
}
}
src/Appwrite/Platform/Workers/StatsUsage.php
+246
View File
@@ -0,0 +1,246 @@
<?php
namespace Appwrite\Platform\Workers;
use Appwrite\Event\StatsUsageDump;
use Exception;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\System\System;
class StatsUsage extends Action
{
private array $stats = [];
private int $lastTriggeredTime = 0;
private int $keys = 0;
private const INFINITY_PERIOD = '_inf_';
private const KEYS_THRESHOLD = 10000;
public static function getName(): string
{
return 'stats-usage';
}
/**
* @throws Exception
*/
public function __construct()
{
$this
->desc('Stats usage worker')
->inject('message')
->inject('getProjectDB')
->inject('queueForStatsUsageDump')
->callback([$this, 'action']);
$this->lastTriggeredTime = time();
}
/**
* @param Message $message
* @param callable $getProjectDB
* @param StatsUsageDump $queueForStatsUsageDump
* @return void
* @throws \Utopia\Database\Exception
* @throws Exception
*/
public function action(Message $message, callable $getProjectDB, StatsUsageDump $queueForStatsUsageDump): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
//Todo Figure out way to preserve keys when the container is being recreated @shimonewman
$aggregationInterval = (int) System::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20');
$project = new Document($payload['project'] ?? []);
$projectId = $project->getInternalId();
foreach ($payload['reduce'] ?? [] as $document) {
if (empty($document)) {
continue;
}
$this->reduce(
project: $project,
document: new Document($document),
metrics: $payload['metrics'],
getProjectDB: $getProjectDB
);
}
$this->stats[$projectId]['project'] = $project;
$this->stats[$projectId]['receivedAt'] = DateTime::now();
foreach ($payload['metrics'] ?? [] as $metric) {
$this->keys++;
if (!isset($this->stats[$projectId]['keys'][$metric['key']])) {
$this->stats[$projectId]['keys'][$metric['key']] = $metric['value'];
continue;
}
$this->stats[$projectId]['keys'][$metric['key']] += $metric['value'];
}
// If keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats)
if (
$this->keys >= self::KEYS_THRESHOLD ||
(time() - $this->lastTriggeredTime > $aggregationInterval && $this->keys > 0)
) {
Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');
$queueForStatsUsageDump
->setStats($this->stats)
->trigger();
$this->stats = [];
$this->keys = 0;
$this->lastTriggeredTime = time();
}
}
/**
* On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files.
* When we remove a parent document we need to deduct his children aggregation from the project scope.
* @param Document $project
* @param Document $document
* @param array $metrics
* @param callable $getProjectDB
* @return void
*/
private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void
{
$dbForProject = $getProjectDB($project);
try {
switch (true) {
case $document->getCollection() === 'users': // users
$sessions = count($document->getAttribute(METRIC_SESSIONS, 0));
if (!empty($sessions)) {
$metrics[] = [
'key' => METRIC_SESSIONS,
'value' => ($sessions * -1),
];
}
break;
case $document->getCollection() === 'databases': // databases
$collections = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
if (!empty($collections['value'])) {
$metrics[] = [
'key' => METRIC_COLLECTIONS,
'value' => ($collections['value'] * -1),
];
}
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
}
break;
case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
$parts = explode('_', $document->getCollection());
$databaseInternalId = $parts[1] ?? 0;
$documents = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
if (!empty($documents['value'])) {
$metrics[] = [
'key' => METRIC_DOCUMENTS,
'value' => ($documents['value'] * -1),
];
$metrics[] = [
'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS),
'value' => ($documents['value'] * -1),
];
}
break;
case $document->getCollection() === 'buckets':
$files = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
$storage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
if (!empty($files['value'])) {
$metrics[] = [
'key' => METRIC_FILES,
'value' => ($files['value'] * -1),
];
}
if (!empty($storage['value'])) {
$metrics[] = [
'key' => METRIC_FILES_STORAGE,
'value' => ($storage['value'] * -1),
];
}
break;
case $document->getCollection() === 'functions':
$deployments = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
$deploymentsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
$builds = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
$buildsStorage = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
$buildsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
$executions = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
$executionsCompute = $dbForProject->getDocument('stats', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
if (!empty($deployments['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS,
'value' => ($deployments['value'] * -1),
];
}
if (!empty($deploymentsStorage['value'])) {
$metrics[] = [
'key' => METRIC_DEPLOYMENTS_STORAGE,
'value' => ($deploymentsStorage['value'] * -1),
];
}
if (!empty($builds['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS,
'value' => ($builds['value'] * -1),
];
}
if (!empty($buildsStorage['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_STORAGE,
'value' => ($buildsStorage['value'] * -1),
];
}
if (!empty($buildsCompute['value'])) {
$metrics[] = [
'key' => METRIC_BUILDS_COMPUTE,
'value' => ($buildsCompute['value'] * -1),
];
}
if (!empty($executions['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS,
'value' => ($executions['value'] * -1),
];
}
if (!empty($executionsCompute['value'])) {
$metrics[] = [
'key' => METRIC_EXECUTIONS_COMPUTE,
'value' => ($executionsCompute['value'] * -1),
];
}
break;
default:
break;
}
} catch (\Throwable $e) {
console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}");
}
}
}
src/Appwrite/Platform/Workers/StatsUsageDump.php
+350
View File
@@ -0,0 +1,350 @@
<?php
namespace Appwrite\Platform\Workers;
use Appwrite\Extend\Exception;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\Registry\Registry;
use Utopia\System\System;
class StatsUsageDump extends Action
{
public const METRIC_COLLECTION_LEVEL_STORAGE = 4;
public const METRIC_DATABASE_LEVEL_STORAGE = 3;
public const METRIC_PROJECT_LEVEL_STORAGE = 2;
protected array $stats = [];
protected Registry $register;
/**
* Metrics to skip writing to logsDB
* As these metrics are calculated separately
* by logs DB
* @var array
*/
protected array $skipBaseMetrics = [
METRIC_DATABASES => true,
METRIC_BUCKETS => true,
METRIC_USERS => true,
METRIC_FUNCTIONS => true,
METRIC_TEAMS => true,
METRIC_MESSAGES => true,
METRIC_MAU => true,
METRIC_WEBHOOKS => true,
METRIC_PLATFORMS => true,
METRIC_PROVIDERS => true,
METRIC_TOPICS => true,
METRIC_KEYS => true,
METRIC_FILES => true,
METRIC_FILES_STORAGE => true,
METRIC_DEPLOYMENTS_STORAGE => true,
METRIC_BUILDS_STORAGE => true,
METRIC_DEPLOYMENTS => true,
METRIC_BUILDS => true,
METRIC_COLLECTIONS => true,
METRIC_DOCUMENTS => true,
];
/**
* Skip metrics associated with parent IDs
* these need to be checked individually with `str_ends_with`
*/
protected array $skipParentIdMetrics = [
'.files',
'.files.storage',
'.collections',
'.documents',
'.deployments',
'.deployments.storage',
'.builds',
'.builds.storage',
];
/**
* @var callable
*/
protected mixed $getLogsDB;
protected array $periods = [
'1h' => 'Y-m-d H:00',
'1d' => 'Y-m-d 00:00',
'inf' => '0000-00-00 00:00'
];
public static function getName(): string
{
return 'stats-usage-dump';
}
/**
* @throws \Exception
*/
public function __construct()
{
$this
->inject('message')
->inject('getProjectDB')
->inject('getLogsDB')
->inject('register')
->callback([$this, 'action']);
}
/**
* @param Message $message
* @param callable $getProjectDB
* @param callable $getLogsDB
* @return void
* @throws Exception
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, callable $getProjectDB, callable $getLogsDB, Registry $register): void
{
$this->getLogsDB = $getLogsDB;
$this->register = $register;
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
foreach ($payload['stats'] ?? [] as $stats) {
$project = new Document($stats['project'] ?? []);
$numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
$receivedAt = $stats['receivedAt'] ?? 'NONE';
if ($numberOfKeys === 0) {
continue;
}
console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys);
try {
/** @var \Utopia\Database\Database $dbForProject */
$dbForProject = $getProjectDB($project);
foreach ($stats['keys'] ?? [] as $key => $value) {
if ($value == 0) {
continue;
}
if (str_contains($key, METRIC_DATABASES_STORAGE)) {
try {
$this->handleDatabaseStorage($key, $dbForProject, $project);
} catch (\Exception $e) {
console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage());
}
continue;
}
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
$document = new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => System::getEnv('_APP_REGION', 'default'),
]);
$documentClone = new Document($document->getArrayCopy());
$dbForProject->createOrUpdateDocumentsWithIncrease(
'stats',
'value',
[$document]
);
$this->writeToLogsDB($project, $documentClone);
}
}
} catch (\Exception $e) {
console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage());
}
}
}
private function handleDatabaseStorage(string $key, Database $dbForProject, Document $project): void
{
$data = explode('.', $key);
$start = microtime(true);
$updateMetric = function (Database $dbForProject, Document $project, int $value, string $key, string $period, string|null $time) {
$id = \md5("{$time}_{$period}_{$key}");
$document = new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => System::getEnv('_APP_REGION', 'default'),
]);
$documentClone = new Document($document->getArrayCopy());
$dbForProject->createOrUpdateDocumentsWithIncrease(
'stats',
'value',
[$document]
);
$this->writeToLogsDB($project, $documentClone);
};
foreach ($this->periods as $period => $format) {
$time = 'inf' === $period ? null : date($format, time());
$id = \md5("{$time}_{$period}_{$key}");
$value = 0;
$previousValue = 0;
try {
$previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value', 0);
} catch (\Exception $e) {
// No previous value
}
switch (count($data)) {
// Collection Level
case self::METRIC_COLLECTION_LEVEL_STORAGE:
Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']');
$databaseInternalId = $data[0];
$collectionInternalId = $data[1];
try {
$value = $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collectionInternalId);
} catch (\Exception $e) {
// Collection not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
// Compare with previous value
$diff = $value - $previousValue;
if ($diff === 0) {
break;
}
// Update Collection
$updateMetric($dbForProject, $project, $diff, $key, $period, $time);
// Update Database
$databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE);
$updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time);
// Update Project
$projectKey = METRIC_DATABASES_STORAGE;
$updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time);
break;
// Database Level
case self::METRIC_DATABASE_LEVEL_STORAGE:
Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']');
$databaseInternalId = $data[0];
$collections = [];
try {
$collections = $dbForProject->find('database_' . $databaseInternalId);
} catch (\Exception $e) {
// Database not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
foreach ($collections as $collection) {
try {
$value += $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId());
} catch (\Exception $e) {
// Collection not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
}
$diff = $value - $previousValue;
if ($diff === 0) {
break;
}
// Update Database
$databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE);
$updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time);
// Update Project
$projectKey = METRIC_DATABASES_STORAGE;
$updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time);
break;
// Project Level
case self::METRIC_PROJECT_LEVEL_STORAGE:
Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']');
// Get all project databases
$databases = $dbForProject->find('database');
// Recalculate all databases
foreach ($databases as $database) {
$collections = $dbForProject->find('database_' . $database->getInternalId());
foreach ($collections as $collection) {
try {
$value += $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
} catch (\Exception $e) {
// Collection not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
}
}
$diff = $value - $previousValue;
// Update Project
$projectKey = METRIC_DATABASES_STORAGE;
$updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time);
break;
}
}
$end = microtime(true);
console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds');
}
protected function writeToLogsDB(Document $project, Document $document)
{
if (!System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', false)) {
Console::log('Dual Writing is disabled. Skipping...');
return;
}
if (array_key_exists($document->getAttribute('metric'), $this->skipBaseMetrics)) {
return;
}
foreach ($this->skipParentIdMetrics as $skipMetric) {
if (str_ends_with($document->getAttribute('metric'), $skipMetric)) {
return;
}
}
/** @var \Utopia\Database\Database $dbForLogs*/
$dbForLogs = call_user_func($this->getLogsDB, $project);
try {
$dbForLogs->createOrUpdateDocumentsWithIncrease(
'stats',
'value',
[$document]
);
Console::success('Usage logs pushed to Logs DB');
} catch (\Throwable $th) {
Console::error($th->getMessage());
}
$this->register->get('pools')->get('logs')->reclaim();
}
}
src/Appwrite/Platform/Workers/Webhooks.php
+9 -9
View File
@@ -3,7 +3,7 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Event\Mail;
use Appwrite\Event\Usage;
use Appwrite\Event\StatsUsage;
use Appwrite\Template\Template;
use Exception;
use Utopia\Database\Database;
@@ -35,9 +35,9 @@ class Webhooks extends Action
->inject('project')
->inject('dbForPlatform')
->inject('queueForMails')
->inject('queueForUsage')
->inject('queueForStatsUsage')
->inject('log')
->callback(fn (Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForMails, $queueForUsage, $log));
->callback(fn (Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage, Log $log) => $this->action($message, $project, $dbForPlatform, $queueForMails, $queueForStatsUsage, $log));
}
/**
@@ -49,7 +49,7 @@ class Webhooks extends Action
* @return void
* @throws Exception
*/
public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log): void
public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage, Log $log): void
{
$this->errors = [];
$payload = $message->getPayload() ?? [];
@@ -66,7 +66,7 @@ class Webhooks extends Action
foreach ($project->getAttribute('webhooks', []) as $webhook) {
if (array_intersect($webhook->getAttribute('events', []), $events)) {
$this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $queueForUsage);
$this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $queueForStatsUsage);
}
}
@@ -85,7 +85,7 @@ class Webhooks extends Action
* @param Mail $queueForMails
* @return void
*/
private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage): void
private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage): void
{
if ($webhook->getAttribute('enabled') !== true) {
return;
@@ -168,7 +168,7 @@ class Webhooks extends Action
$dbForPlatform->purgeCachedDocument('projects', $project->getId());
$this->errors[] = $logs;
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_WEBHOOKS_FAILED, 1)
->addMetric(str_replace('{webhookInternalId}', $webhook->getInternalId(), METRIC_WEBHOOK_ID_FAILED), 1)
;
@@ -178,13 +178,13 @@ class Webhooks extends Action
$webhook->setAttribute('attempts', 0); // Reset attempts on success
$dbForPlatform->updateDocument('webhooks', $webhook->getId(), $webhook);
$dbForPlatform->purgeCachedDocument('projects', $project->getId());
$queueForUsage
$queueForStatsUsage
->addMetric(METRIC_WEBHOOKS_SENT, 1)
->addMetric(str_replace('{webhookInternalId}', $webhook->getInternalId(), METRIC_WEBHOOK_ID_SENT), 1)
;
}
$queueForUsage
$queueForStatsUsage
->setProject($project)
->trigger();
}
src/Appwrite/Specification/Format/OpenAPI3.php
-1
View File
@@ -375,7 +375,6 @@ class OpenAPI3 extends Format
$node['schema']['format'] = 'email';
$node['schema']['x-example'] = 'email@example.com';
break;
case 'Appwrite\Network\Validator\Redirect':
case 'Utopia\Validator\Host':
case 'Utopia\Validator\URL':
$node['schema']['type'] = $validator->getType();
src/Appwrite/Specification/Format/Swagger2.php
-1
View File
@@ -393,7 +393,6 @@ class Swagger2 extends Format
$node['format'] = 'email';
$node['x-example'] = 'email@example.com';
break;
case 'Appwrite\Network\Validator\Redirect':
case 'Utopia\Validator\Host':
case 'Utopia\Validator\URL':
$node['type'] = $validator->getType();
src/Appwrite/Utopia/Response/Model/Platform.php
+1 -1
View File
@@ -41,7 +41,7 @@ class Platform extends Model
])
->addRule('type', [
'type' => self::TYPE_STRING,
'description' => 'Platform type. Possible values are: web, flutter-web, flutter-ios, flutter-android, ios, android, react-native-android, react-native-ios and unity.',
'description' => 'Platform type. Possible values are: web, flutter-web, flutter-ios, flutter-android, ios, android, and unity.',
'default' => '',
'example' => 'web',
])
tests/e2e/Services/Databases/DatabasesBase.php
+67
View File
@@ -399,6 +399,73 @@ trait DatabasesBase
$this->assertEquals(400, $response['headers']['status-code']);
}
/**
* @depends testCreateDatabase
*/
public function testPatchAttribute(array $data): void
{
$databaseId = $data['databaseId'];
$collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'collectionId' => ID::unique(),
'name' => 'patch',
'documentSecurity' => true,
'permissions' => [
Permission::create(Role::user($this->getUser()['$id'])),
],
]);
$this->assertEquals(201, $collection['headers']['status-code']);
$this->assertEquals($collection['body']['name'], 'patch');
$attribute = $this->client->call(Client::METHOD_POST, '/databases/'.$databaseId.'/collections/'.$collection['body']['$id'].'/attributes/string', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'key' => 'title',
'required' => true,
'size' => 100,
]);
$this->assertEquals(202, $attribute['headers']['status-code']);
$this->assertEquals($attribute['body']['size'], 100);
sleep(1);
$index = $this->client->call(Client::METHOD_POST, '/databases/'.$databaseId.'/collections/'.$collection['body']['$id'].'/indexes', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
]), [
'key' => 'titleIndex',
'type' => 'key',
'attributes' => ['title'],
]);
$this->assertEquals(202, $index['headers']['status-code']);
sleep(1);
/**
* Update attribute size to exceed Index maximum length
*/
$attribute = $this->client->call(Client::METHOD_PATCH, '/databases/'.$databaseId.'/collections/'.$collection['body']['$id'].'/attributes/string/'.$attribute['body']['key'], array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]), [
'size' => 1000,
'required' => true,
'default' => null,
]);
$this->assertEquals(400, $attribute['headers']['status-code']);
$this->assertStringContainsString('Index length is longer than the maximum: 76', $attribute['body']['message']);
}
public function testUpdateAttributeEnum(): void
{
$database = $this->client->call(Client::METHOD_POST, '/databases', [
tests/e2e/Services/Health/HealthCustomServerTest.php
+30 -6
View File
@@ -494,12 +494,12 @@ class HealthCustomServerTest extends Scope
return [];
}
public function testUsageSuccess()
public function testStatsResources()
{
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/usage', array_merge([
$response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-resources', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
@@ -511,19 +511,19 @@ class HealthCustomServerTest extends Scope
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/usage?threshold=0', array_merge([
$response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-resources?threshold=0', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(503, $response['headers']['status-code']);
}
public function testUsageDumpSuccess()
public function testUsageSuccess()
{
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/usage-dump', array_merge([
$response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-usage', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
@@ -535,7 +535,31 @@ class HealthCustomServerTest extends Scope
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/usage-dump?threshold=0', array_merge([
$response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-usage?threshold=0', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(503, $response['headers']['status-code']);
}
public function testStatsUsageDumpSuccess()
{
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-usage-dump', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertIsInt($response['body']['size']);
$this->assertLessThan(100, $response['body']['size']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-usage-dump?threshold=0', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
tests/resources/docker/docker-compose.yml
+3 -3
View File
@@ -89,9 +89,9 @@ services:
- _APP_FUNCTIONS_MEMORY_SWAP
- _APP_EXECUTOR_HOST
appwrite-worker-usage:
entrypoint: worker-usage
container_name: appwrite-worker-usage
appwrite-worker-stats-usage:
entrypoint: worker-stats-usage
container_name: appwrite-worker-stats-usage
build:
context: .
restart: unless-stopped
tests/unit/Network/Validators/RedirectTest.php
-43
View File
@@ -1,43 +0,0 @@
<?php
namespace Tests\Unit\Network\Validators;
use Appwrite\Network\Validator\Redirect;
use PHPUnit\Framework\TestCase;
class RedirectTest extends TestCase
{
public function redirectsProvider(): array
{
return [
"localhost" => [["localhost"], [], "http://localhost", true],
"localhost-no-scheme" => [["localhost"], [], "localhost", false],
"expo scheme" => [[], ["exp"], "exp://192.168.0.1", true],
"custom scheme" => [[], ["myapp"], "myapp://", true],
"custom scheme triple slash" => [[], ["myapp"], "myapp:///", true],
"scheme with special chars" => [[], ["my-app+custom.123"], "my-app+custom.123://", true],
"url https" => [["example.com"], [], "https://example.com", true],
"url http" => [["example.com"], [], "http://example.com", true],
"malformed scheme" => [[], [], "http:/example.com", false],
"invalid url" => [[], [], "example.com", false],
"invalid host" => [["notexample.com"], [], "https://example.com", false],
"javascript scheme" => [[], [], "javascript://alert(1)", false],
"javascript scheme with different case" => [[], [], "JaVaScRiPt://alert(1)", false],
"empty string" => [[], [], "", false],
];
}
/**
* @dataProvider redirectsProvider
*/
public function testIsValid(
array $hostnames,
array $schemes,
string $value,
bool $expected
): void {
$validator = new Redirect($hostnames, $schemes);
$this->assertEquals($expected, $validator->isValid($value));
}
}