Merge branch 'refactor-usage-sn' of github.com:appwrite/appwrite into refactor-move-cli-scripts

 Conflicts:

	src/Appwrite/Platform/Services/Tasks.php

.env

@@ -4,12 +4,13 @@ _APP_WORKER_PER_CORE=6
 _APP_CONSOLE_WHITELIST_ROOT=disabled
 _APP_CONSOLE_WHITELIST_EMAILS=
 _APP_CONSOLE_WHITELIST_IPS=
+_APP_CONSOLE_HOSTNAMES=localhost,appwrite.io,*.appwrite.io
 _APP_SYSTEM_EMAIL_NAME=Appwrite
 _APP_SYSTEM_EMAIL_ADDRESS=team@appwrite.io
 _APP_SYSTEM_SECURITY_EMAIL_ADDRESS=security@appwrite.io
 _APP_SYSTEM_RESPONSE_FORMAT=
 _APP_OPTIONS_ABUSE=disabled
-_APP_OPTIONS_ROUTER_PROTECTION=disbled
+_APP_OPTIONS_ROUTER_PROTECTION=disabled
 _APP_OPTIONS_FORCE_HTTPS=disabled
 _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS=disabled
 _APP_OPENSSL_KEY_V1=your-secret-key
@@ -50,10 +51,6 @@ _APP_STORAGE_WASABI_BUCKET=
 _APP_STORAGE_ANTIVIRUS=disabled
 _APP_STORAGE_ANTIVIRUS_HOST=clamav
 _APP_STORAGE_ANTIVIRUS_PORT=3310
-_APP_INFLUXDB_HOST=influxdb
-_APP_INFLUXDB_PORT=8086
-_APP_STATSD_HOST=telegraf
-_APP_STATSD_PORT=8125
 _APP_SMTP_HOST=maildev
 _APP_SMTP_PORT=1025
 _APP_SMTP_SECURE=
@@ -61,6 +58,7 @@ _APP_SMTP_USERNAME=
 _APP_SMTP_PASSWORD=
 _APP_SMS_PROVIDER=sms://username:password@mock
 _APP_SMS_FROM=+123456789
+_APP_SMS_PROJECTS_DENY_LIST=
 _APP_STORAGE_LIMIT=30000000
 _APP_STORAGE_PREVIEW_LIMIT=20000000
 _APP_FUNCTIONS_SIZE_LIMIT=30000000
@@ -79,7 +77,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000
 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600
 _APP_MAINTENANCE_RETENTION_ABUSE=86400
 _APP_MAINTENANCE_RETENTION_AUDIT=1209600
-_APP_USAGE_AGGREGATION_INTERVAL=5
+_APP_USAGE_AGGREGATION_INTERVAL=60000
 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400
 _APP_USAGE_STATS=enabled

.github/workflows/linter.yml

@@ -18,6 +18,10 @@ jobs:
 
     - run: git checkout HEAD^2
 
+    - name: Validate composer.json and composer.lock
+      run: |
+        docker run --rm -v $PWD:/app composer sh -c \
+        "composer validate"
     - name: Run Linter
       run: |
         docker run --rm -v $PWD:/app composer sh -c \

.github/workflows/tests.yml

@@ -75,8 +75,32 @@ jobs:
       - name: Run Unit Tests
         run: docker compose exec appwrite test /usr/src/code/tests/unit
 
-  e2e_test:
-    name: E2E Test
+  e2e_general_test:
+    name: E2E General Test
+    runs-on: ubuntu-latest
+    needs: setup
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: Load Cache
+        uses: actions/cache@v3
+        with:
+          key: ${{ env.CACHE_KEY }}
+          path: /tmp/${{ env.IMAGE }}.tar
+          fail-on-cache-miss: true
+
+      - name: Load and Start Appwrite
+        run: |
+          docker load --input /tmp/${{ env.IMAGE }}.tar
+          docker compose up -d
+          sleep 10
+
+      - name: Run General Tests
+        run: docker compose exec -T appwrite test /usr/src/code/tests/e2e/General --debug
+
+  e2e_service_test:
+    name: E2E Service Test
     runs-on: ubuntu-latest
     needs: setup
     strategy:
@@ -119,4 +143,4 @@ jobs:
           sleep 10
 
       - name: Run ${{matrix.service}} Tests
-        run: docker compose exec -T appwrite test /usr/src/code/tests/e2e/Services/${{matrix.service}} --debug
+        run: docker compose exec -T appwrite test /usr/src/code/tests/e2e/Services/${{matrix.service}} --debug

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "app/console"]
 	path = app/console
 	url = https://github.com/appwrite/console
-	branch = 3.2.5
+	branch = 3.2.16

CHANGES.md

@@ -1,3 +1,66 @@
+# Version 1.4.14
+
+## Changes
+- New usage metrics collection flow [#7005](https://github.com/appwrite/appwrite/pull/7005)
+  - Deprecated influxdb, telegraf containers and removed all of their occurrences from the code.
+  - Removed _APP_INFLUXDB_HOST, _APP_INFLUXDB_PORT, _APP_STATSD_HOST, _APP_STATSD_PORT env variables.
+  - Removed usage labels dependency.
+  - Dropped type attribute from stats collection.
+  - Usage metrics are processed via new usage worker.
+  - updated Metric names.
+  
+# Version 1.4.13
+
+## Notable changes
+
+* Change enum size validation in update controller [#7164](https://github.com/appwrite/appwrite/pull/7164)
+* Bump console to version 3.2.8 in [#7167](https://github.com/appwrite/appwrite/pull/7167)
+
+## Bug fixes
+
+* Fix error after adding bigger enum [#7162](https://github.com/appwrite/appwrite/pull/7162)
+* Add chunkId to abuse key to prevent rate limit for SDKs [#7154](https://github.com/appwrite/appwrite/pull/7154)
+
+## Miscellaneous
+
+* Fix enum test case [#7163](https://github.com/appwrite/appwrite/pull/7163)
+* Add flag to send logs to logger [#7155](https://github.com/appwrite/appwrite/pull/7155)
+* Add a CI task to validate composer file and lock [#7142](https://github.com/appwrite/appwrite/pull/7142)
+
+# Version 1.4.12
+
+## Miscellaneous
+* Bump console to version 3.2.7 [#7148](https://github.com/appwrite/appwrite/pull/7148)
+* Chore update database to 0.45.2 [#7138](https://github.com/appwrite/appwrite/pull/7138)
+* Implement queue thresholds for the health API [#7123](https://github.com/appwrite/appwrite/pull/7123)
+* Add Authorization::skip to the usage worker [#7124](https://github.com/appwrite/appwrite/pull/7124)
+
+## Bug fixes
+* fix: use queueForDeletes in git installation delete endpoint [#7140](https://github.com/appwrite/appwrite/pull/7140)
+* fix: patch script, make errors silent [#7134](https://github.com/appwrite/appwrite/pull/7134)
+* fix: repositories recreation script [#7133](https://github.com/appwrite/appwrite/pull/7133)
+* fix: Only delete repositories linked to the particular project [#7131](https://github.com/appwrite/appwrite/pull/7131)
+
+# Version 1.4.11
+
+## Miscellaneous
+
+* Update database by @abnegate in [#7111](https://github.com/appwrite/appwrite/pull/7111)
+
+# Version 1.4.10
+
+## Bug fixes
+* Handle cases where password history could contain NULLs [#7092](https://github.com/appwrite/appwrite/pull/7092)
+* Missing functionId error on create execution [#7091](https://github.com/appwrite/appwrite/pull/7091)
+* Ensure usage endpoints don't throw 500 when usage is disabled [#7087](https://github.com/appwrite/appwrite/pull/7087)
+* Missing sessionId error when deleting all user sessions [#7085](https://github.com/appwrite/appwrite/pull/7085)
+* Domain validation in Create Proxy rule results in 500 error [#7084](https://github.com/appwrite/appwrite/pull/7084)
+* Fix optional services [#7078](https://github.com/appwrite/appwrite/pull/7078)
+* Fix regression from worker refactor [#7074](https://github.com/appwrite/appwrite/pull/7074)
+* Use getQueueSize() in the Health service's get X queue endpoints [#7073](https://github.com/appwrite/appwrite/pull/7073)
+* Delete linked VCS repos and comments [#7066](https://github.com/appwrite/appwrite/pull/7066)
+
+
 # Version 1.4.9
 
 ## Bug fixes

CONTRIBUTING.md

@@ -210,7 +210,6 @@ Appwrite's current structure is a combination of both [Monolithic](https://en.wi
 │       ├── Task
 │       ├── Template
 │       ├── URL
-│       ├── Usage
 │       └── Utopia
 └── tests # End to end & unit tests
     ├── e2e

Dockerfile

@@ -74,7 +74,6 @@ RUN mkdir -p /storage/uploads && \
 # Executables
 RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/maintenance &&  \
-    chmod +x /usr/local/bin/usage && \
     chmod +x /usr/local/bin/install && \
     chmod +x /usr/local/bin/upgrade && \
     chmod +x /usr/local/bin/migrate && \
@@ -85,6 +84,10 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/ssl && \
     chmod +x /usr/local/bin/test && \
     chmod +x /usr/local/bin/vars && \
+    chmod +x /usr/local/bin/queue-retry && \
+    chmod +x /usr/local/bin/queue-count-failed && \
+    chmod +x /usr/local/bin/queue-count-processing && \
+    chmod +x /usr/local/bin/queue-count-success && \
     chmod +x /usr/local/bin/worker-audits && \
     chmod +x /usr/local/bin/worker-certificates && \
     chmod +x /usr/local/bin/worker-databases && \
@@ -94,17 +97,9 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/worker-mails && \
     chmod +x /usr/local/bin/worker-messaging && \
     chmod +x /usr/local/bin/worker-webhooks && \
-    chmod +x /usr/local/bin/worker-migrations
+    chmod +x /usr/local/bin/worker-migrations && \
+    chmod +x /usr/local/bin/worker-usage
 
-# Cloud Executabless
-RUN chmod +x /usr/local/bin/hamster && \
-    chmod +x /usr/local/bin/volume-sync && \
-    chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \
-    chmod +x /usr/local/bin/patch-delete-project-collections && \
-    chmod +x /usr/local/bin/delete-orphaned-projects && \
-    chmod +x /usr/local/bin/clear-card-cache && \
-    chmod +x /usr/local/bin/calc-users-stats && \
-    chmod +x /usr/local/bin/calc-tier-stats
 
 # Letsencrypt Permissions
 RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/

README-CN.md

@@ -55,7 +55,7 @@ Appwrite 可以提供给开发者用户验证，外部授权，用户数据读
 
 ## 安装
 
-Appwrite 的容器化服务器只需要一行指令就可以运行。您可以使用 docker-compose 在本地主机上运行 Appwrite，也可以在任何其他容器化工具（如 Kubernetes、Docker Swarm 或 Rancher）上运行 Appwrite。
+Appwrite 的容器化服务器只需要一行指令就可以运行。您可以使用 docker-compose 在本地主机上运行 Appwrite，也可以在任何其他容器化工具（如 [Kubernetes](https://kubernetes.io/docs/home/)、[Docker Swarm](https://docs.docker.com/engine/swarm/) 或 [Rancher](https://rancher.com/docs/)）上运行 Appwrite。
 
 启动 Appwrite 服务器的最简单方法是运行我们的 docker-compose 文件。在运行安装命令之前，请确保您的机器上安装了 [Docker](https://dockerdocs.cn/get-docker/index.html)：
 
@@ -66,7 +66,7 @@ docker run -it --rm \
     --volume /var/run/docker.sock:/var/run/docker.sock \
     --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \
     --entrypoint="install" \
-    appwrite/appwrite:1.4.9
+    appwrite/appwrite:1.4.13
 ```
 
 ### Windows
@@ -78,7 +78,7 @@ docker run -it --rm ^
     --volume //var/run/docker.sock:/var/run/docker.sock ^
     --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^
     --entrypoint="install" ^
-    appwrite/appwrite:1.4.9
+    appwrite/appwrite:1.4.13
 ```
 
 #### PowerShell
@@ -88,7 +88,7 @@ docker run -it --rm `
     --volume /var/run/docker.sock:/var/run/docker.sock `
     --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw `
     --entrypoint="install" `
-    appwrite/appwrite:1.4.9
+    appwrite/appwrite:1.4.13
 ```
 
 运行后，可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后，服务器可能需要几分钟才能启动。

README.md

@@ -76,7 +76,7 @@ docker run -it --rm \
     --volume /var/run/docker.sock:/var/run/docker.sock \
     --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \
     --entrypoint="install" \
-    appwrite/appwrite:1.4.9
+    appwrite/appwrite:1.4.13
 ```
 
 ### Windows
@@ -88,7 +88,7 @@ docker run -it --rm ^
     --volume //var/run/docker.sock:/var/run/docker.sock ^
     --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^
     --entrypoint="install" ^
-    appwrite/appwrite:1.4.9
+    appwrite/appwrite:1.4.13
 ```
 
 #### PowerShell
@@ -98,7 +98,7 @@ docker run -it --rm `
     --volume /var/run/docker.sock:/var/run/docker.sock `
     --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw `
     --entrypoint="install" `
-    appwrite/appwrite:1.4.9
+    appwrite/appwrite:1.4.13
 ```
 
 Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation.

app/cli.php

@@ -6,6 +6,7 @@ require_once __DIR__ . '/controllers/general.php';
 use Appwrite\Event\Delete;
 use Appwrite\Event\Certificate;
 use Appwrite\Event\Func;
+use Appwrite\Event\Hamster;
 use Appwrite\Platform\Appwrite;
 use Utopia\CLI\CLI;
 use Utopia\Database\Validator\Authorization;
@@ -124,36 +125,15 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
     };
 }, ['pools', 'dbForConsole', 'cache']);
 
-CLI::setResource('influxdb', function (Registry $register) {
-    $client = $register->get('influxdb'); /** @var InfluxDB\Client $client */
-    $attempts = 0;
-    $max = 10;
-    $sleep = 1;
-
-    do { // check if telegraf database is ready
-        try {
-            $attempts++;
-            $database = $client->selectDB('telegraf');
-            if (in_array('telegraf', $client->listDatabases())) {
-                break; // leave the do-while if successful
-            }
-        } catch (\Throwable $th) {
-            Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
-            if ($attempts >= $max) {
-                throw new \Exception('InfluxDB database not ready yet');
-            }
-            sleep($sleep);
-        }
-    } while ($attempts < $max);
-    return $database;
-}, ['register']);
-
 CLI::setResource('queue', function (Group $pools) {
     return $pools->get('queue')->pop()->getResource();
 }, ['pools']);
 CLI::setResource('queueForFunctions', function (Connection $queue) {
     return new Func($queue);
 }, ['queue']);
+CLI::setResource('queueForHamster', function (Connection $queue) {
+    return new Hamster($queue);
+}, ['queue']);
 CLI::setResource('queueForDeletes', function (Connection $queue) {
     return new Delete($queue);
 }, ['queue']);

app/config/collections.php

@@ -18,6 +18,63 @@ $auth = Config::getParam('auth', []);
  */
 
 $commonCollections = [
+    'cache' => [
+        '$collection' => Database::METADATA,
+        '$id' => 'cache',
+        'name' => 'Cache',
+        'attributes' => [
+            [
+                '$id' => 'resource',
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => 255,
+                'signed' => true,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
+            [
+                '$id' => 'accessedAt',
+                'type' => Database::VAR_DATETIME,
+                'format' => '',
+                'size' => 0,
+                'signed' => false,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => ['datetime'],
+            ],
+            [
+                '$id' => 'signature',
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => 255,
+                'signed' => true,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
+        ],
+        'indexes' => [
+            [
+                '$id' => '_key_accessedAt',
+                'type' => Database::INDEX_KEY,
+                'attributes' => ['accessedAt'],
+                'lengths' => [],
+                'orders' => [],
+            ],
+            [
+                '$id' => '_key_resource',
+                'type' => Database::INDEX_KEY,
+                'attributes' => ['resource'],
+                'lengths' => [],
+                'orders' => [],
+            ],
+        ],
+    ],
+
     'users' => [
         '$collection' => ID::custom(Database::METADATA),
         '$id' => ID::custom('users'),
@@ -1270,10 +1327,10 @@ $commonCollections = [
         ]
     ],
 
-    'stats' => [
+    'stats_v2' => [
         '$collection' => ID::custom(Database::METADATA),
-        '$id' => ID::custom('stats'),
-        'name' => 'Stats',
+        '$id' => ID::custom('stats_v2'),
+        'name' => 'stats_v2',
         'attributes' => [
             [
                 '$id' => ID::custom('metric'),
@@ -1302,7 +1359,7 @@ $commonCollections = [
                 'type' => Database::VAR_INTEGER,
                 'format' => '',
                 'size' => 8,
-                'signed' => false,
+                'signed' => true,
                 'required' => true,
                 'default' => null,
                 'array' => false,
@@ -1330,17 +1387,6 @@ $commonCollections = [
                 'array' => false,
                 'filters' => [],
             ],
-            [
-                '$id' => ID::custom('type'),
-                'type' => Database::VAR_INTEGER,
-                'format' => '',
-                'size' => 1,
-                'signed' => false,
-                'required' => true,
-                'default' => 0, // 0 -> count, 1 -> sum
-                'array' => false,
-                'filters' => [],
-            ],
         ],
         'indexes' => [
             [
@@ -1359,7 +1405,7 @@ $commonCollections = [
             ],
             [
                 '$id' => ID::custom('_key_metric_period_time'),
-                'type' => Database::INDEX_KEY,
+                'type' => Database::INDEX_UNIQUE,
                 'attributes' => ['metric', 'period', 'time'],
                 'lengths' => [],
                 'orders' => [Database::ORDER_DESC],
@@ -2883,63 +2929,6 @@ $projectCollections = array_merge([
         ],
     ],
 
-    'cache' => [
-        '$collection' => Database::METADATA,
-        '$id' => 'cache',
-        'name' => 'Cache',
-        'attributes' => [
-            [
-                '$id' => 'resource',
-                'type' => Database::VAR_STRING,
-                'format' => '',
-                'size' => 255,
-                'signed' => true,
-                'required' => false,
-                'default' => null,
-                'array' => false,
-                'filters' => [],
-            ],
-            [
-                '$id' => 'accessedAt',
-                'type' => Database::VAR_DATETIME,
-                'format' => '',
-                'size' => 0,
-                'signed' => false,
-                'required' => false,
-                'default' => null,
-                'array' => false,
-                'filters' => ['datetime'],
-            ],
-            [
-                '$id' => 'signature',
-                'type' => Database::VAR_STRING,
-                'format' => '',
-                'size' => 255,
-                'signed' => true,
-                'required' => false,
-                'default' => null,
-                'array' => false,
-                'filters' => [],
-            ],
-        ],
-        'indexes' => [
-            [
-                '$id' => '_key_accessedAt',
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['accessedAt'],
-                'lengths' => [],
-                'orders' => [],
-            ],
-            [
-                '$id' => '_key_resource',
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['resource'],
-                'lengths' => [],
-                'orders' => [],
-            ],
-        ],
-    ],
-
     'variables' => [
         '$collection' => Database::METADATA,
         '$id' => 'variables',

app/config/errors.php

@@ -209,6 +209,7 @@ return [
         'name' => Exception::USER_AUTH_METHOD_UNSUPPORTED,
         'description' => 'The requested authentication method is either disabled or unsupported. Please check the supported authentication methods in the Appwrite console.',
         'code' => 501,
+        'publish' => false,
     ],
     Exception::USER_PHONE_ALREADY_EXISTS => [
         'name' => Exception::USER_PHONE_ALREADY_EXISTS,
@@ -240,6 +241,16 @@ return [
         'description' => 'OAuth2 provider returned some error.',
         'code' => 424,
     ],
+    Exception::USER_EMAIL_ALREADY_VERIFIED => [
+        'name' => Exception::USER_EMAIL_ALREADY_VERIFIED,
+        'description' => 'User email is already verified',
+        'code' => 409,
+    ],
+    Exception::USER_PHONE_ALREADY_VERIFIED => [
+        'name' => Exception::USER_PHONE_ALREADY_VERIFIED,
+        'description' => 'User phone is already verified',
+        'code' => 409
+    ],
 
     /** Teams */
     Exception::TEAM_NOT_FOUND => [
@@ -669,6 +680,7 @@ return [
         'name' => Exception::RULE_VERIFICATION_FAILED,
         'description' => 'Domain verification failed. Please check if your DNS records are correct and try again.',
         'code' => 401,
+        'publish' => true
     ],
     Exception::PROJECT_SMTP_CONFIG_INVALID => [
         'name' => Exception::PROJECT_SMTP_CONFIG_INVALID,
@@ -749,4 +761,17 @@ return [
         'description' => 'Too many messages.',
         'code' => 1013,
     ],
+    Exception::MIGRATION_PROVIDER_ERROR => [
+        'name' => Exception::MIGRATION_PROVIDER_ERROR,
+        'description' => 'An error occurred on the provider\'s side. Please try again later.',
+        'code' => 400,
+    ],
+
+    /** Health */
+    Exception::QUEUE_SIZE_EXCEEDED => [
+        'name' => Exception::QUEUE_SIZE_EXCEEDED,
+        'description' => 'Queue size threshold hit.',
+        'code' => 503,
+        'publish' => false
+    ],
 ];

app/config/locale/templates/email-inner-base.tpl

@@ -1,15 +1,9 @@
-<p>{{hello}}</p>
-
-<br>
-
+<p>{{hello}},</p>
 <p>{{body}}</p>
-
-<a href="{{redirect}}" target="_blank">{{redirect}}</a>
-
+<p><a href="{{redirect}}" target="_blank">{{redirect}}</a></p>
 <p>{{footer}}</p>
-
-<br>
-
-<p>{{thanks}}</p>
-
-<p>{{signature}}</p>
+<p style="margin-bottom: 32px">
+    {{thanks}},
+    <br/>
+    {{signature}}
+</p>

app/config/locale/translations/tr.json

@@ -3,30 +3,36 @@
     "settings.locale": "tr",
     "settings.direction": "ltr",
     "emails.sender": "%s Takımı",
-    "emails.verification.subject": "",
-    "emails.verification.hello": "",
-    "emails.verification.body": "",
-    "emails.verification.footer": "",
-    "emails.verification.thanks": "",
-    "emails.verification.signature": "",
-    "emails.magicSession.subject": "",
-    "emails.magicSession.hello": "",
-    "emails.magicSession.body": "",
-    "emails.magicSession.footer": "",
-    "emails.magicSession.thanks": "",
-    "emails.magicSession.signature": "",
-    "emails.recovery.subject": "",
-    "emails.recovery.hello": "",
-    "emails.recovery.body": "",
-    "emails.recovery.footer": "",
-    "emails.recovery.thanks": "",
-    "emails.recovery.signature": "",
-    "emails.invitation.subject": "",
-    "emails.invitation.hello": "",
-    "emails.invitation.body": "",
-    "emails.invitation.footer": "",
-    "emails.invitation.thanks": "",
-    "emails.invitation.signature": "",
+    "emails.verification.subject": "Hesabını Doğrula",
+    "emails.verification.hello": "Merhaba {{user}}",
+    "emails.verification.body": "Eposta adresini doğrulamak için bu bağlantıyı kullanın.",
+    "emails.verification.footer": "Eğer bu eposta adresini doğrulamak isteyen siz değilseniz devam etmeyin.",
+    "emails.verification.thanks": "Teşekkürler",
+    "emails.verification.signature": "{{project}} takımı",
+    "emails.magicSession.subject": "Giriş",
+    "emails.magicSession.hello": "Merhaba,",
+    "emails.magicSession.body": "Giriş yapmak için tıklayın.",
+    "emails.magicSession.footer": "Eğer bu eposta adresini kullanarak giriş yapmak istemediyseniz devam etmeyin.",
+    "emails.magicSession.thanks": "Teşekkürler",
+    "emails.magicSession.signature": "{{project}} takımı",
+    "emails.recovery.subject": "Şifremi Sıfırla",
+    "emails.recovery.hello": "Merhaba {{user}}",
+    "emails.recovery.body": "{{project}} şifrenizi sıfırlamak için bu bağlantıyı kullanın.",
+    "emails.recovery.footer": "Eğer şifre sıfırlama talebinde bulunmadıysanız devam etmeyin.",
+    "emails.recovery.thanks": "Teşekkürler",
+    "emails.recovery.signature": "{{project}} takımı",
+    "emails.invitation.subject": "%s üzerinde %s Takımına Davet",
+    "emails.invitation.hello": "Merhaba",
+    "emails.invitation.body": "Bu epostayı aldınız, çünkü {{owner}} sizi {{project}} üzerinde {{team}} takımının üyesi olmaya davet etti.",
+    "emails.invitation.footer": "Eğer ilgilenmiyorsanız devam etmeyin.",
+    "emails.invitation.thanks": "Teşekkürler",
+    "emails.invitation.signature": "{{project}} takımı",
+    "emails.certificate.subject": "%s için sertifika hatası",
+    "emails.certificate.hello": "Merhaba",
+    "emails.certificate.body": "Alan adınız '{{domain}}' için sertifika oluşturulamadı. Deneme sayısı {{attempt}} ve hata sebebi: {{error}}",
+    "emails.certificate.footer": "Geçmiş sertifikanız ilk denemeden sonra 30 gün daha geçerli kalacaktır. Bu konuyu araştırmanızı öneriyoruz, aksi taktirde alan adınız SSL sertifikasız kalacaktır.",
+    "emails.certificate.thanks": "Teşekkürler",
+    "emails.certificate.signature": "{{project}} takımı",
     "locale.country.unknown": "Bilinmeyen",
     "countries.af": "Afganistan",
     "countries.ao": "Angola",
@@ -229,4 +235,4 @@
     "continents.na": "Kuzey Amerika",
     "continents.oc": "Okyanusya",
     "continents.sa": "Güney Amerika"
-}
+}

app/config/platforms.php

@@ -15,7 +15,7 @@ return [
             [
                 'key' => 'web',
                 'name' => 'Web',
-                'version' => '13.0.0',
+                'version' => '13.0.1',
                 'url' => 'https://github.com/appwrite/sdk-for-web',
                 'package' => 'https://www.npmjs.com/package/appwrite',
                 'enabled' => true,
@@ -63,7 +63,7 @@ return [
             [
                 'key' => 'flutter',
                 'name' => 'Flutter',
-                'version' => '11.0.0',
+                'version' => '11.0.1',
                 'url' => 'https://github.com/appwrite/sdk-for-flutter',
                 'package' => 'https://pub.dev/packages/appwrite',
                 'enabled' => true,
@@ -81,7 +81,7 @@ return [
             [
                 'key' => 'apple',
                 'name' => 'Apple',
-                'version' => '4.0.1',
+                'version' => '4.0.2',
                 'url' => 'https://github.com/appwrite/sdk-for-apple',
                 'package' => 'https://github.com/appwrite/sdk-for-apple',
                 'enabled' => true,
@@ -116,7 +116,7 @@ return [
             [
                 'key' => 'android',
                 'name' => 'Android',
-                'version' => '4.0.0',
+                'version' => '4.0.1',
                 'url' => 'https://github.com/appwrite/sdk-for-android',
                 'package' => 'https://search.maven.org/artifact/io.appwrite/sdk-for-android',
                 'enabled' => true,
@@ -185,7 +185,7 @@ return [
             [
                 'key' => 'web',
                 'name' => 'Console',
-                'version' => '0.3.0',
+                'version' => '0.5.0',
                 'url' => 'https://github.com/appwrite/sdk-for-console',
                 'package' => '',
                 'enabled' => true,
@@ -195,15 +195,15 @@ return [
                 'family' => APP_PLATFORM_CONSOLE,
                 'prism' => 'javascript',
                 'source' => \realpath(__DIR__ . '/../sdks/console-web'),
-                'gitUrl' => 'git@github.com:appwrite/sdk-for-console.git',
-                'gitBranch' => 'dev',
+                'gitUrl' => 'https://github.com/appwrite/sdk-for-console.git',
+                'gitBranch' => 'main',
                 'gitRepoName' => 'sdk-for-console',
                 'gitUserName' => 'appwrite',
             ],
             [
                 'key' => 'cli',
                 'name' => 'Command Line',
-                'version' => '4.1.0',
+                'version' => '4.2.0',
                 'url' => 'https://github.com/appwrite/sdk-for-cli',
                 'package' => 'https://www.npmjs.com/package/appwrite-cli',
                 'enabled' => true,
@@ -231,7 +231,7 @@ return [
             [
                 'key' => 'nodejs',
                 'name' => 'Node.js',
-                'version' => '11.0.0',
+                'version' => '11.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-node',
                 'package' => 'https://www.npmjs.com/package/node-appwrite',
                 'enabled' => true,
@@ -249,7 +249,7 @@ return [
             [
                 'key' => 'deno',
                 'name' => 'Deno',
-                'version' => '9.0.0',
+                'version' => '9.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-deno',
                 'package' => 'https://deno.land/x/appwrite',
                 'enabled' => true,
@@ -267,7 +267,7 @@ return [
             [
                 'key' => 'php',
                 'name' => 'PHP',
-                'version' => '10.0.0',
+                'version' => '10.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-php',
                 'package' => 'https://packagist.org/packages/appwrite/appwrite',
                 'enabled' => true,
@@ -285,7 +285,7 @@ return [
             [
                 'key' => 'python',
                 'name' => 'Python',
-                'version' => '4.0.0',
+                'version' => '4.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-python',
                 'package' => 'https://pypi.org/project/appwrite/',
                 'enabled' => true,
@@ -303,7 +303,7 @@ return [
             [
                 'key' => 'ruby',
                 'name' => 'Ruby',
-                'version' => '10.0.0',
+                'version' => '10.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-ruby',
                 'package' => 'https://rubygems.org/gems/appwrite',
                 'enabled' => true,
@@ -321,7 +321,7 @@ return [
             [
                 'key' => 'go',
                 'name' => 'Go',
-                'version' => '3.0.0',
+                'version' => '3.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-go',
                 'package' => '',
                 'enabled' => false,
@@ -339,7 +339,7 @@ return [
             [
                 'key' => 'java',
                 'name' => 'Java',
-                'version' => '3.0.0',
+                'version' => '3.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-java',
                 'package' => '',
                 'enabled' => false,
@@ -357,7 +357,7 @@ return [
             [
                 'key' => 'dotnet',
                 'name' => '.NET',
-                'version' => '0.6.0',
+                'version' => '0.7.0',
                 'url' => 'https://github.com/appwrite/sdk-for-dotnet',
                 'package' => 'https://www.nuget.org/packages/Appwrite',
                 'enabled' => true,
@@ -375,7 +375,7 @@ return [
             [
                 'key' => 'dart',
                 'name' => 'Dart',
-                'version' => '10.0.0',
+                'version' => '10.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-dart',
                 'package' => 'https://pub.dev/packages/dart_appwrite',
                 'enabled' => true,
@@ -393,7 +393,7 @@ return [
             [
                 'key' => 'kotlin',
                 'name' => 'Kotlin',
-                'version' => '4.0.0',
+                'version' => '4.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-kotlin',
                 'package' => 'https://search.maven.org/artifact/io.appwrite/sdk-for-kotlin',
                 'enabled' => true,
@@ -415,7 +415,7 @@ return [
             [
                 'key' => 'swift',
                 'name' => 'Swift',
-                'version' => '4.0.1',
+                'version' => '4.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-swift',
                 'package' => 'https://github.com/appwrite/sdk-for-swift',
                 'enabled' => true,

app/config/services.php

@@ -170,7 +170,7 @@ return [
         'docs' => false,
         'docsUrl' => '',
         'tests' => false,
-        'optional' => true,
+        'optional' => false,
         'icon' => '',
     ],
     'functions' => [
@@ -196,7 +196,7 @@ return [
         'docs' => true,
         'docsUrl' => 'https://appwrite.io/docs/proxy',
         'tests' => false,
-        'optional' => true,
+        'optional' => false,
         'icon' => '/images/services/proxy.png',
     ],
     'mock' => [
@@ -248,7 +248,7 @@ return [
         'docs' => true,
         'docsUrl' => 'https://appwrite.io/docs/migrations',
         'tests' => true,
-        'optional' => true,
+        'optional' => false,
         'icon' => '/images/services/migrations.png',
     ],
 ];

app/config/variables.php

@@ -115,14 +115,6 @@ return [
                 'question' => '',
                 'filter' => ''
             ],
-            // [
-            //     'name' => '_APP_CONSOLE_WHITELIST_DOMAINS',
-            //     'description' => 'This option allows you to limit creation of users to Appwrite console for users sharing the same email domains. This option is very useful for team working with company emails domain.\n\nTo enable this option, pass a list of allowed email domains separated by a comma.',
-            //     'introduction' => '',
-            //     'default' => '',
-            //     'required' => false,
-            //     'question' => '',
-            // ],
             [
                 'name' => '_APP_CONSOLE_WHITELIST_IPS',
                 'description' => "This last option allows you to limit creation of users in Appwrite console for users sharing the same set of IP addresses. This option is very useful for team working with a VPN service or a company IP.\n\nTo enable/activate this option, pass a list of allowed IP addresses separated by a comma.",
@@ -132,6 +124,15 @@ return [
                 'question' => '',
                 'filter' => ''
             ],
+            [
+                'name' => '_APP_CONSOLE_HOSTNAMES',
+                'description' => 'This option allows you to add additional hostnames to your Appwrite console. This option is very useful for allowing access to the console project from additional domains. To enable it, pass a list of allowed hostnames separated by a comma.',
+                'introduction' => '1.5.0',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
             [
                 'name' => '_APP_SYSTEM_EMAIL_NAME',
                 'description' => 'This is the sender name value that will appear on email messages sent to developers from the Appwrite console. The default value is: \'Appwrite\'. You can use url encoded strings for spaces and special chars.',
@@ -337,7 +338,7 @@ return [
     ],
     [
         'category' => 'InfluxDB',
-        'description' => 'Appwrite uses an InfluxDB server for managing time-series data and server stats. The InfluxDB env vars are used to allow Appwrite server to connect to the InfluxDB container.',
+        'description' => 'Deprecated since 1.4.8.',
         'variables' => [
             [
                 'name' => '_APP_INFLUXDB_HOST',
@@ -361,7 +362,7 @@ return [
     ],
     [
         'category' => 'StatsD',
-        'description' => 'Appwrite uses a StatsD server for aggregating and sending stats data over a fast UDP connection. The StatsD env vars are used to allow Appwrite server to connect to the StatsD container.',
+        'description' => 'Deprecated since 1.4.8.',
         'variables' => [
             [
                 'name' => '_APP_STATSD_HOST',

app/controllers/api/account.php

@@ -45,6 +45,8 @@ use Utopia\Validator\WhiteList;
 use Appwrite\Auth\Validator\PasswordHistory;
 use Appwrite\Auth\Validator\PasswordDictionary;
 use Appwrite\Auth\Validator\PersonalData;
+use Appwrite\Event\Delete;
+use Appwrite\Hooks\Hooks;
 
 $oauthDefaultSuccess = '/auth/oauth2/success';
 $oauthDefaultFailure = '/auth/oauth2/failure';
@@ -58,7 +60,6 @@ App::post('/v1/account')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'create')
@@ -77,7 +78,8 @@ App::post('/v1/account')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $name, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents) {
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $name, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
 
         $email = \strtolower($email);
         if ('console' === $project->getId()) {
@@ -118,6 +120,8 @@ App::post('/v1/account')
             }
         }
 
+        $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]);
+
         $passwordHistory = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
         $password = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS);
         try {
@@ -174,8 +178,6 @@ App::post('/v1/account/sessions/email')
     ->label('audits.event', 'session.create')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
-    ->label('usage.metric', 'sessions.{scope}.requests.create')
-    ->label('usage.params', ['provider:email'])
     ->label('sdk.auth', [])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createEmailSession')
@@ -195,7 +197,8 @@ App::post('/v1/account/sessions/email')
     ->inject('locale')
     ->inject('geodb')
     ->inject('queueForEvents')
-    ->action(function (string $email, string $password, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents) {
+    ->inject('hooks')
+    ->action(function (string $email, string $password, Request $request, Response $response, Document $user, Database $dbForProject, Document $project, Locale $locale, Reader $geodb, Event $queueForEvents, Hooks $hooks) {
 
         $email = \strtolower($email);
         $protocol = $request->getProtocol();
@@ -214,6 +217,8 @@ App::post('/v1/account/sessions/email')
 
         $user->setAttributes($profile->getArrayCopy());
 
+        $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, false]);
+
         $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
 
         $detector = new Detector($request->getUserAgent('UNKNOWN'));
@@ -428,8 +433,6 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
     ->label('abuse-limit', 50)
     ->label('abuse-key', 'ip:{ip}')
     ->label('docs', false)
-    ->label('usage.metric', 'sessions.{scope}.requests.create')
-    ->label('usage.params', ['provider:{request.provider}'])
     ->param('provider', '', new WhiteList(\array_keys(Config::getParam('providers')), true), 'OAuth2 provider.')
     ->param('code', '', new Text(2048, 0), 'OAuth2 code. This is a temporary code that the will be later exchanged for an access token.', true)
     ->param('state', '', new Text(2048), 'OAuth2 state params.', true)
@@ -550,11 +553,19 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
         if (!$user->isEmpty()) {
             $userId = $user->getId();
 
-            $identitiesWithMatchingEmail = $dbForProject->find('identities', [
+            $identityWithMatchingEmail = $dbForProject->findOne('identities', [
                 Query::equal('providerEmail', [$email]),
                 Query::notEqual('userId', $userId),
             ]);
-            if (!empty($identitiesWithMatchingEmail)) {
+            if (!empty($identityWithMatchingEmail)) {
+                throw new Exception(Exception::USER_ALREADY_EXISTS);
+            }
+
+            $userWithMatchingEmail = $dbForProject->find('users', [
+                Query::equal('email', [$email]),
+                Query::notEqual('$id', $userId),
+            ]);
+            if (!empty($userWithMatchingEmail)) {
                 throw new Exception(Exception::USER_ALREADY_EXISTS);
             }
         }
@@ -794,7 +805,6 @@ App::get('/v1/account/identities')
     ->desc('List Identities')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'listIdentities')
@@ -842,14 +852,13 @@ App::get('/v1/account/identities')
     });
 
 App::delete('/v1/account/identities/:identityId')
-    ->desc('Delete Identity')
+    ->desc('Delete identity')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
     ->label('event', 'users.[userId].identities.[identityId].delete')
     ->label('audits.event', 'identity.delete')
     ->label('audits.resource', 'identity/{request.$identityId}')
     ->label('audits.userId', '{user.$id}')
-    ->label('usage.metric', 'identities.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'deleteIdentity')
@@ -859,7 +868,8 @@ App::delete('/v1/account/identities/:identityId')
     ->param('identityId', '', new UID(), 'Identity ID.')
     ->inject('response')
     ->inject('dbForProject')
-    ->action(function (string $identityId, Response $response, Database $dbForProject) {
+    ->inject('queueForEvents')
+    ->action(function (string $identityId, Response $response, Database $dbForProject, Event $queueForEvents) {
 
         $identity = $dbForProject->getDocument('identities', $identityId);
 
@@ -869,12 +879,17 @@ App::delete('/v1/account/identities/:identityId')
 
         $dbForProject->deleteDocument('identities', $identityId);
 
+        $queueForEvents
+            ->setParam('userId', $identity->getAttribute('userId'))
+            ->setParam('identityId', $identity->getId())
+            ->setPayload($response->output($identity, Response::MODEL_IDENTITY));
+
         return $response->noContent();
     });
 
 App::post('/v1/account/sessions/magic-url')
     ->desc('Create magic URL session')
-    ->groups(['api', 'account'])
+    ->groups(['api', 'account', 'auth'])
     ->label('scope', 'public')
     ->label('auth.type', 'magic-url')
     ->label('audits.event', 'session.create')
@@ -887,8 +902,8 @@ App::post('/v1/account/sessions/magic-url')
     ->label('sdk.response.code', Response::STATUS_CODE_CREATED)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_TOKEN)
-    ->label('abuse-limit', 10)
-    ->label('abuse-key', 'url:{url},email:{param-email}')
+    ->label('abuse-limit', 60)
+    ->label('abuse-key', ['url:{url},email:{param-email}', 'url:{url},ip:{ip}'])
     ->param('userId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.')
     ->param('email', '', new Email(), 'User email.')
     ->param('url', '', fn($clients) => new Host($clients), 'URL to redirect the user back to your app from the magic URL login. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients'])
@@ -1001,7 +1016,7 @@ App::post('/v1/account/sessions/magic-url')
 
         $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
         $message
-            ->setParam('{{body}}', $body)
+            ->setParam('{{body}}', $body, escapeHtml: false)
             ->setParam('{{hello}}', $locale->getText("emails.magicSession.hello"))
             ->setParam('{{footer}}', $locale->getText("emails.magicSession.footer"))
             ->setParam('{{thanks}}', $locale->getText("emails.magicSession.thanks"))
@@ -1056,11 +1071,11 @@ App::post('/v1/account/sessions/magic-url')
 
         $emailVariables = [
             'direction' => $locale->getText('settings.direction'),
-            /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
+            /* {{user}}, {{team}}, {{redirect}} and {{project}} are required in default and custom templates */
             'user' => '',
             'team' => '',
-            'project' => $project->getAttribute('name'),
-            'redirect' => $url
+            'redirect' => $url,
+            'project' => $project->getAttribute('name')
         ];
 
         $queueForMails
@@ -1094,8 +1109,6 @@ App::put('/v1/account/sessions/magic-url')
     ->label('audits.event', 'session.update')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
-    ->label('usage.metric', 'sessions.{scope}.requests.create')
-    ->label('usage.params', ['provider:magic-url'])
     ->label('sdk.auth', [])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateMagicURLSession')
@@ -1210,7 +1223,7 @@ App::put('/v1/account/sessions/magic-url')
 
 App::post('/v1/account/sessions/phone')
     ->desc('Create phone session')
-    ->groups(['api', 'account'])
+    ->groups(['api', 'account', 'auth'])
     ->label('scope', 'public')
     ->label('auth.type', 'phone')
     ->label('audits.event', 'session.create')
@@ -1224,7 +1237,7 @@ App::post('/v1/account/sessions/phone')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_TOKEN)
     ->label('abuse-limit', 10)
-    ->label('abuse-key', 'url:{url},phone:{param-phone}')
+    ->label('abuse-key', ['url:{url},phone:{param-phone}', 'url:{url},ip:{ip}'])
     ->param('userId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.')
     ->param('phone', '', new Phone(), 'Phone number. Format this number with a leading \'+\' and a country code, e.g., +16175551212.')
     ->inject('request')
@@ -1323,9 +1336,12 @@ App::post('/v1/account/sessions/phone')
         $message = $message->setParam('{{token}}', $secret);
         $message = $message->render();
 
+        var_dump($request->getIP());
+        var_dump($project->getId());
         $queueForMessaging
             ->setRecipient($phone)
             ->setMessage($message)
+            ->setProject($project)
             ->trigger();
 
         $queueForEvents->setPayload(
@@ -1469,8 +1485,6 @@ App::post('/v1/account/sessions/anonymous')
     ->label('audits.event', 'session.create')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
-    ->label('usage.metric', 'sessions.{scope}.requests.create')
-    ->label('usage.params', ['provider:anonymous'])
     ->label('sdk.auth', [])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createAnonymousSession')
@@ -1648,7 +1662,6 @@ App::get('/v1/account')
     ->desc('Get account')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'get')
@@ -1669,7 +1682,6 @@ App::get('/v1/account/prefs')
     ->desc('Get account preferences')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'getPrefs')
@@ -1692,7 +1704,6 @@ App::get('/v1/account/sessions')
     ->desc('List sessions')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'listSessions')
@@ -1731,7 +1742,6 @@ App::get('/v1/account/logs')
     ->desc('List logs')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'listLogs')
@@ -1792,7 +1802,6 @@ App::get('/v1/account/sessions/:sessionId')
     ->desc('Get session')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'getSession')
@@ -1840,7 +1849,6 @@ App::patch('/v1/account/name')
     ->label('scope', 'account')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateName')
@@ -1875,7 +1883,6 @@ App::patch('/v1/account/password')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updatePassword')
@@ -1893,7 +1900,8 @@ App::patch('/v1/account/password')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $password, string $oldPassword, ?\DateTime $requestTimestamp, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents) {
+    ->inject('hooks')
+    ->action(function (string $password, string $oldPassword, ?\DateTime $requestTimestamp, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
 
         // Check old password only if its an existing user.
         if (!empty($user->getAttribute('passwordUpdate')) && !Auth::passwordVerify($oldPassword, $user->getAttribute('password'), $user->getAttribute('hash'), $user->getAttribute('hashOptions'))) { // Double check user password
@@ -1920,6 +1928,8 @@ App::patch('/v1/account/password')
             }
         }
 
+        $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]);
+
         $user
             ->setAttribute('password', $newPassword)
             ->setAttribute('passwordHistory', $history)
@@ -1941,7 +1951,6 @@ App::patch('/v1/account/email')
     ->label('scope', 'account')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateEmail')
@@ -1958,7 +1967,9 @@ App::patch('/v1/account/email')
     ->inject('user')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $email, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) {
+    ->inject('project')
+    ->inject('hooks')
+    ->action(function (string $email, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Document $project, Hooks $hooks) {
         // passwordUpdate will be empty if the user has never set a password
         $passwordUpdate = $user->getAttribute('passwordUpdate');
 
@@ -1969,6 +1980,8 @@ App::patch('/v1/account/email')
             throw new Exception(Exception::USER_INVALID_CREDENTIALS);
         }
 
+        $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, false]);
+
         $email = \strtolower($email);
 
         // Makes sure this email is not already used in another identity
@@ -2011,7 +2024,6 @@ App::patch('/v1/account/phone')
     ->label('scope', 'account')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updatePhone')
@@ -2028,7 +2040,9 @@ App::patch('/v1/account/phone')
     ->inject('user')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $phone, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents) {
+    ->inject('project')
+    ->inject('hooks')
+    ->action(function (string $phone, string $password, ?\DateTime $requestTimestamp, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Document $project, Hooks $hooks) {
         // passwordUpdate will be empty if the user has never set a password
         $passwordUpdate = $user->getAttribute('passwordUpdate');
 
@@ -2039,6 +2053,8 @@ App::patch('/v1/account/phone')
             throw new Exception(Exception::USER_INVALID_CREDENTIALS);
         }
 
+        $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, false]);
+
         $user
             ->setAttribute('phone', $phone)
             ->setAttribute('phoneVerification', false) // After this user needs to confirm phone number again
@@ -2070,7 +2086,6 @@ App::patch('/v1/account/prefs')
     ->label('scope', 'account')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updatePrefs')
@@ -2104,7 +2119,6 @@ App::patch('/v1/account/status')
     ->label('scope', 'account')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateStatus')
@@ -2148,7 +2162,6 @@ App::delete('/v1/account/sessions/:sessionId')
     ->label('event', 'users.[userId].sessions.[sessionId].delete')
     ->label('audits.event', 'session.delete')
     ->label('audits.resource', 'user/{user.$id}')
-    ->label('usage.metric', 'sessions.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'deleteSession')
@@ -2225,7 +2238,6 @@ App::patch('/v1/account/sessions/:sessionId')
     ->label('audits.event', 'session.update')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
-    ->label('usage.metric', 'sessions.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateSession')
@@ -2310,7 +2322,6 @@ App::delete('/v1/account/sessions')
     ->label('event', 'users.[userId].sessions.[sessionId].delete')
     ->label('audits.event', 'session.delete')
     ->label('audits.resource', 'user/{user.$id}')
-    ->label('usage.metric', 'sessions.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'deleteSessions')
@@ -2372,7 +2383,6 @@ App::post('/v1/account/recovery')
     ->label('audits.event', 'recovery.create')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createRecovery')
@@ -2381,7 +2391,7 @@ App::post('/v1/account/recovery')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_TOKEN)
     ->label('abuse-limit', 10)
-    ->label('abuse-key', ['url:{url},email:{param-email}', 'ip:{ip}'])
+    ->label('abuse-key', ['url:{url},email:{param-email}', 'url:{url},ip:{ip}'])
     ->param('email', '', new Email(), 'User email.')
     ->param('url', '', fn ($clients) => new Host($clients), 'URL to redirect the user back to your app from the recovery email. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', false, ['clients'])
     ->inject('request')
@@ -2454,7 +2464,7 @@ App::post('/v1/account/recovery')
 
         $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
         $message
-            ->setParam('{{body}}', $body)
+            ->setParam('{{body}}', $body, escapeHtml: false)
             ->setParam('{{hello}}', $locale->getText("emails.recovery.hello"))
             ->setParam('{{footer}}', $locale->getText("emails.recovery.footer"))
             ->setParam('{{thanks}}', $locale->getText("emails.recovery.thanks"))
@@ -2509,11 +2519,11 @@ App::post('/v1/account/recovery')
 
         $emailVariables = [
             'direction' => $locale->getText('settings.direction'),
-            /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
+            /* {{user}}, {{team}}, {{redirect}} and {{project}} are required in default and custom templates */
             'user' => $profile->getAttribute('name'),
             'team' => '',
-            'project' => $projectName,
-            'redirect' => $url
+            'redirect' => $url,
+            'project' => $projectName
         ];
 
         $queueForMails
@@ -2550,7 +2560,6 @@ App::put('/v1/account/recovery')
     ->label('audits.event', 'recovery.update')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateRecovery')
@@ -2569,7 +2578,8 @@ App::put('/v1/account/recovery')
     ->inject('dbForProject')
     ->inject('project')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $secret, string $password, string $passwordAgain, Response $response, Document $user, Database $dbForProject, Document $project, Event $queueForEvents) {
+    ->inject('hooks')
+    ->action(function (string $userId, string $secret, string $password, string $passwordAgain, Response $response, Document $user, Database $dbForProject, Document $project, Event $queueForEvents, Hooks $hooks) {
         if ($password !== $passwordAgain) {
             throw new Exception(Exception::USER_PASSWORD_MISMATCH);
         }
@@ -2603,6 +2613,8 @@ App::put('/v1/account/recovery')
             $history = array_slice($history, (count($history) - $historyLimit), $historyLimit);
         }
 
+        $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]);
+
         $profile = $dbForProject->updateDocument('users', $profile->getId(), $profile
                 ->setAttribute('password', $newPassword)
                 ->setAttribute('passwordHistory', $history)
@@ -2637,7 +2649,6 @@ App::post('/v1/account/verification')
     ->label('event', 'users.[userId].verification.[tokenId].create')
     ->label('audits.event', 'verification.create')
     ->label('audits.resource', 'user/{response.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createVerification')
@@ -2662,6 +2673,10 @@ App::post('/v1/account/verification')
             throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP Disabled');
         }
 
+        if ($user->getAttribute('emailVerification')) {
+            throw new Exception(Exception::USER_EMAIL_ALREADY_VERIFIED);
+        }
+
         $roles = Authorization::getRoles();
         $isPrivilegedUser = Auth::isPrivilegedUser($roles);
         $isAppUser = Auth::isAppUser($roles);
@@ -2701,7 +2716,7 @@ App::post('/v1/account/verification')
 
         $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
         $message
-            ->setParam('{{body}}', $body)
+            ->setParam('{{body}}', $body, escapeHtml: false)
             ->setParam('{{hello}}', $locale->getText("emails.verification.hello"))
             ->setParam('{{footer}}', $locale->getText("emails.verification.footer"))
             ->setParam('{{thanks}}', $locale->getText("emails.verification.thanks"))
@@ -2756,11 +2771,11 @@ App::post('/v1/account/verification')
 
         $emailVariables = [
             'direction' => $locale->getText('settings.direction'),
-            /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
+            /* {{user}}, {{team}}, {{redirect}} and {{project}} are required in default and custom templates */
             'user' => $user->getAttribute('name'),
             'team' => '',
-            'project' => $projectName,
-            'redirect' => $url
+            'redirect' => $url,
+            'project' => $projectName
         ];
 
         $queueForMails
@@ -2794,7 +2809,6 @@ App::put('/v1/account/verification')
     ->label('event', 'users.[userId].verification.[tokenId].update')
     ->label('audits.event', 'verification.update')
     ->label('audits.resource', 'user/{response.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateVerification')
@@ -2850,12 +2864,12 @@ App::put('/v1/account/verification')
 
 App::post('/v1/account/verification/phone')
     ->desc('Create phone verification')
-    ->groups(['api', 'account'])
+    ->groups(['api', 'account', 'auth'])
     ->label('scope', 'account')
+    ->label('auth.type', 'phone')
     ->label('event', 'users.[userId].verification.[tokenId].create')
     ->label('audits.event', 'verification.create')
     ->label('audits.resource', 'user/{response.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createPhoneVerification')
@@ -2864,7 +2878,7 @@ App::post('/v1/account/verification/phone')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_TOKEN)
     ->label('abuse-limit', 10)
-    ->label('abuse-key', 'userId:{userId}')
+    ->label('abuse-key', ['url:{url},userId:{userId}', 'url:{url},ip:{ip}'])
     ->inject('request')
     ->inject('response')
     ->inject('user')
@@ -2883,6 +2897,10 @@ App::post('/v1/account/verification/phone')
             throw new Exception(Exception::USER_PHONE_NOT_FOUND);
         }
 
+        if ($user->getAttribute('phoneVerification')) {
+            throw new Exception(Exception::USER_PHONE_ALREADY_VERIFIED);
+        }
+
         $roles = Authorization::getRoles();
         $isPrivilegedUser = Auth::isPrivilegedUser($roles);
         $isAppUser = Auth::isAppUser($roles);
@@ -2921,9 +2939,12 @@ App::post('/v1/account/verification/phone')
         $message = $message->setParam('{{token}}', $secret);
         $message = $message->render();
 
+        var_dump($request->getIP());
+        var_dump($project->getId());
         $queueForMessaging
             ->setRecipient($user->getAttribute('phone'))
             ->setMessage($message)
+            ->setProject($project)
             ->trigger()
         ;
 
@@ -2951,7 +2972,6 @@ App::put('/v1/account/verification/phone')
     ->label('event', 'users.[userId].verification.[tokenId].update')
     ->label('audits.event', 'verification.update')
     ->label('audits.resource', 'user/{response.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updatePhoneVerification')
@@ -3002,3 +3022,40 @@ App::put('/v1/account/verification/phone')
 
         $response->dynamic($verificationDocument, Response::MODEL_TOKEN);
     });
+
+App::delete('/v1/account')
+    ->desc('Delete account')
+    ->groups(['api', 'account'])
+    ->label('event', 'users.[userId].delete')
+    ->label('scope', 'account')
+    ->label('audits.event', 'user.delete')
+    ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.delete')
+    ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
+    ->label('sdk.namespace', 'account')
+    ->label('sdk.method', 'delete')
+    ->label('sdk.description', '/docs/references/account/delete.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT)
+    ->label('sdk.response.model', Response::MODEL_NONE)
+    ->inject('user')
+    ->inject('response')
+    ->inject('dbForProject')
+    ->inject('queueForEvents')
+    ->inject('queueForDeletes')
+    ->action(function (Document $user, Response $response, Database $dbForProject, Event $queueForEvents, Delete $queueForDeletes) {
+        if ($user->isEmpty()) {
+            throw new Exception(Exception::USER_NOT_FOUND);
+        }
+
+        $dbForProject->deleteDocument('users', $user->getId());
+
+        $queueForDeletes
+            ->setType(DELETE_TYPE_DOCUMENT)
+            ->setDocument($user);
+
+        $queueForEvents
+            ->setParam('userId', $user->getId())
+            ->setPayload($response->output($user, Response::MODEL_USER));
+
+        $response->noContent();
+    });

app/controllers/api/databases.php

@@ -402,7 +402,6 @@ App::post('/v1/databases')
     ->label('scope', 'databases.write')
     ->label('audits.event', 'database.create')
     ->label('audits.resource', 'database/{response.$id}')
-    ->label('usage.metric', 'databases.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'create')
@@ -476,7 +475,6 @@ App::get('/v1/databases')
     ->desc('List databases')
     ->groups(['api', 'database'])
     ->label('scope', 'databases.read')
-    ->label('usage.metric', 'databases.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'list')
@@ -524,7 +522,6 @@ App::get('/v1/databases/:databaseId')
     ->desc('Get database')
     ->groups(['api', 'database'])
     ->label('scope', 'databases.read')
-    ->label('usage.metric', 'databases.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'get')
@@ -639,7 +636,6 @@ App::put('/v1/databases/:databaseId')
     ->label('event', 'databases.[databaseId].update')
     ->label('audits.event', 'database.update')
     ->label('audits.resource', 'database/{response.$id}')
-    ->label('usage.metric', 'databases.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'update')
@@ -684,7 +680,6 @@ App::delete('/v1/databases/:databaseId')
     ->label('event', 'databases.[databaseId].delete')
     ->label('audits.event', 'database.delete')
     ->label('audits.resource', 'database/{request.databaseId}')
-    ->label('usage.metric', 'databases.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'delete')
@@ -729,8 +724,6 @@ App::post('/v1/databases/:databaseId/collections')
     ->label('scope', 'collections.write')
     ->label('audits.event', 'collection.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{response.$id}')
-    ->label('usage.metric', 'collections.{scope}.requests.create')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'createCollection')
@@ -796,8 +789,6 @@ App::get('/v1/databases/:databaseId/collections')
     ->desc('List collections')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
-    ->label('usage.metric', 'collections.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listCollections')
@@ -855,8 +846,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId')
     ->desc('Get collection')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
-    ->label('usage.metric', 'collections.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'getCollection')
@@ -891,8 +880,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs')
     ->desc('List collection logs')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
-    ->label('usage.metric', 'collections.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listCollectionLogs')
@@ -990,8 +977,6 @@ App::put('/v1/databases/:databaseId/collections/:collectionId')
     ->label('event', 'databases.[databaseId].collections.[collectionId].update')
     ->label('audits.event', 'collection.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateCollection')
@@ -1060,8 +1045,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId')
     ->label('event', 'databases.[databaseId].collections.[collectionId].delete')
     ->label('audits.event', 'collection.delete')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.delete')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'deleteCollection')
@@ -1117,8 +1100,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/string
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'createStringAttribute')
@@ -1175,8 +1156,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/email'
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createEmailAttribute')
@@ -1219,8 +1198,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createEnumAttribute')
@@ -1231,7 +1208,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
     ->param('databaseId', '', new UID(), 'Database ID.')
     ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).')
     ->param('key', '', new Key(), 'Attribute Key.')
-    ->param('elements', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE, min: 0), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.')
+    ->param('elements', [], new ArrayList(new Text(DATABASE::LENGTH_KEY), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . DATABASE::LENGTH_KEY . ' characters long.')
     ->param('required', null, new Boolean(), 'Is attribute required?')
     ->param('default', null, new Text(0), 'Default value for attribute when not provided. Cannot be set when attribute is required.', true)
     ->param('array', false, new Boolean(), 'Is attribute an array?', true)
@@ -1240,16 +1217,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
     ->inject('queueForDatabase')
     ->inject('queueForEvents')
     ->action(function (string $databaseId, string $collectionId, string $key, array $elements, ?bool $required, ?string $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) {
-        // use length of longest string as attribute size
-        $size = 0;
-        foreach ($elements as $element) {
-            $length = \strlen($element);
-            if ($length === 0) {
-                throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Each enum element must not be empty');
-            }
-            $size = ($length > $size) ? $length : $size;
-        }
-
         if (!is_null($default) && !in_array($default, $elements)) {
             throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Default value not found in elements');
         }
@@ -1257,7 +1224,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
         $attribute = createAttribute($databaseId, $collectionId, new Document([
             'key' => $key,
             'type' => Database::VAR_STRING,
-            'size' => $size,
+            'size' => Database::LENGTH_KEY,
             'required' => $required,
             'default' => $default,
             'array' => $array,
@@ -1278,8 +1245,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/ip')
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createIpAttribute')
@@ -1322,8 +1287,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/url')
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createUrlAttribute')
@@ -1366,8 +1329,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/intege
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createIntegerAttribute')
@@ -1439,8 +1400,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/float'
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createFloatAttribute')
@@ -1515,8 +1474,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/boolea
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createBooleanAttribute')
@@ -1558,8 +1515,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/dateti
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createDatetimeAttribute')
@@ -1604,8 +1559,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/relati
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createRelationshipAttribute')
@@ -1683,8 +1636,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes')
     ->desc('List attributes')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
-    ->label('usage.metric', 'collections.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listAttributes')
@@ -1758,8 +1709,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes/:key')
     ->desc('Get attribute')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
-    ->label('usage.metric', 'collections.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'getAttribute')
@@ -1837,8 +1786,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/strin
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateStringAttribute')
@@ -1878,8 +1825,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/email
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateEmailAttribute')
@@ -1919,8 +1864,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/enum/
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateEnumAttribute')
@@ -1930,7 +1873,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/enum/
     ->param('databaseId', '', new UID(), 'Database ID.')
     ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).')
     ->param('key', '', new Key(), 'Attribute Key.')
-    ->param('elements', null, new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.')
+    ->param('elements', null, new ArrayList(new Text(DATABASE::LENGTH_KEY), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . DATABASE::LENGTH_KEY . ' characters long.')
     ->param('required', null, new Boolean(), 'Is attribute required?')
     ->param('default', null, new Nullable(new Text(0)), 'Default value for attribute when not provided. Cannot be set when attribute is required.')
     ->inject('response')
@@ -1962,8 +1905,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/ip/:k
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateIpAttribute')
@@ -2003,8 +1944,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/url/:
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateUrlAttribute')
@@ -2044,8 +1983,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/integ
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateIntegerAttribute')
@@ -2095,8 +2032,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/float
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateFloatAttribute')
@@ -2146,8 +2081,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/boole
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateBooleanAttribute')
@@ -2186,8 +2119,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/datet
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'documents.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateDatetimeAttribute')
@@ -2226,8 +2157,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/:key/
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'documents.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateRelationshipAttribute')
@@ -2282,8 +2211,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.delete')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'deleteAttribute')
@@ -2393,8 +2320,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes')
     ->label('scope', 'collections.write')
     ->label('audits.event', 'index.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'createIndex')
@@ -2553,8 +2478,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes')
     ->desc('List indexes')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
-    ->label('usage.metric', 'collections.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listIndexes')
@@ -2618,8 +2541,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes/:key')
     ->desc('Get index')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
-    ->label('usage.metric', 'collections.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'getIndex')
@@ -2662,8 +2583,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/indexes/:key')
     ->label('event', 'databases.[databaseId].collections.[collectionId].indexes.[indexId].update')
     ->label('audits.event', 'index.delete')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'deleteIndex')
@@ -2728,8 +2647,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents')
     ->label('scope', 'documents.write')
     ->label('audits.event', 'document.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'documents.{scope}.requests.create')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -2966,8 +2883,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents')
     ->desc('List documents')
     ->groups(['api', 'database'])
     ->label('scope', 'documents.read')
-    ->label('usage.metric', 'documents.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listDocuments')
@@ -3093,8 +3008,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
     ->desc('Get document')
     ->groups(['api', 'database'])
     ->label('scope', 'documents.read')
-    ->label('usage.metric', 'documents.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'getDocument')
@@ -3188,8 +3101,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
     ->desc('List document logs')
     ->groups(['api', 'database'])
     ->label('scope', 'documents.read')
-    ->label('usage.metric', 'documents.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listDocumentLogs')
@@ -3292,8 +3203,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum
     ->label('scope', 'documents.write')
     ->label('audits.event', 'document.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}')
-    ->label('usage.metric', 'documents.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -3522,8 +3431,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
     ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].delete')
     ->label('audits.event', 'document.delete')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{request.documentId}')
-    ->label('usage.metric', 'documents.{scope}.requests.delete')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -3632,7 +3539,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
 
 App::get('/v1/databases/usage')
     ->desc('Get usage stats for the database')
-    ->groups(['api', 'database'])
+    ->groups(['api', 'database', 'usage'])
     ->label('scope', 'collections.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'databases')
@@ -3640,117 +3547,77 @@ App::get('/v1/databases/usage')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_DATABASES)
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), '`Date range.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), '`Date range.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (string $range, Response $response, Database $dbForProject) {
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            METRIC_DATABASES,
+            METRIC_COLLECTIONS,
+            METRIC_DOCUMENTS,
+        ];
 
-            $metrics = [
-                'databases.$all.count.total',
-                'documents.$all.count.total',
-                'collections.$all.count.total',
-                'databases.$all.requests.create',
-                'databases.$all.requests.read',
-                'databases.$all.requests.update',
-                'databases.$all.requests.delete',
-                'collections.$all.requests.create',
-                'collections.$all.requests.read',
-                'collections.$all.requests.update',
-                'collections.$all.requests.delete',
-                'documents.$all.requests.create',
-                'documents.$all.requests.read',
-                'documents.$all.requests.update',
-                'documents.$all.requests.delete'
-            ];
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
+            foreach ($metrics as $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
 
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    // Added 3'rd level to Index [period, metric, time] because of order by.
-                    $stats[$metric] = array_reverse($stats[$metric]);
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'databasesCount' => $stats['databases.$all.count.total'] ?? [],
-                'documentsCount' => $stats['documents.$all.count.total'] ?? [],
-                'collectionsCount' => $stats['collections.$all.count.total'] ?? [],
-                'documentsCreate' =>  $stats['documents.$all.requests.create'] ?? [],
-                'documentsRead' =>  $stats['documents.$all.requests.read'] ?? [],
-                'documentsUpdate' => $stats['documents.$all.requests.update'] ?? [],
-                'documentsDelete' => $stats['documents.$all.requests.delete'] ?? [],
-                'collectionsCreate' => $stats['collections.$all.requests.create'] ?? [],
-                'collectionsRead' =>  $stats['collections.$all.requests.read'] ?? [],
-                'collectionsUpdate' => $stats['collections.$all.requests.update'] ?? [],
-                'collectionsDelete' => $stats['collections.$all.requests.delete'] ?? [],
-                'databasesCreate' => $stats['databases.$all.requests.create'] ?? [],
-                'databasesRead' =>  $stats['databases.$all.requests.read'] ?? [],
-                'databasesUpdate' => $stats['databases.$all.requests.update'] ?? [],
-                'databasesDelete' => $stats['databases.$all.requests.delete'] ?? [],
-            ]);
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
         }
-
-        $response->dynamic($usage, Response::MODEL_USAGE_DATABASES);
+    }
+        $response->dynamic(new Document([
+            'range' => $range,
+            'databasesTotal'   => $usage[$metrics[0]]['total'],
+            'collectionsTotal' => $usage[$metrics[1]]['total'],
+            'documentsTotal'   => $usage[$metrics[2]]['total'],
+            'databases'   => $usage[$metrics[0]]['data'],
+            'collections' => $usage[$metrics[1]]['data'],
+            'documents'   => $usage[$metrics[2]]['data'],
+        ]), Response::MODEL_USAGE_DATABASES);
     });
 
 App::get('/v1/databases/:databaseId/usage')
     ->desc('Get usage stats for the database')
-    ->groups(['api', 'database'])
+    ->groups(['api', 'database', 'usage'])
     ->label('scope', 'collections.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'databases')
@@ -3759,108 +3626,82 @@ App::get('/v1/databases/:databaseId/usage')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_DATABASE)
     ->param('databaseId', '', new UID(), 'Database ID.')
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), '`Date range.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), '`Date range.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (string $databaseId, string $range, Response $response, Database $dbForProject) {
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $database =  $dbForProject->getDocument('databases', $databaseId);
 
-            $metrics = [
-                'collections.' . $databaseId . '.count.total',
-                'collections.' . $databaseId . '.requests.create',
-                'collections.' . $databaseId . '.requests.read',
-                'collections.' . $databaseId . '.requests.update',
-                'collections.' . $databaseId . '.requests.delete',
-                'documents.' . $databaseId . '.count.total',
-                'documents.' . $databaseId . '.requests.create',
-                'documents.' . $databaseId . '.requests.read',
-                'documents.' . $databaseId . '.requests.update',
-                'documents.' . $databaseId . '.requests.delete'
-            ];
-
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    // TODO@kodumbeats explore performance if query is ordered by time ASC
-                    $stats[$metric] = array_reverse($stats[$metric]);
-                }
-            });
-
-            $usage = new Document([
-                'range' => $range,
-                'collectionsCount' => $stats["collections.{$databaseId}.count.total"] ?? [],
-                'collectionsCreate' => $stats["collections.{$databaseId}.requests.create"] ?? [],
-                'collectionsRead' =>  $stats["collections.{$databaseId}.requests.read"] ?? [],
-                'collectionsUpdate' => $stats["collections.{$databaseId}.requests.update"] ?? [],
-                'collectionsDelete' => $stats["collections.{$databaseId}.requests.delete"] ?? [],
-                'documentsCount' => $stats["documents.{$databaseId}.count.total"] ?? [],
-                'documentsCreate' =>  $stats["documents.{$databaseId}.requests.create"] ?? [],
-                'documentsRead' =>  $stats["documents.{$databaseId}.requests.read"] ?? [],
-                'documentsUpdate' => $stats["documents.{$databaseId}.requests.update"] ?? [],
-                'documentsDelete' => $stats["documents.{$databaseId}.requests.delete"] ?? [],
-            ]);
+        if ($database->isEmpty()) {
+            throw new Exception(Exception::DATABASE_NOT_FOUND);
         }
 
-        $response->dynamic($usage, Response::MODEL_USAGE_DATABASE);
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS),
+            str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS),
+        ];
+
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
+            foreach ($metrics as $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
+
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
+                }
+            }
+        });
+
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
+        }
+    }
+
+        $response->dynamic(new Document([
+            'range' => $range,
+            'collectionsTotal'   => $usage[$metrics[0]]['total'],
+            'documentsTotal'   => $usage[$metrics[1]]['total'],
+            'collections'   => $usage[$metrics[0]]['data'],
+            'documents'   => $usage[$metrics[1]]['data'],
+        ]), Response::MODEL_USAGE_DATABASE);
     });
 
 App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
     ->alias('/v1/database/:collectionId/usage', ['databaseId' => 'default'])
     ->desc('Get usage stats for a collection')
-    ->groups(['api', 'database'])
+    ->groups(['api', 'database', 'usage'])
     ->label('scope', 'collections.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'databases')
@@ -3869,7 +3710,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_COLLECTION)
     ->param('databaseId', '', new UID(), 'Database ID.')
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true)
     ->param('collectionId', '', new UID(), 'Collection ID.')
     ->inject('response')
     ->inject('dbForProject')
@@ -3883,84 +3724,60 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
             throw new Exception(Exception::COLLECTION_NOT_FOUND);
         }
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collectionDocument->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS),
+        ];
 
-            $metrics = [
-                "documents.{$databaseId}/{$collectionId}.count.total",
-                "documents.{$databaseId}/{$collectionId}.requests.create",
-                "documents.{$databaseId}/{$collectionId}.requests.read",
-                "documents.{$databaseId}/{$collectionId}.requests.update",
-                "documents.{$databaseId}/{$collectionId}.requests.delete",
-            ];
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
+            foreach ($metrics as $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
 
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' =>  $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'documentsCount' => $stats["documents.{$databaseId}/{$collectionId}.count.total"] ?? [],
-                'documentsCreate' => $stats["documents.{$databaseId}/{$collectionId}.requests.create"] ?? [],
-                'documentsRead' => $stats["documents.{$databaseId}/{$collectionId}.requests.read"] ?? [],
-                'documentsUpdate' =>  $stats["documents.{$databaseId}/{$collectionId}.requests.update"] ?? [],
-                'documentsDelete' =>  $stats["documents.{$databaseId}/{$collectionId}.requests.delete" ?? []]
-            ]);
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
         }
+    }
 
-        $response->dynamic($usage, Response::MODEL_USAGE_COLLECTION);
+        $response->dynamic(new Document([
+            'range' => $range,
+            'documentsTotal'   => $usage[$metrics[0]]['total'],
+            'documents'   =>  $usage[$metrics[0]]['data'],
+        ]), Response::MODEL_USAGE_COLLECTION);
     });

app/controllers/api/functions.php

@@ -6,13 +6,13 @@ use Appwrite\Event\Build;
 use Appwrite\Event\Delete;
 use Appwrite\Event\Event;
 use Appwrite\Event\Func;
+use Appwrite\Event\Usage;
 use Appwrite\Event\Validator\FunctionEvent;
 use Appwrite\Utopia\Response\Model\Rule;
 use Appwrite\Extend\Exception;
 use Appwrite\Utopia\Database\Validator\CustomId;
 use Appwrite\Messaging\Adapter\Realtime;
 use Utopia\Validator\Assoc;
-use Appwrite\Usage\Stats;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Helpers\Permission;
 use Utopia\Database\Helpers\Role;
@@ -242,12 +242,16 @@ App::post('/v1/functions')
 
         // Git connect logic
         if (!empty($providerRepositoryId)) {
+            $teamId = $project->getAttribute('teamId', '');
+
             $repository = $dbForConsole->createDocument('repositories', new Document([
                 '$id' => ID::unique(),
                 '$permissions' => [
-                    Permission::read(Role::any()),
-                    Permission::update(Role::any()),
-                    Permission::delete(Role::any()),
+                    Permission::read(Role::team(ID::custom($teamId))),
+                    Permission::update(Role::team(ID::custom($teamId), 'owner')),
+                    Permission::update(Role::team(ID::custom($teamId), 'developer')),
+                    Permission::delete(Role::team(ID::custom($teamId), 'owner')),
+                    Permission::delete(Role::team(ID::custom($teamId), 'developer')),
                 ],
                 'installationId' => $installation->getId(),
                 'installationInternalId' => $installation->getInternalId(),
@@ -452,9 +456,9 @@ App::get('/v1/functions/:functionId/usage')
     ->label('sdk.method', 'getFunctionUsage')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS)
+    ->label('sdk.response.model', Response::MODEL_USAGE_FUNCTION)
     ->param('functionId', '', new UID(), 'Function ID.')
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d']), 'Date range.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (string $functionId, string $range, Response $response, Database $dbForProject) {
@@ -465,97 +469,85 @@ App::get('/v1/functions/:functionId/usage')
             throw new Exception(Exception::FUNCTION_NOT_FOUND);
         }
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS),
+            str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE),
+            str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS),
+            str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE),
+            str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE),
+            str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS),
+            str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE),
+        ];
 
-            $metrics = [
-                "executions.$functionId.compute.total",
-                "executions.$functionId.compute.success",
-                "executions.$functionId.compute.failure",
-                "executions.$functionId.compute.time",
-                "builds.$functionId.compute.total",
-                "builds.$functionId.compute.success",
-                "builds.$functionId.compute.failure",
-                "builds.$functionId.compute.time",
-            ];
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
+            foreach ($metrics as $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
 
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'executionsTotal' => $stats["executions.$functionId.compute.total"] ?? [],
-                'executionsFailure' => $stats["executions.$functionId.compute.failure"] ?? [],
-                'executionsSuccess' => $stats["executions.$functionId.compute.success"] ?? [],
-                'executionsTime' => $stats["executions.$functionId.compute.time"] ?? [],
-                'buildsTotal' => $stats["builds.$functionId.compute.total"] ?? [],
-                'buildsFailure' => $stats["builds.$functionId.compute.failure"] ?? [],
-                'buildsSuccess' => $stats["builds.$functionId.compute.success"] ?? [],
-                'buildsTime' => $stats["builds.$functionId.compute.time" ?? []]
-            ]);
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
         }
+    }
 
-        $response->dynamic($usage, Response::MODEL_USAGE_FUNCTION);
+        $response->dynamic(new Document([
+            'range' => $range,
+            'deploymentsTotal' => $usage[$metrics[0]]['total'],
+            'deploymentsStorageTotal' => $usage[$metrics[1]]['total'],
+            'buildsTotal' => $usage[$metrics[2]]['total'],
+            'buildsStorageTotal' => $usage[$metrics[3]]['total'],
+            'buildsTimeTotal' => $usage[$metrics[4]]['total'],
+            'executionsTotal' => $usage[$metrics[5]]['total'],
+            'executionsTimeTotal' => $usage[$metrics[6]]['total'],
+            'deployments' => $usage[$metrics[0]]['data'],
+            'deploymentsStorage' => $usage[$metrics[1]]['data'],
+            'builds' => $usage[$metrics[2]]['data'],
+            'buildsStorage' => $usage[$metrics[3]]['data'],
+            'buildsTime' => $usage[$metrics[4]]['data'],
+            'executions' => $usage[$metrics[5]]['data'],
+            'executionsTime' => $usage[$metrics[6]]['data'],
+        ]), Response::MODEL_USAGE_FUNCTION);
     });
 
 App::get('/v1/functions/usage')
     ->desc('Get functions usage')
-    ->groups(['api', 'functions', 'usage'])
+    ->groups(['api', 'functions'])
     ->label('scope', 'functions.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'functions')
@@ -563,97 +555,87 @@ App::get('/v1/functions/usage')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS)
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d']), 'Date range.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (string $range, Response $response, Database $dbForProject) {
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            METRIC_FUNCTIONS,
+            METRIC_DEPLOYMENTS,
+            METRIC_DEPLOYMENTS_STORAGE,
+            METRIC_BUILDS,
+            METRIC_BUILDS_STORAGE,
+            METRIC_BUILDS_COMPUTE,
+            METRIC_EXECUTIONS,
+            METRIC_EXECUTIONS_COMPUTE,
+        ];
 
-            $metrics = [
-                'executions.$all.compute.total',
-                'executions.$all.compute.failure',
-                'executions.$all.compute.success',
-                'executions.$all.compute.time',
-                'builds.$all.compute.total',
-                'builds.$all.compute.failure',
-                'builds.$all.compute.success',
-                'builds.$all.compute.time',
-            ];
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
+            foreach ($metrics as $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
 
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'executionsTotal' => $stats[$metrics[0]] ?? [],
-                'executionsFailure' => $stats[$metrics[1]] ?? [],
-                'executionsSuccess' => $stats[$metrics[2]] ?? [],
-                'executionsTime' => $stats[$metrics[3]] ?? [],
-                'buildsTotal' => $stats[$metrics[4]] ?? [],
-                'buildsFailure' => $stats[$metrics[5]] ?? [],
-                'buildsSuccess' => $stats[$metrics[6]] ?? [],
-                'buildsTime' => $stats[$metrics[7]] ?? [],
-            ]);
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
         }
-
-        $response->dynamic($usage, Response::MODEL_USAGE_FUNCTIONS);
+    }
+        $response->dynamic(new Document([
+            'range' => $range,
+            'functionsTotal' => $usage[$metrics[0]]['total'],
+            'deploymentsTotal' => $usage[$metrics[1]]['total'],
+            'deploymentsStorageTotal' => $usage[$metrics[2]]['total'],
+            'buildsTotal' => $usage[$metrics[3]]['total'],
+            'buildsStorageTotal' => $usage[$metrics[4]]['total'],
+            'buildsTimeTotal' => $usage[$metrics[5]]['total'],
+            'executionsTotal' => $usage[$metrics[6]]['total'],
+            'executionsTimeTotal' => $usage[$metrics[7]]['total'],
+            'functions' => $usage[$metrics[0]]['data'],
+            'deployments' => $usage[$metrics[1]]['data'],
+            'deploymentsStorage' => $usage[$metrics[2]]['data'],
+            'builds' => $usage[$metrics[3]]['data'],
+            'buildsStorage' => $usage[$metrics[4]]['data'],
+            'buildsTime' => $usage[$metrics[5]]['data'],
+            'executions' => $usage[$metrics[6]]['data'],
+            'executionsTime' => $usage[$metrics[7]]['data'],
+        ]), Response::MODEL_USAGE_FUNCTIONS);
     });
 
 App::put('/v1/functions/:functionId')
@@ -1516,11 +1498,11 @@ App::post('/v1/functions/:functionId/executions')
     ->inject('dbForProject')
     ->inject('user')
     ->inject('queueForEvents')
-    ->inject('usage')
+    ->inject('queueForUsage')
     ->inject('mode')
     ->inject('queueForFunctions')
     ->inject('geodb')
-    ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $queueForEvents, Stats $usage, string $mode, Func $queueForFunctions, Reader $geodb) {
+    ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $queueForEvents, Usage $queueForUsage, string $mode, Func $queueForFunctions, Reader $geodb) {
 
         $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId));
 
@@ -1658,6 +1640,8 @@ App::post('/v1/functions/:functionId/executions')
                 ->setJWT($jwt)
                 ->setProject($project)
                 ->setUser($user)
+                ->setParam('functionId', $function->getId())
+                ->setParam('executionId', $execution->getId())
                 ->trigger();
 
             return $response
@@ -1746,6 +1730,13 @@ App::post('/v1/functions/:functionId/executions')
                 ->setAttribute('responseStatusCode', 500)
                 ->setAttribute('errors', $th->getMessage() . '\nError Code: ' . $th->getCode());
             Console::error($th->getMessage());
+        } finally {
+            $queueForUsage
+                ->addMetric(METRIC_EXECUTIONS, 1)
+                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
+                ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project
+                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function
+            ;
         }
 
         if ($function->getAttribute('logging')) {
@@ -1753,14 +1744,6 @@ App::post('/v1/functions/:functionId/executions')
             $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution));
         }
 
-        // TODO revise this later using route label
-        $usage
-            ->setParam('functionId', $function->getId())
-            ->setParam('executions.{scope}.compute', 1)
-            ->setParam('executionStatus', $execution->getAttribute('status', ''))
-            ->setParam('executionTime', $execution->getAttribute('duration')); // ms
-
-
         $roles = Authorization::getRoles();
         $isPrivilegedUser = Auth::isPrivilegedUser($roles);
         $isAppUser = Auth::isAppUser($roles);

app/controllers/api/health.php

@@ -14,7 +14,9 @@ use Utopia\Registry\Registry;
 use Utopia\Storage\Device;
 use Utopia\Storage\Device\Local;
 use Utopia\Storage\Storage;
+use Utopia\Validator\Integer;
 use Utopia\Validator\Text;
+use Utopia\Validator\WhiteList;
 
 App::get('/v1/health')
     ->desc('Get HTTP')
@@ -87,22 +89,18 @@ App::get('/v1/health/db')
                             'ping' => \round((\microtime(true) - $checkStart) / 1000)
                         ]);
                     } else {
-                        $output[] = new Document([
-                            'name' => $key . " ($database)",
-                            'status' => 'fail',
-                            'ping' => \round((\microtime(true) - $checkStart) / 1000)
-                        ]);
+                        $failure[] = $database;
                     }
                 } catch (\Throwable $th) {
-                    $output[] = new Document([
-                        'name' => $key . " ($database)",
-                        'status' => 'fail',
-                        'ping' => \round((\microtime(true) - $checkStart) / 1000)
-                    ]);
+                    $failure[] = $database;
                 }
             }
         }
 
+        if (!empty($failure)) {
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'DB failure on: ' . implode(", ", $failure));
+        }
+
         $response->dynamic(new Document([
             'statuses' => $output,
             'total' => count($output),
@@ -348,11 +346,20 @@ App::get('/v1/health/queue/webhooks')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::WEBHOOK_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/logs')
@@ -366,11 +373,20 @@ App::get('/v1/health/queue/logs')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::AUDITS_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/certificates')
@@ -384,11 +400,20 @@ App::get('/v1/health/queue/certificates')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::CERTIFICATES_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/builds')
@@ -402,11 +427,20 @@ App::get('/v1/health/queue/builds')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::BUILDS_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/databases')
@@ -421,11 +455,20 @@ App::get('/v1/health/queue/databases')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
     ->param('name', 'database_db_main', new Text(256), 'Queue name for which to check the queue size', true)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (string $name, Connection $queue, Response $response) {
+    ->action(function (string $name, int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client($name, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/deletes')
@@ -439,11 +482,20 @@ App::get('/v1/health/queue/deletes')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::DELETE_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/mails')
@@ -457,11 +509,20 @@ App::get('/v1/health/queue/mails')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::MAILS_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/messaging')
@@ -475,11 +536,20 @@ App::get('/v1/health/queue/messaging')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::MESSAGING_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/migrations')
@@ -493,11 +563,20 @@ App::get('/v1/health/queue/migrations')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::MIGRATIONS_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/functions')
@@ -511,11 +590,20 @@ App::get('/v1/health/queue/functions')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/storage/local')
@@ -536,10 +624,10 @@ App::get('/v1/health/storage/local')
 
         foreach (
             [
-            'Uploads' => APP_STORAGE_UPLOADS,
-            'Cache' => APP_STORAGE_CACHE,
-            'Config' => APP_STORAGE_CONFIG,
-            'Certs' => APP_STORAGE_CERTIFICATES
+                'Uploads' => APP_STORAGE_UPLOADS,
+                'Cache' => APP_STORAGE_CACHE,
+                'Config' => APP_STORAGE_CONFIG,
+                'Certs' => APP_STORAGE_CERTIFICATES
             ] as $key => $volume
         ) {
             $device = new Local($volume);
@@ -600,8 +688,49 @@ App::get('/v1/health/anti-virus')
         $response->dynamic(new Document($output), Response::MODEL_HEALTH_ANTIVIRUS);
     });
 
+App::get('/v1/health/queue/failed/:name')
+    ->desc('Get number of failed queue jobs')
+    ->groups(['api', 'health'])
+    ->label('scope', 'health.read')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
+    ->label('sdk.namespace', 'health')
+    ->label('sdk.method', 'getFailedJobs')
+    ->param('queueName', '', new WhiteList([
+        Event::DATABASE_QUEUE_NAME,
+        Event::DELETE_QUEUE_NAME,
+        Event::AUDITS_QUEUE_NAME,
+        Event::MAILS_QUEUE_NAME,
+        Event::FUNCTIONS_QUEUE_NAME,
+        Event::USAGE_QUEUE_NAME,
+        Event::WEBHOOK_CLASS_NAME,
+        Event::CERTIFICATES_QUEUE_NAME,
+        Event::BUILDS_QUEUE_NAME,
+        Event::MESSAGING_QUEUE_NAME,
+        Event::MIGRATIONS_QUEUE_NAME,
+        Event::HAMSTER_CLASS_NAME
+    ]), 'The name of the queue')
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
+    ->label('sdk.description', '/docs/references/health/get-failed-queue-jobs.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_OK)
+    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
+    ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->inject('response')
+    ->inject('queue')
+    ->action(function (string $name, int|string $threshold, Response $response, Connection $queue) {
+        $threshold = \intval($threshold);
+
+        $client = new Client($name, $queue);
+        $failed = $client->countFailedJobs();
+
+        if ($failed >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue failed jobs threshold hit. Current size is {$failed} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $failed ]), Response::MODEL_HEALTH_QUEUE);
+    });
+
 App::get('/v1/health/stats') // Currently only used internally
-    ->desc('Get system stats')
+->desc('Get system stats')
     ->groups(['api', 'health'])
     ->label('scope', 'root')
     // ->label('sdk.auth', [APP_AUTH_TYPE_KEY])

app/controllers/api/migrations.php

@@ -208,6 +208,16 @@ App::post('/v1/migrations/firebase')
     ->inject('queueForEvents')
     ->inject('queueForMigrations')
     ->action(function (array $resources, string $serviceAccount, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) {
+        $serviceAccountData = json_decode($serviceAccount, true);
+
+        if (empty($serviceAccountData)) {
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Invalid Service Account JSON');
+        }
+
+        if (!isset($serviceAccountData['project_id']) || !isset($serviceAccountData['client_email']) || !isset($serviceAccountData['private_key'])) {
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Invalid Service Account JSON');
+        }
+
         $migration = $dbForProject->createDocument('migrations', new Document([
             '$id' => ID::unique(),
             'status' => 'pending',
@@ -449,15 +459,26 @@ App::get('/v1/migrations/appwrite/report')
     ->inject('project')
     ->inject('user')
     ->action(function (array $resources, string $endpoint, string $projectID, string $key, Response $response) {
-        try {
-            $appwrite = new Appwrite($projectID, $endpoint, $key);
+        $appwrite = new Appwrite($projectID, $endpoint, $key);
 
-            $response
-                ->setStatusCode(Response::STATUS_CODE_OK)
-                ->dynamic(new Document($appwrite->report($resources)), Response::MODEL_MIGRATION_REPORT);
+        try {
+            $report = $appwrite->report($resources);
         } catch (\Throwable $e) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage());
+            switch ($e->getCode()) {
+                case 401:
+                    throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, 'Source Error: ' . $e->getMessage());
+                case 429:
+                    throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Source Error: Rate Limit Exceeded, Is your Cloud Provider blocking Appwrite\'s IP?');
+                case 500:
+                    throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
+            }
+
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
         }
+
+        $response
+            ->setStatusCode(Response::STATUS_CODE_OK)
+            ->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT);
     });
 
 App::get('/v1/migrations/firebase/report')
@@ -475,15 +496,36 @@ App::get('/v1/migrations/firebase/report')
     ->param('serviceAccount', '', new Text(65536), 'JSON of the Firebase service account credentials')
     ->inject('response')
     ->action(function (array $resources, string $serviceAccount, Response $response) {
-        try {
-            $firebase = new Firebase(json_decode($serviceAccount, true));
+        $serviceAccount = json_decode($serviceAccount, true);
 
-            $response
-                ->setStatusCode(Response::STATUS_CODE_OK)
-                ->dynamic(new Document($firebase->report($resources)), Response::MODEL_MIGRATION_REPORT);
-        } catch (\Exception $e) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage());
+        if (empty($serviceAccount)) {
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Invalid Service Account JSON');
         }
+
+        if (!isset($serviceAccount['project_id']) || !isset($serviceAccount['client_email']) || !isset($serviceAccount['private_key'])) {
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Invalid Service Account JSON');
+        }
+
+        $firebase = new Firebase($serviceAccount);
+
+        try {
+            $report = $firebase->report($resources);
+        } catch (\Throwable $e) {
+            switch ($e->getCode()) {
+                case 401:
+                    throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, 'Source Error: ' . $e->getMessage());
+                case 429:
+                    throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Source Error: Rate Limit Exceeded, Is your Cloud Provider blocking Appwrite\'s IP?');
+                case 500:
+                    throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
+            }
+
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
+        }
+
+        $response
+            ->setStatusCode(Response::STATUS_CODE_OK)
+            ->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT);
     });
 
 App::get('/v1/migrations/firebase/report/oauth')
@@ -869,15 +911,26 @@ App::get('/v1/migrations/supabase/report')
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (array $resources, string $endpoint, string $apiKey, string $databaseHost, string $username, string $password, int $port, Response $response) {
-        try {
-            $supabase = new Supabase($endpoint, $apiKey, $databaseHost, 'postgres', $username, $password, $port);
+        $supabase = new Supabase($endpoint, $apiKey, $databaseHost, 'postgres', $username, $password, $port);
 
-            $response
-                ->setStatusCode(Response::STATUS_CODE_OK)
-                ->dynamic(new Document($supabase->report($resources)), Response::MODEL_MIGRATION_REPORT);
-        } catch (\Exception $e) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage());
+        try {
+            $report = $supabase->report($resources);
+        } catch (\Throwable $e) {
+            switch ($e->getCode()) {
+                case 401:
+                    throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, 'Source Error: ' . $e->getMessage());
+                case 429:
+                    throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Source Error: Rate Limit Exceeded, Is your Cloud Provider blocking Appwrite\'s IP?');
+                case 500:
+                    throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
+            }
+
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
         }
+
+        $response
+            ->setStatusCode(Response::STATUS_CODE_OK)
+            ->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT);
     });
 
 App::get('/v1/migrations/nhost/report')
@@ -901,15 +954,26 @@ App::get('/v1/migrations/nhost/report')
     ->param('port', 5432, new Integer(true), 'Source\'s Database Port.', true)
     ->inject('response')
     ->action(function (array $resources, string $subdomain, string $region, string $adminSecret, string $database, string $username, string $password, int $port, Response $response) {
-        try {
-            $nhost = new NHost($subdomain, $region, $adminSecret, $database, $username, $password, $port);
+        $nhost = new NHost($subdomain, $region, $adminSecret, $database, $username, $password, $port);
 
-            $response
-                ->setStatusCode(Response::STATUS_CODE_OK)
-                ->dynamic(new Document($nhost->report($resources)), Response::MODEL_MIGRATION_REPORT);
-        } catch (\Exception $e) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage());
+        try {
+            $report = $nhost->report($resources);
+        } catch (\Throwable $e) {
+            switch ($e->getCode()) {
+                case 401:
+                    throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, 'Source Error: ' . $e->getMessage());
+                case 429:
+                    throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Source Error: Rate Limit Exceeded, Is your Cloud Provider blocking Appwrite\'s IP?');
+                case 500:
+                    throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
+            }
+
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
         }
+
+        $response
+            ->setStatusCode(Response::STATUS_CODE_OK)
+            ->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT);
     });
 
 App::patch('/v1/migrations/:migrationId')

app/controllers/api/project.php

@@ -3,8 +3,10 @@
 use Appwrite\Extend\Exception;
 use Appwrite\Utopia\Response;
 use Utopia\App;
+use Utopia\Config\Config;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
+use Utopia\Database\Validator\Datetime as DateTimeValidator;
 use Utopia\Database\Exception\Duplicate as DuplicateException;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Helpers\Permission;
@@ -14,11 +16,10 @@ use Utopia\Database\Validator\Authorization;
 use Utopia\Database\Validator\UID;
 use Utopia\Validator\Text;
 use Utopia\Validator\WhiteList;
-use Utopia\Database\DateTime;
 
 App::get('/v1/project/usage')
     ->desc('Get usage stats for a project')
-    ->groups(['api'])
+    ->groups(['api', 'usage'])
     ->label('scope', 'projects.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'project')
@@ -26,100 +27,141 @@ App::get('/v1/project/usage')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_PROJECT)
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
+    ->param('startDate', '', new DateTimeValidator(), 'Starting date for the usage')
+    ->param('endDate', '', new DateTimeValidator(), 'End date for the usage')
+    ->param('period', '1d', new WhiteList(['1h', '1d']), 'Period used', true)
     ->inject('response')
     ->inject('dbForProject')
-    ->action(function (string $range, Response $response, Database $dbForProject) {
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+    ->action(function (string $startDate, string $endDate, string $period, Response $response, Database $dbForProject) {
+        $stats = $total = $usage = [];
+        $format = 'Y-m-d 00:00:00';
+        $firstDay = (new DateTime($startDate))->format($format);
+        $lastDay = (new DateTime($endDate))->format($format);
 
-            $metrics = [
-                'project.$all.network.requests',
-                'project.$all.network.bandwidth',
-                'project.$all.storage.size',
-                'users.$all.count.total',
-                'databases.$all.count.total',
-                'documents.$all.count.total',
-                'executions.$all.compute.total',
-                'buckets.$all.count.total'
-            ];
+        $metrics = [
+           'total' => [
+                METRIC_EXECUTIONS,
+                METRIC_DOCUMENTS,
+                METRIC_DATABASES,
+                METRIC_USERS,
+                METRIC_BUCKETS,
+                METRIC_FILES_STORAGE
+           ],
+           'period' => [
+                METRIC_NETWORK_REQUESTS,
+                METRIC_NETWORK_INBOUND,
+                METRIC_NETWORK_OUTBOUND,
+                METRIC_USERS,
+                METRIC_EXECUTIONS
+             ]
+        ];
 
-            $stats = [];
+        $factor = match ($period) {
+            '1h' => 3600,
+            '1d' => 86400,
+        };
 
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
+        $limit = match ($period) {
+            '1h' => (new DateTime($startDate))->diff(new DateTime($endDate))->days * 24,
+            '1d' => (new DateTime($startDate))->diff(new DateTime($endDate))->days
+        };
 
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
+        $format = match ($period) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
 
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
+        Authorization::skip(function () use ($dbForProject, $firstDay, $lastDay, $period, $metrics, &$total, &$stats) {
+            foreach ($metrics['total'] as $metric) {
+                $result = $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
+                $total[$metric] = $result['value'] ?? 0;
+            }
 
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
+            foreach ($metrics['period'] as $metric) {
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::greaterThanEqual('time', $firstDay),
+                    Query::lessThan('time', $lastDay),
+                    Query::orderDesc('time'),
+                ]);
+
+                $stats[$metric] = [];
+                foreach ($results as $result) {
+                    $stats[$metric][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'requests' => $stats[$metrics[0]] ?? [],
-                'network' => $stats[$metrics[1]] ?? [],
-                'storage' => $stats[$metrics[2]] ?? [],
-                'users' => $stats[$metrics[3]] ?? [],
-                'databases' => $stats[$metrics[4]] ?? [],
-                'documents' => $stats[$metrics[5]] ?? [],
-                'executions' => $stats[$metrics[6]] ?? [],
-                'buckets' => $stats[$metrics[7]] ?? [],
-            ]);
+        $now = time();
+        foreach ($metrics['period'] as $metric) {
+            $usage[$metric] = [];
+            $leap = $now - ($limit * $factor);
+            while ($leap < $now) {
+                $leap += $factor;
+                $formatDate = date($format, $leap);
+                $usage[$metric][] = [
+                    'value' => $stats[$metric][$formatDate]['value'] ?? 0,
+                    'date' => $formatDate,
+                ];
+            }
         }
 
-        $response->dynamic($usage, Response::MODEL_USAGE_PROJECT);
+        $executionsBreakdown = array_map(function ($function) use ($dbForProject) {
+            $id = $function->getId();
+            $name = $function->getAttribute('name');
+            $metric = str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS);
+            $value = $dbForProject->findOne('stats_v2', [
+                Query::equal('metric', [$metric]),
+                Query::equal('period', ['inf'])
+            ]);
+
+            return [
+                'resourceId' => $id,
+                'name' => $name,
+                'value' => $value['value'] ?? 0,
+            ];
+        }, $dbForProject->find('functions'));
+
+        $bucketsBreakdown = array_map(function ($bucket) use ($dbForProject) {
+            $id = $bucket->getId();
+            $name = $bucket->getAttribute('name');
+            $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE);
+            $value = $dbForProject->findOne('stats_v2', [
+                Query::equal('metric', [$metric]),
+                Query::equal('period', ['inf'])
+            ]);
+
+            return [
+                'resourceId' => $id,
+                'name' => $name,
+                'value' => $value['value'] ?? 0,
+            ];
+        }, $dbForProject->find('buckets'));
+
+        $response->dynamic(new Document([
+            'requests' => ($usage[METRIC_NETWORK_REQUESTS]),
+            'network' => ($usage[METRIC_NETWORK_INBOUND] + $usage[METRIC_NETWORK_OUTBOUND]),
+            'users' => ($usage[METRIC_USERS]),
+            'executions' => ($usage[METRIC_EXECUTIONS]),
+            'executionsTotal' => $total[METRIC_EXECUTIONS],
+            'documentsTotal' => $total[METRIC_DOCUMENTS],
+            'databasesTotal' => $total[METRIC_DATABASES],
+            'usersTotal' => $total[METRIC_USERS],
+            'bucketsTotal' => $total[METRIC_BUCKETS],
+            'filesStorageTotal' => $total[METRIC_FILES_STORAGE],
+            'executionsBreakdown' => $executionsBreakdown,
+            'bucketsBreakdown' => $bucketsBreakdown
+        ]), Response::MODEL_USAGE_PROJECT);
     });
 
-// Variables
 
+// Variables
 App::post('/v1/project/variables')
     ->desc('Create Variable')
     ->groups(['api'])

app/controllers/api/projects.php

@@ -293,120 +293,6 @@ App::get('/v1/projects/:projectId')
         $response->dynamic($project, Response::MODEL_PROJECT);
     });
 
-App::get('/v1/projects/:projectId/usage')
-    ->desc('Get usage stats for a project')
-    ->groups(['api', 'projects', 'usage'])
-    ->label('scope', 'projects.read')
-    ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
-    ->label('sdk.namespace', 'projects')
-    ->label('sdk.method', 'getUsage')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_USAGE_PROJECT)
-    ->param('projectId', '', new UID(), 'Project unique ID.')
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
-    ->inject('response')
-    ->inject('dbForConsole')
-    ->inject('dbForProject')
-    ->inject('register')
-    ->action(function (string $projectId, string $range, Response $response, Database $dbForConsole, Database $dbForProject, Registry $register) {
-
-        $project = $dbForConsole->getDocument('projects', $projectId);
-
-        if ($project->isEmpty()) {
-            throw new Exception(Exception::PROJECT_NOT_FOUND);
-        }
-
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
-
-            $dbForProject->setNamespace("_{$project->getInternalId()}");
-
-            $metrics = [
-                'project.$all.network.requests',
-                'project.$all.network.bandwidth',
-                'project.$all.storage.size',
-                'users.$all.count.total',
-                'databases.$all.count.total',
-                'documents.$all.count.total',
-                'executions.$all.compute.total',
-                'buckets.$all.count.total'
-            ];
-
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
-                }
-            });
-
-            $usage = new Document([
-                'range' => $range,
-                'requests' => $stats[$metrics[0]] ?? [],
-                'network' => $stats[$metrics[1]] ?? [],
-                'storage' => $stats[$metrics[2]] ?? [],
-                'users' => $stats[$metrics[3]] ?? [],
-                'databases' => $stats[$metrics[4]] ?? [],
-                'documents' => $stats[$metrics[5]] ?? [],
-                'executions' => $stats[$metrics[6]] ?? [],
-                'buckets' => $stats[$metrics[7]] ?? [],
-            ]);
-        }
-
-        $response->dynamic($usage, Response::MODEL_USAGE_PROJECT);
-    });
-
 App::patch('/v1/projects/:projectId')
     ->desc('Update project')
     ->groups(['api', 'projects'])
@@ -1704,7 +1590,7 @@ App::get('/v1/projects/:projectId/templates/email/:type/:locale')
             $message
                 ->setParam('{{hello}}', $localeObj->getText("emails.{$type}.hello"))
                 ->setParam('{{footer}}', $localeObj->getText("emails.{$type}.footer"))
-                ->setParam('{{body}}', $localeObj->getText('emails.' . $type . '.body'))
+                ->setParam('{{body}}', $localeObj->getText('emails.' . $type . '.body'), escapeHtml: false)
                 ->setParam('{{thanks}}', $localeObj->getText("emails.{$type}.thanks"))
                 ->setParam('{{signature}}', $localeObj->getText("emails.{$type}.signature"))
                 ->setParam('{{direction}}', $localeObj->getText('settings.direction'));

app/controllers/api/proxy.php

@@ -14,6 +14,7 @@ use Utopia\Database\Helpers\ID;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\UID;
 use Utopia\Domains\Domain;
+use Utopia\Logger\Log;
 use Utopia\Validator\Domain as ValidatorDomain;
 use Utopia\Validator\Text;
 use Utopia\Validator\WhiteList;
@@ -87,7 +88,11 @@ App::post('/v1/proxy/rules')
             $resourceInternalId = $function->getInternalId();
         }
 
-        $domain = new Domain($domain);
+        try {
+            $domain = new Domain($domain);
+        } catch (\Exception) {
+            throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Domain may not start with http:// or https://.');
+        }
 
         $ruleId = ID::unique();
         $rule = new Document([
@@ -274,7 +279,8 @@ App::patch('/v1/proxy/rules/:ruleId/verification')
     ->inject('queueForEvents')
     ->inject('project')
     ->inject('dbForConsole')
-    ->action(function (string $ruleId, Response $response, Certificate $queueForCertificates, Event $queueForEvents, Document $project, Database $dbForConsole) {
+    ->inject('log')
+    ->action(function (string $ruleId, Response $response, Certificate $queueForCertificates, Event $queueForEvents, Document $project, Database $dbForConsole, Log $log) {
         $rule = $dbForConsole->getDocument('rules', $ruleId);
 
         if ($rule->isEmpty() || $rule->getAttribute('projectInternalId') !== $project->getInternalId()) {
@@ -294,7 +300,14 @@ App::patch('/v1/proxy/rules/:ruleId/verification')
         $validator = new CNAME($target->get()); // Verify Domain with DNS records
         $domain = new Domain($rule->getAttribute('domain', ''));
 
+        $validationStart = \microtime(true);
         if (!$validator->isValid($domain->get())) {
+            $log->addExtra('dnsTiming', \strval(\microtime(true) - $validationStart));
+            $log->addTag('dnsDomain', $domain->get());
+
+            $error = $validator->getLogs();
+            $log->addExtra('dnsResponse', \is_array($error) ? \json_encode($error) : \strval($error));
+
             throw new Exception(Exception::RULE_VERIFICATION_FAILED);
         }
 

app/controllers/api/storage.php

@@ -44,6 +44,7 @@ use Utopia\Validator\Text;
 use Utopia\Validator\WhiteList;
 use Utopia\DSN\DSN;
 use Utopia\Swoole\Request;
+use Utopia\Storage\Compression\Compression;
 
 App::post('/v1/storage/buckets')
     ->desc('Create bucket')
@@ -52,7 +53,6 @@ App::post('/v1/storage/buckets')
     ->label('event', 'buckets.[bucketId].create')
     ->label('audits.event', 'bucket.create')
     ->label('audits.resource', 'bucket/{response.$id}')
-    ->label('usage.metric', 'buckets.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'createBucket')
@@ -67,7 +67,7 @@ App::post('/v1/storage/buckets')
     ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true)
     ->param('maximumFileSize', (int) App::getEnv('_APP_STORAGE_LIMIT', 0), new Range(1, (int) App::getEnv('_APP_STORAGE_LIMIT', 0)), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true)
     ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true)
-    ->param('compression', COMPRESSION_TYPE_NONE, new WhiteList([COMPRESSION_TYPE_NONE, COMPRESSION_TYPE_GZIP, COMPRESSION_TYPE_ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . COMPRESSION_TYPE_NONE . ',  [' . COMPRESSION_TYPE_GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . COMPRESSION_TYPE_ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
+    ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ',  [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
     ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true)
     ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true)
     ->inject('response')
@@ -148,7 +148,6 @@ App::get('/v1/storage/buckets')
     ->desc('List buckets')
     ->groups(['api', 'storage'])
     ->label('scope', 'buckets.read')
-    ->label('usage.metric', 'buckets.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'listBuckets')
@@ -197,7 +196,6 @@ App::get('/v1/storage/buckets/:bucketId')
     ->desc('Get bucket')
     ->groups(['api', 'storage'])
     ->label('scope', 'buckets.read')
-    ->label('usage.metric', 'buckets.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'getBucket')
@@ -226,7 +224,6 @@ App::put('/v1/storage/buckets/:bucketId')
     ->label('event', 'buckets.[bucketId].update')
     ->label('audits.event', 'bucket.update')
     ->label('audits.resource', 'bucket/{response.$id}')
-    ->label('usage.metric', 'buckets.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'updateBucket')
@@ -241,7 +238,7 @@ App::put('/v1/storage/buckets/:bucketId')
     ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true)
     ->param('maximumFileSize', null, new Range(1, (int) App::getEnv('_APP_STORAGE_LIMIT', 0)), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human((int)App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true)
     ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true)
-    ->param('compression', COMPRESSION_TYPE_NONE, new WhiteList([COMPRESSION_TYPE_NONE, COMPRESSION_TYPE_GZIP, COMPRESSION_TYPE_ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . COMPRESSION_TYPE_NONE . ', [' . COMPRESSION_TYPE_GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . COMPRESSION_TYPE_ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
+    ->param('compression', Compression::NONE, new WhiteList([Compression::NONE, Compression::GZIP, Compression::ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . Compression::NONE . ', [' . Compression::GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . Compression::ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
     ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true)
     ->param('antivirus', true, new Boolean(true), 'Is virus scanning enabled? For file size above ' . Storage::human(APP_LIMIT_ANTIVIRUS, 0) . ' AntiVirus scanning is skipped even if it\'s enabled', true)
     ->inject('response')
@@ -294,7 +291,6 @@ App::delete('/v1/storage/buckets/:bucketId')
     ->label('audits.event', 'bucket.delete')
     ->label('event', 'buckets.[bucketId].delete')
     ->label('audits.resource', 'bucket/{request.bucketId}')
-    ->label('usage.metric', 'buckets.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'deleteBucket')
@@ -337,8 +333,6 @@ App::post('/v1/storage/buckets/:bucketId/files')
     ->label('audits.event', 'file.create')
     ->label('event', 'buckets.[bucketId].files.[fileId].create')
     ->label('audits.resource', 'file/{response.$id}')
-    ->label('usage.metric', 'files.{scope}.requests.create')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -538,19 +532,24 @@ App::post('/v1/storage/buckets/:bucketId/files')
             $fileHash = $deviceFiles->getFileHash($path); // Get file hash before compression and encryption
             $data = '';
             // Compression
-            $algorithm = $bucket->getAttribute('compression', COMPRESSION_TYPE_NONE);
-            if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != COMPRESSION_TYPE_NONE) {
+            $algorithm = $bucket->getAttribute('compression', Compression::NONE);
+            if ($fileSize <= APP_STORAGE_READ_BUFFER && $algorithm != Compression::NONE) {
                 $data = $deviceFiles->read($path);
                 switch ($algorithm) {
-                    case COMPRESSION_TYPE_ZSTD:
+                    case Compression::ZSTD:
                         $compressor = new Zstd();
                         break;
-                    case COMPRESSION_TYPE_GZIP:
+                    case Compression::GZIP:
                     default:
                         $compressor = new GZIP();
                         break;
                 }
                 $data = $compressor->compress($data);
+            } else {
+                // reset the algorithm to none as we do not compress the file
+                // if file size exceedes the APP_STORAGE_READ_BUFFER
+                // regardless the bucket compression algoorithm
+                $algorithm = Compression::NONE;
             }
 
             if ($bucket->getAttribute('encryption', true) && $fileSize <= APP_STORAGE_READ_BUFFER) {
@@ -622,7 +621,17 @@ App::post('/v1/storage/buckets/:bucketId/files')
                         ->setAttribute('metadata', $metadata)
                         ->setAttribute('chunksUploaded', $chunksUploaded);
 
-                    $file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file);
+                    /**
+                     * Validate create permission and skip authorization in updateDocument
+                     * Without this, the file creation will fail when user doesn't have update permission
+                     * However as with chunk upload even if we are updating, we are essentially creating a file
+                     * adding it's new chunk so we validate create permission instead of update
+                     */
+                    $validator = new Authorization(Database::PERMISSION_CREATE);
+                    if (!$validator->isValid($bucket->getCreate())) {
+                        throw new Exception(Exception::USER_UNAUTHORIZED);
+                    }
+                    $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file));
                 }
             } catch (AuthorizationException) {
                 throw new Exception(Exception::USER_UNAUTHORIZED);
@@ -659,7 +668,17 @@ App::post('/v1/storage/buckets/:bucketId/files')
                         ->setAttribute('chunksUploaded', $chunksUploaded)
                         ->setAttribute('metadata', $metadata);
 
-                    $file = $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file);
+                    /**
+                     * Validate create permission and skip authorization in updateDocument
+                     * Without this, the file creation will fail when user doesn't have update permission
+                     * However as with chunk upload even if we are updating, we are essentially creating a file
+                     * adding it's new chunk so we validate create permission instead of update
+                     */
+                    $validator = new Authorization(Database::PERMISSION_CREATE);
+                    if (!$validator->isValid($bucket->getCreate())) {
+                        throw new Exception(Exception::USER_UNAUTHORIZED);
+                    }
+                    $file = Authorization::skip(fn() => $dbForProject->updateDocument('bucket_' . $bucket->getInternalId(), $fileId, $file));
                 }
             } catch (AuthorizationException) {
                 throw new Exception(Exception::USER_UNAUTHORIZED);
@@ -689,8 +708,6 @@ App::get('/v1/storage/buckets/:bucketId/files')
     ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('usage.metric', 'files.{scope}.requests.read')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'listFiles')
     ->label('sdk.description', '/docs/references/storage/list-files.md')
@@ -770,8 +787,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId')
     ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('usage.metric', 'files.{scope}.requests.read')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'getFile')
     ->label('sdk.description', '/docs/references/storage/get-file.md')
@@ -821,8 +836,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
     ->label('cache', true)
     ->label('cache.resourceType', 'bucket/{request.bucketId}')
     ->label('cache.resource', 'file/{request.fileId}')
-    ->label('usage.metric', 'files.{scope}.requests.read')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'getFilePreview')
@@ -872,14 +885,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
             throw new Exception(Exception::USER_UNAUTHORIZED);
         }
 
-        if ((\strpos($request->getAccept(), 'image/webp') === false) && ('webp' === $output)) { // Fallback webp to jpeg when no browser support
-            $output = 'jpg';
-        }
-
-        $inputs = Config::getParam('storage-inputs');
-        $outputs = Config::getParam('storage-outputs');
-        $fileLogos = Config::getParam('storage-logos');
-
         if ($fileSecurity && !$valid) {
             $file = $dbForProject->getDocument('bucket_' . $bucket->getInternalId(), $fileId);
         } else {
@@ -890,9 +895,17 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
             throw new Exception(Exception::STORAGE_FILE_NOT_FOUND);
         }
 
+        if ((\strpos($request->getAccept(), 'image/webp') === false) && ('webp' === $output)) { // Fallback webp to jpeg when no browser support
+            $output = 'jpg';
+        }
+
+        $inputs = Config::getParam('storage-inputs');
+        $outputs = Config::getParam('storage-outputs');
+        $fileLogos = Config::getParam('storage-logos');
+
         $path = $file->getAttribute('path');
         $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION));
-        $algorithm = $file->getAttribute('algorithm', 'none');
+        $algorithm = $file->getAttribute('algorithm', Compression::NONE);
         $cipher = $file->getAttribute('openSSLCipher');
         $mime = $file->getAttribute('mimeType');
         if (!\in_array($mime, $inputs) || $file->getAttribute('sizeActual') > (int) App::getEnv('_APP_STORAGE_PREVIEW_LIMIT', 20000000)) {
@@ -903,7 +916,7 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
                 $path = $fileLogos['default_image'];
             }
 
-            $algorithm = 'none';
+            $algorithm = Compression::NONE;
             $cipher = null;
             $background = (empty($background)) ? 'eceff1' : $background;
             $type = \strtolower(\pathinfo($path, PATHINFO_EXTENSION));
@@ -915,12 +928,17 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
         }
 
         if (empty($output)) {
+            // when file extension is provided but it's not one of our
+            // supported outputs we fallback to `jpg`
+            if (!empty($type) && !array_key_exists($type, $outputs)) {
+                $type = 'jpg';
+            }
+
             // when file extension is not provided and the mime type is not one of our supported outputs
             // we fallback to `jpg` output format
             $output = empty($type) ? (array_search($mime, $outputs) ?? 'jpg') : $type;
         }
 
-
         $source = $deviceFiles->read($path);
 
         if (!empty($cipher)) { // Decrypt
@@ -935,11 +953,11 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
         }
 
         switch ($algorithm) {
-            case 'zstd':
+            case Compression::ZSTD:
                 $compressor = new Zstd();
                 $source = $compressor->decompress($source);
                 break;
-            case 'gzip':
+            case Compression::GZIP:
                 $compressor = new GZIP();
                 $source = $compressor->decompress($source);
                 break;
@@ -987,8 +1005,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
     ->desc('Get file for download')
     ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
-    ->label('usage.metric', 'files.{scope}.requests.read')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'getFileDownload')
@@ -1080,15 +1096,15 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
             );
         }
 
-        switch ($file->getAttribute('algorithm', 'none')) {
-            case 'zstd':
+        switch ($file->getAttribute('algorithm', Compression::NONE)) {
+            case Compression::ZSTD:
                 if (empty($source)) {
                     $source = $deviceFiles->read($path);
                 }
                 $compressor = new Zstd();
                 $source = $compressor->decompress($source);
                 break;
-            case 'gzip':
+            case Compression::GZIP:
                 if (empty($source)) {
                     $source = $deviceFiles->read($path);
                 }
@@ -1129,8 +1145,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
     ->desc('Get file for view')
     ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
-    ->label('usage.metric', 'files.{scope}.requests.read')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'getFileView')
@@ -1231,15 +1245,15 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
             );
         }
 
-        switch ($file->getAttribute('algorithm', 'none')) {
-            case 'zstd':
+        switch ($file->getAttribute('algorithm', Compression::NONE)) {
+            case Compression::ZSTD:
                 if (empty($source)) {
                     $source = $deviceFiles->read($path);
                 }
                 $compressor = new Zstd();
                 $source = $compressor->decompress($source);
                 break;
-            case 'gzip':
+            case Compression::GZIP:
                 if (empty($source)) {
                     $source = $deviceFiles->read($path);
                 }
@@ -1253,10 +1267,12 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
                 $response->send(substr($source, $start, ($end - $start + 1)));
             }
             $response->send($source);
+            return;
         }
 
         if (!empty($rangeHeader)) {
             $response->send($deviceFiles->read($path, $start, ($end - $start + 1)));
+            return;
         }
 
         $size = $deviceFiles->getFileSize($path);
@@ -1284,8 +1300,6 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
     ->label('event', 'buckets.[bucketId].files.[fileId].update')
     ->label('audits.event', 'file.update')
     ->label('audits.resource', 'file/{response.$id}')
-    ->label('usage.metric', 'files.{scope}.requests.update')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -1394,8 +1408,6 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
     ->label('event', 'buckets.[bucketId].files.[fileId].delete')
     ->label('audits.event', 'file.delete')
     ->label('audits.resource', 'file/{request.fileId}')
-    ->label('usage.metric', 'files.{scope}.requests.delete')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -1487,7 +1499,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
 
 App::get('/v1/storage/usage')
     ->desc('Get usage stats for storage')
-    ->groups(['api', 'storage', 'usage'])
+    ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'storage')
@@ -1495,109 +1507,78 @@ App::get('/v1/storage/usage')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_STORAGE)
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (string $range, Response $response, Database $dbForProject) {
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            METRIC_BUCKETS,
+            METRIC_FILES,
+            METRIC_FILES_STORAGE,
+        ];
 
-            $metrics = [
-                'project.$all.storage.size',
-                'buckets.$all.count.total',
-                'buckets.$all.requests.create',
-                'buckets.$all.requests.read',
-                'buckets.$all.requests.update',
-                'buckets.$all.requests.delete',
-                'files.$all.storage.size',
-                'files.$all.count.total',
-                'files.$all.requests.create',
-                'files.$all.requests.read',
-                'files.$all.requests.update',
-                'files.$all.requests.delete',
-            ];
+        $total = [];
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
+            foreach ($metrics as $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
 
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'bucketsCount' => $stats['buckets.$all.count.total'],
-                'bucketsCreate' => $stats['buckets.$all.requests.create'],
-                'bucketsRead' => $stats['buckets.$all.requests.read'],
-                'bucketsUpdate' => $stats['buckets.$all.requests.update'],
-                'bucketsDelete' => $stats['buckets.$all.requests.delete'],
-                'storage' => $stats['project.$all.storage.size'],
-                'filesCount' => $stats['files.$all.count.total'],
-                'filesCreate' => $stats['files.$all.requests.create'],
-                'filesRead' => $stats['files.$all.requests.read'],
-                'filesUpdate' => $stats['files.$all.requests.update'],
-                'filesDelete' => $stats['files.$all.requests.delete'],
-            ]);
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
         }
-
-        $response->dynamic($usage, Response::MODEL_USAGE_STORAGE);
+    }
+        $response->dynamic(new Document([
+            'range' => $range,
+            'bucketsTotal' => $usage[$metrics[0]]['total'],
+            'filesTotal' => $usage[$metrics[1]]['total'],
+            'filesStorageTotal' => $usage[$metrics[2]]['total'],
+            'buckets' => $usage[$metrics[0]]['data'],
+            'files' =>  $usage[$metrics[1]]['data'],
+            'storage' =>  $usage[$metrics[2]]['data'],
+        ]), Response::MODEL_USAGE_STORAGE);
     });
 
 App::get('/v1/storage/:bucketId/usage')
-    ->desc('Get usage stats for a storage bucket')
-    ->groups(['api', 'storage', 'usage'])
+    ->desc('Get usage stats for storage bucket')
+    ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'storage')
@@ -1606,7 +1587,7 @@ App::get('/v1/storage/:bucketId/usage')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_BUCKETS)
     ->param('bucketId', '', new UID(), 'Bucket ID.')
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (string $bucketId, string $range, Response $response, Database $dbForProject) {
@@ -1617,86 +1598,65 @@ App::get('/v1/storage/:bucketId/usage')
             throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
         }
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES),
+            str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE),
+        ];
 
-            $metrics = [
-                "files.{$bucketId}.count.total",
-                "files.{$bucketId}.storage.size",
-                "files.{$bucketId}.requests.create",
-                "files.{$bucketId}.requests.read",
-                "files.{$bucketId}.requests.update",
-                "files.{$bucketId}.requests.delete",
-            ];
 
-            $stats = [];
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
+            foreach ($metrics as $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
 
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'filesCount' => $stats[$metrics[0]],
-                'filesStorage' => $stats[$metrics[1]],
-                'filesCreate' => $stats[$metrics[2]],
-                'filesRead' => $stats[$metrics[3]],
-                'filesUpdate' => $stats[$metrics[4]],
-                'filesDelete' => $stats[$metrics[5]],
-            ]);
+
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
         }
+    }
 
-        $response->dynamic($usage, Response::MODEL_USAGE_BUCKETS);
+        $response->dynamic(new Document([
+            'range' => $range,
+            'filesTotal' => $usage[$metrics[0]]['total'],
+            'filesStorageTotal' => $usage[$metrics[1]]['total'],
+            'files' => $usage[$metrics[0]]['data'],
+            'storage' => $usage[$metrics[1]]['data'],
+        ]), Response::MODEL_USAGE_BUCKETS);
     });

app/controllers/api/teams.php

@@ -380,6 +380,7 @@ App::post('/v1/teams/:teamId/memberships')
     ->param('roles', [], new ArrayList(new Key(), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of strings. Use this param to set the user roles in the team. A role can be any string. Learn more about [roles and permissions](https://appwrite.io/docs/permissions). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each 32 characters long.')
     ->param('url', '', fn($clients) => new Host($clients), 'URL to redirect the user back to your app from the invitation email.  Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients']) // TODO add our own built-in confirm page
     ->param('name', '', new Text(128), 'Name of the new team member. Max length: 128 chars.', true)
+    ->inject('request')
     ->inject('response')
     ->inject('project')
     ->inject('user')
@@ -388,7 +389,7 @@ App::post('/v1/teams/:teamId/memberships')
     ->inject('queueForMails')
     ->inject('queueForMessaging')
     ->inject('queueForEvents')
-    ->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, EventPhone $queueForMessaging, Event $queueForEvents) {
+    ->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Request $request, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, EventPhone $queueForMessaging, Event $queueForEvents) {
         $isAPIKey = Auth::isAppUser(Authorization::getRoles());
         $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
 
@@ -533,8 +534,8 @@ App::post('/v1/teams/:teamId/memberships')
             } catch (Duplicate $th) {
                 throw new Exception(Exception::TEAM_INVITE_ALREADY_EXISTS);
             }
-            $team->setAttribute('total', $team->getAttribute('total', 0) + 1);
-            $team = Authorization::skip(fn() => $dbForProject->updateDocument('teams', $team->getId(), $team));
+
+            Authorization::skip(fn() => $dbForProject->increaseDocumentAttribute('teams', $team->getId(), 'total', 1));
 
             $dbForProject->deleteCachedDocument('users', $invitee->getId());
         } else {
@@ -556,7 +557,7 @@ App::post('/v1/teams/:teamId/memberships')
 
                 $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
                 $message
-                    ->setParam('{{body}}', $body)
+                    ->setParam('{{body}}', $body, escapeHtml: false)
                     ->setParam('{{hello}}', $locale->getText("emails.invitation.hello"))
                     ->setParam('{{footer}}', $locale->getText("emails.invitation.footer"))
                     ->setParam('{{thanks}}', $locale->getText("emails.invitation.thanks"))
@@ -612,11 +613,11 @@ App::post('/v1/teams/:teamId/memberships')
                 $emailVariables = [
                     'owner' => $user->getAttribute('name'),
                     'direction' => $locale->getText('settings.direction'),
-                    /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
+                    /* {{user}}, {{team}}, {{redirect}} and {{project}} are required in default and custom templates */
                     'user' => $user->getAttribute('name'),
                     'team' => $team->getAttribute('name'),
-                    'project' => $projectName,
-                    'redirect' => $url
+                    'redirect' => $url,
+                    'project' => $projectName
                 ];
 
                 $queueForMails
@@ -638,9 +639,12 @@ App::post('/v1/teams/:teamId/memberships')
                 $message = $message->setParam('{{token}}', $url);
                 $message = $message->render();
 
+                var_dump($request->getIP());
+                var_dump($project->getId());
                 $queueForMessaging
                     ->setRecipient($phone)
                     ->setMessage($message)
+                    ->setProject($project)
                     ->trigger();
             }
         }

app/controllers/api/users.php

@@ -39,10 +39,12 @@ use Utopia\Validator\Integer;
 use Appwrite\Auth\Validator\PasswordHistory;
 use Appwrite\Auth\Validator\PasswordDictionary;
 use Appwrite\Auth\Validator\PersonalData;
+use Appwrite\Hooks\Hooks;
 
 /** TODO: Remove function when we move to using utopia/platform */
-function createUser(string $hash, mixed $hashOptions, string $userId, ?string $email, ?string $password, ?string $phone, string $name, Document $project, Database $dbForProject, Event $queueForEvents): Document
+function createUser(string $hash, mixed $hashOptions, string $userId, ?string $email, ?string $password, ?string $phone, string $name, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks): Document
 {
+    $plaintextPassword = $password;
     $hashOptionsObject = (\is_string($hashOptions)) ? \json_decode($hashOptions, true) : $hashOptions; // Cast to JSON array
     $passwordHistory = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
 
@@ -65,13 +67,13 @@ function createUser(string $hash, mixed $hashOptions, string $userId, ?string $e
 
         if ($project->getAttribute('auths', [])['personalDataCheck'] ?? false) {
             $personalDataValidator = new PersonalData($userId, $email, $name, $phone);
-            if (!$personalDataValidator->isValid($password)) {
+            if (!$personalDataValidator->isValid($plaintextPassword)) {
                 throw new Exception(Exception::USER_PASSWORD_PERSONAL_DATA);
             }
         }
 
         $password = (!empty($password)) ? ($hash === 'plaintext' ? Auth::passwordHash($password, $hash, $hashOptionsObject) : $password) : null;
-        $user = $dbForProject->createDocument('users', new Document([
+        $user = new Document([
             '$id' => $userId,
             '$permissions' => [
                 Permission::read(Role::any()),
@@ -85,7 +87,7 @@ function createUser(string $hash, mixed $hashOptions, string $userId, ?string $e
             'status' => true,
             'labels' => [],
             'password' => $password,
-            'passwordHistory' => is_null($password) && $passwordHistory === 0 ? [] : [$password],
+            'passwordHistory' => is_null($password) || $passwordHistory === 0 ? [] : [$password],
             'passwordUpdate' => (!empty($password)) ? DateTime::now() : null,
             'hash' => $hash === 'plaintext' ? Auth::DEFAULT_ALGO : $hash,
             'hashOptions' => $hash === 'plaintext' ? Auth::DEFAULT_ALGO_OPTIONS : $hashOptionsObject + ['type' => $hash],
@@ -97,7 +99,13 @@ function createUser(string $hash, mixed $hashOptions, string $userId, ?string $e
             'tokens' => null,
             'memberships' => null,
             'search' => implode(' ', [$userId, $email, $phone, $name]),
-        ]));
+        ]);
+
+        if ($hash === 'plaintext') {
+            $hooks->trigger('passwordValidator', [$dbForProject, $project, $plaintextPassword, &$user, true]);
+        }
+
+        $user = $dbForProject->createDocument('users', $user);
     } catch (Duplicate $th) {
         throw new Exception(Exception::USER_ALREADY_EXISTS);
     }
@@ -114,7 +122,6 @@ App::post('/v1/users')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'create')
@@ -131,8 +138,9 @@ App::post('/v1/users')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
-        $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents);
+    ->inject('hooks')
+    ->action(function (string $userId, ?string $email, ?string $phone, ?string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
+        $user = createUser('plaintext', '{}', $userId, $email, $password, $phone, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -146,7 +154,6 @@ App::post('/v1/users/bcrypt')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createBcryptUser')
@@ -162,8 +169,9 @@ App::post('/v1/users/bcrypt')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
-        $user = createUser('bcrypt', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
+        $user = createUser('bcrypt', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -177,7 +185,6 @@ App::post('/v1/users/md5')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createMD5User')
@@ -193,8 +200,9 @@ App::post('/v1/users/md5')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
-        $user = createUser('md5', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
+        $user = createUser('md5', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -208,7 +216,6 @@ App::post('/v1/users/argon2')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createArgon2User')
@@ -224,8 +231,9 @@ App::post('/v1/users/argon2')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
-        $user = createUser('argon2', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
+        $user = createUser('argon2', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -239,7 +247,6 @@ App::post('/v1/users/sha')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createSHAUser')
@@ -256,14 +263,15 @@ App::post('/v1/users/sha')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $passwordVersion, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $passwordVersion, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
         $options = '{}';
 
         if (!empty($passwordVersion)) {
             $options = '{"version":"' . $passwordVersion . '"}';
         }
 
-        $user = createUser('sha', $options, $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
+        $user = createUser('sha', $options, $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -277,7 +285,6 @@ App::post('/v1/users/phpass')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createPHPassUser')
@@ -293,8 +300,9 @@ App::post('/v1/users/phpass')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
-        $user = createUser('phpass', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
+        $user = createUser('phpass', '{}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -308,7 +316,6 @@ App::post('/v1/users/scrypt')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createScryptUser')
@@ -329,7 +336,8 @@ App::post('/v1/users/scrypt')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $passwordSalt, int $passwordCpu, int $passwordMemory, int $passwordParallel, int $passwordLength, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $passwordSalt, int $passwordCpu, int $passwordMemory, int $passwordParallel, int $passwordLength, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
         $options = [
             'salt' => $passwordSalt,
             'costCpu' => $passwordCpu,
@@ -338,7 +346,7 @@ App::post('/v1/users/scrypt')
             'length' => $passwordLength
         ];
 
-        $user = createUser('scrypt', \json_encode($options), $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
+        $user = createUser('scrypt', \json_encode($options), $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -352,7 +360,6 @@ App::post('/v1/users/scrypt-modified')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createScryptModifiedUser')
@@ -371,8 +378,9 @@ App::post('/v1/users/scrypt-modified')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $passwordSalt, string $passwordSaltSeparator, string $passwordSignerKey, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
-        $user = createUser('scryptMod', '{"signerKey":"' . $passwordSignerKey . '","saltSeparator":"' . $passwordSaltSeparator . '","salt":"' . $passwordSalt . '"}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents);
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $passwordSalt, string $passwordSaltSeparator, string $passwordSignerKey, string $name, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
+        $user = createUser('scryptMod', '{"signerKey":"' . $passwordSignerKey . '","saltSeparator":"' . $passwordSaltSeparator . '","salt":"' . $passwordSalt . '"}', $userId, $email, $password, null, $name, $project, $dbForProject, $queueForEvents, $hooks);
 
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -383,7 +391,6 @@ App::get('/v1/users')
     ->desc('List users')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'list')
@@ -432,7 +439,6 @@ App::get('/v1/users/:userId')
     ->desc('Get user')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'get')
@@ -458,7 +464,6 @@ App::get('/v1/users/:userId/prefs')
     ->desc('Get user preferences')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'getPrefs')
@@ -486,7 +491,6 @@ App::get('/v1/users/:userId/sessions')
     ->desc('List user sessions')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'listSessions')
@@ -528,7 +532,6 @@ App::get('/v1/users/:userId/memberships')
     ->desc('List user memberships')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'listMemberships')
@@ -568,7 +571,6 @@ App::get('/v1/users/:userId/logs')
     ->desc('List user logs')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'listLogs')
@@ -650,7 +652,6 @@ App::get('/v1/users/identities')
     ->desc('List Identities')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'listIdentities')
@@ -703,7 +704,6 @@ App::patch('/v1/users/:userId/status')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updateStatus')
@@ -739,7 +739,6 @@ App::put('/v1/users/:userId/labels')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updateLabels')
@@ -777,7 +776,6 @@ App::patch('/v1/users/:userId/verification/phone')
     ->label('scope', 'users.write')
     ->label('audits.event', 'verification.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updatePhoneVerification')
@@ -814,7 +812,6 @@ App::patch('/v1/users/:userId/name')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updateName')
@@ -852,7 +849,6 @@ App::patch('/v1/users/:userId/password')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updatePassword')
@@ -866,7 +862,8 @@ App::patch('/v1/users/:userId/password')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $password, Response $response, Document $project, Database $dbForProject, Event $queueForEvents) {
+    ->inject('hooks')
+    ->action(function (string $userId, string $password, Response $response, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
 
         $user = $dbForProject->getDocument('users', $userId);
 
@@ -881,6 +878,8 @@ App::patch('/v1/users/:userId/password')
             }
         }
 
+        $hooks->trigger('passwordValidator', [$dbForProject, $project, $password, &$user, true]);
+
         $newPassword = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS);
 
         $historyLimit = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
@@ -917,7 +916,6 @@ App::patch('/v1/users/:userId/email')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updateEmail')
@@ -973,7 +971,6 @@ App::patch('/v1/users/:userId/phone')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updatePhone')
@@ -1018,7 +1015,6 @@ App::patch('/v1/users/:userId/verification')
     ->label('audits.event', 'verification.update')
     ->label('audits.resource', 'user/{request.userId}')
     ->label('audits.userId', '{request.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updateEmailVerification')
@@ -1051,7 +1047,6 @@ App::patch('/v1/users/:userId/prefs')
     ->groups(['api', 'users'])
     ->label('event', 'users.[userId].update.prefs')
     ->label('scope', 'users.write')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updatePrefs')
@@ -1087,7 +1082,6 @@ App::delete('/v1/users/:userId/sessions/:sessionId')
     ->label('scope', 'users.write')
     ->label('audits.event', 'session.delete')
     ->label('audits.resource', 'user/{request.userId}')
-    ->label('usage.metric', 'sessions.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'deleteSession')
@@ -1127,11 +1121,10 @@ App::delete('/v1/users/:userId/sessions/:sessionId')
 App::delete('/v1/users/:userId/sessions')
     ->desc('Delete user sessions')
     ->groups(['api', 'users'])
-    ->label('event', 'users.[userId].sessions.[sessionId].delete')
+    ->label('event', 'users.[userId].sessions.delete')
     ->label('scope', 'users.write')
     ->label('audits.event', 'session.delete')
     ->label('audits.resource', 'user/{user.$id}')
-    ->label('usage.metric', 'sessions.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'deleteSessions')
@@ -1174,7 +1167,6 @@ App::delete('/v1/users/:userId')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.delete')
     ->label('audits.resource', 'user/{request.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'delete')
@@ -1211,13 +1203,12 @@ App::delete('/v1/users/:userId')
     });
 
 App::delete('/v1/users/identities/:identityId')
-    ->desc('Delete Identity')
+    ->desc('Delete identity')
     ->groups(['api', 'users'])
     ->label('event', 'users.[userId].identities.[identityId].delete')
     ->label('scope', 'users.write')
     ->label('audits.event', 'identity.delete')
     ->label('audits.resource', 'identity/{request.$identityId}')
-    ->label('usage.metric', 'users.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'deleteIdentity')
@@ -1227,7 +1218,8 @@ App::delete('/v1/users/identities/:identityId')
     ->param('identityId', '', new UID(), 'Identity ID.')
     ->inject('response')
     ->inject('dbForProject')
-    ->action(function (string $identityId, Response $response, Database $dbForProject) {
+    ->inject('queueForEvents')
+    ->action(function (string $identityId, Response $response, Database $dbForProject, Event $queueForEvents) {
 
         $identity = $dbForProject->getDocument('identities', $identityId);
 
@@ -1237,12 +1229,17 @@ App::delete('/v1/users/identities/:identityId')
 
         $dbForProject->deleteDocument('identities', $identityId);
 
+        $queueForEvents
+            ->setParam('userId', $identity->getAttribute('userId'))
+            ->setParam('identityId', $identity->getId())
+            ->setPayload($response->output($identity, Response::MODEL_IDENTITY));
+
         return $response->noContent();
     });
 
 App::get('/v1/users/usage')
     ->desc('Get usage stats for the users API')
-    ->groups(['api', 'users', 'usage'])
+    ->groups(['api', 'users'])
     ->label('scope', 'users.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'users')
@@ -1250,97 +1247,69 @@ App::get('/v1/users/usage')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_USERS)
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
-    ->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(fn ($value) => "oauth-" . $value, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->inject('register')
-    ->action(function (string $range, string $provider, Response $response, Database $dbForProject) {
+    ->action(function (string $range, Response $response, Database $dbForProject) {
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            METRIC_USERS,
+            METRIC_SESSIONS,
+        ];
 
-            $metrics = [
-                'users.$all.count.total',
-                'users.$all.requests.create',
-                'users.$all.requests.read',
-                'users.$all.requests.update',
-                'users.$all.requests.delete',
-                'sessions.$all.requests.create',
-                'sessions.$all.requests.delete',
-                "sessions.$provider.requests.create",
-            ];
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
+            foreach ($metrics as $count => $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
 
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'usersCount' => $stats['users.$all.count.total'] ?? [],
-                'usersCreate' => $stats['users.$all.requests.create'] ?? [],
-                'usersRead' => $stats['users.$all.requests.read'] ?? [],
-                'usersUpdate' => $stats['users.$all.requests.update'] ?? [],
-                'usersDelete' => $stats['users.$all.requests.delete'] ?? [],
-                'sessionsCreate' => $stats['sessions.$all.requests.create'] ?? [],
-                'sessionsProviderCreate' => $stats["sessions.$provider.requests.create"] ?? [],
-                'sessionsDelete' => $stats['sessions.$all.requests.delete' ?? []]
-            ]);
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
         }
+    }
 
-        $response->dynamic($usage, Response::MODEL_USAGE_USERS);
+        $response->dynamic(new Document([
+            'range' => $range,
+            'usersTotal'   => $usage[$metrics[0]]['total'],
+            'sessionsTotal' => $usage[$metrics[1]]['total'],
+            'users'   => $usage[$metrics[0]]['data'],
+            'sessions' => $usage[$metrics[1]]['data'],
+        ]), Response::MODEL_USAGE_USERS);
     });

app/controllers/api/vcs.php

@@ -857,10 +857,10 @@ App::post('/v1/vcs/github/events')
                 $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId);
 
                 //find functionId from functions table
-                $repositories = $dbForConsole->find('repositories', [
+                $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
                     Query::equal('providerRepositoryId', [$providerRepositoryId]),
                     Query::limit(100),
-                ]);
+                ]));
 
                 // create new deployment only on push and not when branch is created
                 if (!$providerBranchCreated) {
@@ -877,13 +877,13 @@ App::post('/v1/vcs/github/events')
                     ]);
 
                     foreach ($installations as $installation) {
-                        $repositories = $dbForConsole->find('repositories', [
+                        $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
                             Query::equal('installationInternalId', [$installation->getInternalId()]),
                             Query::limit(1000)
-                        ]);
+                        ]));
 
                         foreach ($repositories as $repository) {
-                            $dbForConsole->deleteDocument('repositories', $repository->getId());
+                            Authorization::skip(fn () => $dbForConsole->deleteDocument('repositories', $repository->getId()));
                         }
 
                         $dbForConsole->deleteDocument('installations', $installation->getId());
@@ -915,10 +915,10 @@ App::post('/v1/vcs/github/events')
                     $providerCommitAuthor = $commitDetails["commitAuthor"] ?? '';
                     $providerCommitMessage = $commitDetails["commitMessage"] ?? '';
 
-                    $repositories = $dbForConsole->find('repositories', [
+                    $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
                         Query::equal('providerRepositoryId', [$providerRepositoryId]),
                         Query::orderDesc('$createdAt')
-                    ]);
+                    ]));
 
                     $createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $external, $dbForConsole, $queueForBuilds, $getProjectDB, $request);
                 } elseif ($parsedPayload["action"] == "closed") {
@@ -929,10 +929,10 @@ App::post('/v1/vcs/github/events')
                     $external = $parsedPayload["external"] ?? true;
 
                     if ($external) {
-                        $repositories = $dbForConsole->find('repositories', [
+                        $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
                             Query::equal('providerRepositoryId', [$providerRepositoryId]),
                             Query::orderDesc('$createdAt')
-                        ]);
+                        ]));
 
                         foreach ($repositories as $repository) {
                             $providerPullRequestIds = $repository->getAttribute('providerPullRequestIds', []);
@@ -1046,8 +1046,8 @@ App::delete('/v1/vcs/installations/:installationId')
     ->inject('response')
     ->inject('project')
     ->inject('dbForConsole')
-    ->inject('deletes')
-    ->action(function (string $installationId, Response $response, Document $project, Database $dbForConsole, Delete $deletes) {
+    ->inject('queueForDeletes')
+    ->action(function (string $installationId, Response $response, Document $project, Database $dbForConsole, Delete $queueForDeletes) {
         $installation = $dbForConsole->getDocument('installations', $installationId);
 
         if ($installation->isEmpty()) {
@@ -1058,7 +1058,7 @@ App::delete('/v1/vcs/installations/:installationId')
             throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove installation from DB');
         }
 
-        $deletes
+        $queueForDeletes
             ->setType(DELETE_TYPE_DOCUMENT)
             ->setDocument($installation);
 
@@ -1092,9 +1092,9 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor
             throw new Exception(Exception::INSTALLATION_NOT_FOUND);
         }
 
-        $repository = $dbForConsole->getDocument('repositories', $repositoryId, [
+        $repository = Authorization::skip(fn () => $dbForConsole->getDocument('repositories', $repositoryId, [
             Query::equal('projectInternalId', [$project->getInternalId()])
-        ]);
+        ]));
 
         if ($repository->isEmpty()) {
             throw new Exception(Exception::REPOSITORY_NOT_FOUND);
@@ -1109,7 +1109,7 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor
 
         // TODO: Delete from array when PR is closed
 
-        $repository = $dbForConsole->updateDocument('repositories', $repository->getId(), $repository);
+        $repository = Authorization::skip(fn () => $dbForConsole->updateDocument('repositories', $repository->getId(), $repository));
 
         $privateKey = App::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY');
         $githubAppId = App::getEnv('_APP_VCS_GITHUB_APP_ID');

app/controllers/general.php

@@ -603,61 +603,57 @@ App::error()
     ->inject('response')
     ->inject('project')
     ->inject('logger')
-    ->inject('loggerBreadcrumbs')
-    ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, array $loggerBreadcrumbs) {
-
+    ->inject('log')
+    ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log) {
         $version = App::getEnv('_APP_VERSION', 'UNKNOWN');
         $route = $utopia->getRoute();
+        $publish = true;
 
-        if ($logger) {
-            if ($error->getCode() >= 500 || $error->getCode() === 0) {
-                try {
-                    /** @var Utopia\Database\Document $user */
-                    $user = $utopia->getResource('user');
-                } catch (\Throwable $th) {
-                    // All good, user is optional information for logger
-                }
+        if ($error instanceof AppwriteException) {
+            $publish = $error->isPublishable();
+        }
 
-                $log = new Utopia\Logger\Log();
-
-                if (isset($user) && !$user->isEmpty()) {
-                    $log->setUser(new User($user->getId()));
-                }
-
-                $log->setNamespace("http");
-                $log->setServer(\gethostname());
-                $log->setVersion($version);
-                $log->setType(Log::TYPE_ERROR);
-                $log->setMessage($error->getMessage());
-
-                $log->addTag('database', $project->getAttribute('database', 'console'));
-                $log->addTag('method', $route->getMethod());
-                $log->addTag('url', $route->getPath());
-                $log->addTag('verboseType', get_class($error));
-                $log->addTag('code', $error->getCode());
-                $log->addTag('projectId', $project->getId());
-                $log->addTag('hostname', $request->getHostname());
-                $log->addTag('locale', (string)$request->getParam('locale', $request->getHeader('x-appwrite-locale', '')));
-
-                $log->addExtra('file', $error->getFile());
-                $log->addExtra('line', $error->getLine());
-                $log->addExtra('trace', $error->getTraceAsString());
-                $log->addExtra('detailedTrace', $error->getTrace());
-                $log->addExtra('roles', Authorization::getRoles());
-
-                $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD");
-                $log->setAction($action);
-
-                $isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
-                $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
-
-                foreach ($loggerBreadcrumbs as $loggerBreadcrumb) {
-                    $log->addBreadcrumb($loggerBreadcrumb);
-                }
-
-                $responseCode = $logger->addLog($log);
-                Console::info('Log pushed with status code: ' . $responseCode);
+        if ($logger && ($publish || $error->getCode() === 0)) {
+            try {
+                /** @var Utopia\Database\Document $user */
+                $user = $utopia->getResource('user');
+            } catch (\Throwable $th) {
+                // All good, user is optional information for logger
             }
+
+            if (isset($user) && !$user->isEmpty()) {
+                $log->setUser(new User($user->getId()));
+            }
+
+            $log->setNamespace("http");
+            $log->setServer(\gethostname());
+            $log->setVersion($version);
+            $log->setType(Log::TYPE_ERROR);
+            $log->setMessage($error->getMessage());
+
+            $log->addTag('database', $project->getAttribute('database', 'console'));
+            $log->addTag('method', $route->getMethod());
+            $log->addTag('url', $route->getPath());
+            $log->addTag('verboseType', get_class($error));
+            $log->addTag('code', $error->getCode());
+            $log->addTag('projectId', $project->getId());
+            $log->addTag('hostname', $request->getHostname());
+            $log->addTag('locale', (string)$request->getParam('locale', $request->getHeader('x-appwrite-locale', '')));
+
+            $log->addExtra('file', $error->getFile());
+            $log->addExtra('line', $error->getLine());
+            $log->addExtra('trace', $error->getTraceAsString());
+            $log->addExtra('detailedTrace', $error->getTrace());
+            $log->addExtra('roles', Authorization::getRoles());
+
+            $action = $route->getLabel("sdk.namespace", "UNKNOWN_NAMESPACE") . '.' . $route->getLabel("sdk.method", "UNKNOWN_METHOD");
+            $log->setAction($action);
+
+            $isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
+            $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
+
+            $responseCode = $logger->addLog($log);
+            Console::info('Log pushed with status code: ' . $responseCode);
         }
 
         $code = $error->getCode();

app/controllers/shared/api.php

@@ -8,8 +8,8 @@ use Appwrite\Event\Event;
 use Appwrite\Event\Func;
 use Appwrite\Event\Mail;
 use Appwrite\Extend\Exception;
+use Appwrite\Event\Usage;
 use Appwrite\Messaging\Adapter\Realtime;
-use Appwrite\Usage\Stats;
 use Appwrite\Utopia\Response;
 use Appwrite\Utopia\Request;
 use Utopia\App;
@@ -48,43 +48,95 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar
     return $label;
 };
 
-$databaseListener = function (string $event, Document $document, Stats $usage) {
-    $multiplier = 1;
+$databaseListener = function (string $event, Document $document, Document $project, Usage $queueForUsage, Database $dbForProject) {
+
+    $value = 1;
     if ($event === Database::EVENT_DOCUMENT_DELETE) {
-        $multiplier = -1;
+        $value = -1;
     }
 
-    $collection = $document->getCollection();
-    switch ($collection) {
-        case 'users':
-            $usage->setParam('users.{scope}.count.total', 1 * $multiplier);
+    switch (true) {
+        case $document->getCollection() === 'teams':
+            $queueForUsage
+                ->addMetric(METRIC_TEAMS, $value); // per project
             break;
-        case 'databases':
-            $usage->setParam('databases.{scope}.count.total', 1 * $multiplier);
+        case $document->getCollection() === 'users':
+            $queueForUsage
+                ->addMetric(METRIC_USERS, $value); // per project
+            if ($event === Database::EVENT_DOCUMENT_DELETE) {
+                $queueForUsage
+                    ->addReduce($document);
+            }
             break;
-        case 'buckets':
-            $usage->setParam('buckets.{scope}.count.total', 1 * $multiplier);
+        case $document->getCollection() === 'sessions': // sessions
+            $queueForUsage
+                ->addMetric(METRIC_SESSIONS, $value); //per project
             break;
-        case 'deployments':
-            $usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size') * $multiplier);
+        case $document->getCollection() === 'databases': // databases
+            $queueForUsage
+                ->addMetric(METRIC_DATABASES, $value); // per project
+
+            if ($event === Database::EVENT_DOCUMENT_DELETE) {
+                $queueForUsage
+                    ->addReduce($document);
+            }
+            break;
+        case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
+            $parts = explode('_', $document->getCollection());
+            $databaseInternalId = $parts[1] ?? 0;
+            $queueForUsage
+                ->addMetric(METRIC_COLLECTIONS, $value) // per project
+                ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database
+            ;
+
+            if ($event === Database::EVENT_DOCUMENT_DELETE) {
+                $queueForUsage
+                    ->addReduce($document);
+            }
+            break;
+        case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents
+            $parts = explode('_', $document->getCollection());
+            $databaseInternalId   = $parts[1] ?? 0;
+            $collectionInternalId = $parts[3] ?? 0;
+            $queueForUsage
+                ->addMetric(METRIC_DOCUMENTS, $value)  // per project
+                ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database
+                ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value);  // per collection
+            break;
+        case $document->getCollection() === 'buckets': //buckets
+            $queueForUsage
+                ->addMetric(METRIC_BUCKETS, $value); // per project
+            if ($event === Database::EVENT_DOCUMENT_DELETE) {
+                $queueForUsage
+                    ->addReduce($document);
+            }
+            break;
+        case str_starts_with($document->getCollection(), 'bucket_'): // files
+            $parts = explode('_', $document->getCollection());
+            $bucketInternalId  = $parts[1];
+            $queueForUsage
+                ->addMetric(METRIC_FILES, $value) // per project
+                ->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project
+                ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES), $value) // per bucket
+                ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket
+            break;
+        case $document->getCollection() === 'functions':
+            $queueForUsage
+                ->addMetric(METRIC_FUNCTIONS, $value); // per project
+
+            if ($event === Database::EVENT_DOCUMENT_DELETE) {
+                $queueForUsage
+                    ->addReduce($document);
+            }
+            break;
+        case $document->getCollection() === 'deployments':
+            $queueForUsage
+                ->addMetric(METRIC_DEPLOYMENTS, $value) // per project
+                ->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project
+                ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value)// per function
+                ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), $document->getAttribute('size') * $value);
             break;
         default:
-            if (strpos($collection, 'bucket_') === 0) {
-                $usage
-                    ->setParam('bucketId', $document->getAttribute('bucketId'))
-                    ->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal') * $multiplier)
-                    ->setParam('files.{scope}.count.total', 1 * $multiplier);
-            } elseif (strpos($collection, 'database_') === 0) {
-                $usage
-                    ->setParam('databaseId', $document->getAttribute('databaseId'));
-                if (strpos($collection, '_collection_') !== false) {
-                    $usage
-                        ->setParam('collectionId', $document->getAttribute('$collectionId'))
-                        ->setParam('documents.{scope}.count.total', 1 * $multiplier);
-                } else {
-                    $usage->setParam('collections.{scope}.count.total', 1 * $multiplier);
-                }
-            }
             break;
     }
 };
@@ -103,8 +155,8 @@ App::init()
     ->inject('dbForProject')
     ->inject('mode')
     ->inject('queueForMails')
-    ->inject('usage')
-    ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, string $mode, Mail $queueForMails, Stats $usage) use ($databaseListener) {
+    ->inject('queueForUsage')
+    ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, string $mode, Mail $queueForMails, Usage $queueForUsage) use ($databaseListener) {
 
         $route = $utopia->getRoute();
 
@@ -121,13 +173,17 @@ App::init()
         $abuseKeyLabel = (!is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel;
 
         foreach ($abuseKeyLabel as $abuseKey) {
+            $start = $request->getContentRangeStart();
+            $end = $request->getContentRangeEnd();
             $timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $dbForProject);
             $timeLimit
+                ->setParam('{projectId}', $project->getId())
                 ->setParam('{userId}', $user->getId())
                 ->setParam('{userAgent}', $request->getUserAgent(''))
                 ->setParam('{ip}', $request->getIP())
                 ->setParam('{url}', $request->getHostname() . $route->getPath())
-                ->setParam('{method}', $request->getMethod());
+                ->setParam('{method}', $request->getMethod())
+                ->setParam('{chunkId}', (int) ($start / ($end + 1 - $start)));
             $timeLimitArray[] = $timeLimit;
         }
 
@@ -186,19 +242,14 @@ App::init()
             ->setProject($project)
             ->setUser($user);
 
-        $usage
-            ->setParam('projectInternalId', $project->getInternalId())
-            ->setParam('projectId', $project->getId())
-            ->setParam('project.{scope}.network.requests', 1)
-            ->setParam('httpMethod', $request->getMethod())
-            ->setParam('project.{scope}.network.inbound', 0)
-            ->setParam('project.{scope}.network.outbound', 0);
 
         $queueForDeletes->setProject($project);
         $queueForDatabase->setProject($project);
 
-        $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage));
-        $dbForProject->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage));
+        $dbForProject
+            ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject))
+            ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject))
+        ;
 
         $useCache = $route->getLabel('cache', false);
 
@@ -285,7 +336,7 @@ App::init()
                 break;
 
             case 'magic-url':
-                if ($project->getAttribute('usersAuthMagicURL', true) === false) {
+                if (($auths['usersAuthMagicURL'] ?? true) === false) {
                     throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Magic URL authentication is disabled for this project');
                 }
                 break;
@@ -296,6 +347,12 @@ App::init()
                 }
                 break;
 
+            case 'phone':
+                if (($auths['phone'] ?? true) === false) {
+                    throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Phone authentication is disabled for this project');
+                }
+                break;
+
             case 'invites':
                 if (($auths['invites'] ?? true) === false) {
                     throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Invites authentication is disabled for this project');
@@ -309,7 +366,7 @@ App::init()
                 break;
 
             default:
-                throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Unsupported authentication route');
+                throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Unsupported authentication type: ' . $route->getLabel('auth.type', ''));
                 break;
         }
     });
@@ -362,14 +419,14 @@ App::shutdown()
     ->inject('user')
     ->inject('queueForEvents')
     ->inject('queueForAudits')
-    ->inject('usage')
+    ->inject('queueForUsage')
     ->inject('queueForDeletes')
     ->inject('queueForDatabase')
     ->inject('dbForProject')
     ->inject('queueForFunctions')
     ->inject('mode')
     ->inject('dbForConsole')
-    ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Stats $usage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) {
+    ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) {
 
         $responsePayload = $response->getPayload();
 
@@ -522,35 +579,48 @@ App::shutdown()
             }
         }
 
-        if (
-            App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled'
-            && $project->getId()
-            && !empty($route->getLabel('sdk.namespace', null))
-        ) { // Don't calculate console usage on admin mode
-            $metric = $route->getLabel('usage.metric', '');
-            $usageParams = $route->getLabel('usage.params', []);
 
-            if (!empty($metric)) {
-                $usage->setParam($metric, 1);
-                foreach ($usageParams as $param) {
-                    $param = $parseLabel($param, $responsePayload, $requestParams, $user);
-                    $parts = explode(':', $param);
-                    if (count($parts) != 2) {
-                        throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Usage params not properly set');
-                    }
-                    $usage->setParam($parts[0], $parts[1]);
+
+        if ($project->getId() !== 'console') {
+            if ($mode !== APP_MODE_ADMIN) {
+                $fileSize = 0;
+                $file = $request->getFiles('file');
+                if (!empty($file)) {
+                    $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
+                }
+
+                $queueForUsage
+                    ->addMetric(METRIC_NETWORK_REQUESTS, 1)
+                    ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize)
+                    ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize());
+            }
+
+            $queueForUsage
+                ->setProject($project)
+                ->trigger();
+        }
+
+        /**
+         * Update user last activity
+         */
+        if (!$user->isEmpty()) {
+            $accessedAt = $user->getAttribute('accessedAt', '');
+            if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCCESS)) > $accessedAt) {
+                $user->setAttribute('accessedAt', DateTime::now());
+
+                if (APP_MODE_ADMIN !== $mode) {
+                    $dbForProject->updateDocument('users', $user->getId(), $user);
+                } else {
+                    $dbForConsole->updateDocument('users', $user->getId(), $user);
                 }
             }
+        }
+    });
 
-            $fileSize = 0;
-            $file = $request->getFiles('file');
-            if (!empty($file)) {
-                $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
-            }
-
-            $usage
-                ->setParam('project.{scope}.network.inbound', $request->getSize() + $fileSize)
-                ->setParam('project.{scope}.network.outbound', $response->getSize())
-                ->submit();
+App::init()
+    ->groups(['usage'])
+    ->action(function () {
+        if (App::getEnv('_APP_USAGE_STATS', 'enabled') !== 'enabled') {
+            throw new Exception(Exception::GENERAL_USAGE_DISABLED);
         }
     });

app/controllers/shared/api/auth.php

@@ -32,7 +32,7 @@ App::init()
                 break;
 
             case 'magic-url':
-                if ($project->getAttribute('usersAuthMagicURL', true) === false) {
+                if (($auths['usersAuthMagicURL'] ?? true) === false) {
                     throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Magic URL authentication is disabled for this project');
                 }
                 break;
@@ -43,6 +43,12 @@ App::init()
                 }
                 break;
 
+            case 'phone':
+                if (($auths['phone'] ?? true) === false) {
+                    throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Phone authentication is disabled for this project');
+                }
+                break;
+
             case 'invites':
                 if (($auths['invites'] ?? true) === false) {
                     throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Invites authentication is disabled for this project');

app/http.php

@@ -263,10 +263,9 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
                 // All good, user is optional information for logger
             }
 
-            $loggerBreadcrumbs = $app->getResource("loggerBreadcrumbs");
             $route = $app->getRoute();
 
-            $log = new Utopia\Logger\Log();
+            $log = $app->getResource("log");
 
             if (isset($user) && !$user->isEmpty()) {
                 $log->setUser(new User($user->getId()));
@@ -298,10 +297,6 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
             $isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
             $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
 
-            foreach ($loggerBreadcrumbs as $loggerBreadcrumb) {
-                $log->addBreadcrumb($loggerBreadcrumb);
-            }
-
             $responseCode = $logger->addLog($log);
             Console::info('Log pushed with status code: ' . $responseCode);
         }

app/init.php

@@ -19,6 +19,7 @@ ini_set('default_socket_timeout', -1);
 error_reporting(E_ALL);
 
 use Appwrite\Event\Migration;
+use Appwrite\Event\Usage;
 use Appwrite\Extend\Exception;
 use Appwrite\Auth\Auth;
 use Appwrite\Event\Audit;
@@ -32,7 +33,6 @@ use Appwrite\Network\Validator\Email;
 use Appwrite\Network\Validator\Origin;
 use Appwrite\OpenSSL\OpenSSL;
 use Appwrite\URL\URL as AppwriteURL;
-use Appwrite\Usage\Stats;
 use Utopia\App;
 use Utopia\Logger\Logger;
 use Utopia\Cache\Adapter\Redis as RedisCache;
@@ -72,14 +72,17 @@ use Ahc\Jwt\JWTException;
 use Appwrite\Event\Build;
 use Appwrite\Event\Certificate;
 use Appwrite\Event\Func;
+use Appwrite\Hooks\Hooks;
 use MaxMind\Db\Reader;
 use PHPMailer\PHPMailer\PHPMailer;
 use Swoole\Database\PDOProxy;
+use Utopia\Logger\Log;
 use Utopia\Queue;
 use Utopia\Queue\Connection;
 use Utopia\Storage\Storage;
 use Utopia\VCS\Adapter\Git\GitHub as VcsGitHub;
 use Utopia\Validator\Range;
+use Utopia\Validator\Hostname;
 use Utopia\Validator\IP;
 use Utopia\Validator\URL;
 use Utopia\Validator\WhiteList;
@@ -109,8 +112,8 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return
 const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
-const APP_CACHE_BUSTER = 515;
-const APP_VERSION_STABLE = '1.4.9';
+const APP_CACHE_BUSTER = 329;
+const APP_VERSION_STABLE = '1.4.13';
 const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
 const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
 const APP_DATABASE_ATTRIBUTE_IP = 'ip';
@@ -173,10 +176,6 @@ const DELETE_TYPE_SESSIONS = 'sessions';
 const DELETE_TYPE_CACHE_BY_TIMESTAMP = 'cacheByTimeStamp';
 const DELETE_TYPE_CACHE_BY_RESOURCE  = 'cacheByResource';
 const DELETE_TYPE_SCHEDULES = 'schedules';
-// Compression type
-const COMPRESSION_TYPE_NONE = 'none';
-const COMPRESSION_TYPE_GZIP = 'gzip';
-const COMPRESSION_TYPE_ZSTD = 'zstd';
 // Mail Types
 const MAIL_TYPE_VERIFICATION = 'verification';
 const MAIL_TYPE_MAGIC_SESSION = 'magicSession';
@@ -242,6 +241,7 @@ Config::load('platforms', __DIR__ . '/config/platforms.php');
 Config::load('collections', __DIR__ . '/config/collections.php');
 Config::load('runtimes', __DIR__ . '/config/runtimes.php');
 Config::load('runtimes-v2', __DIR__ . '/config/runtimes-v2.php');
+Config::load('usage', __DIR__ . '/config/usage.php');
 Config::load('roles', __DIR__ . '/config/roles.php');  // User roles and scopes
 Config::load('scopes', __DIR__ . '/config/scopes.php');  // User roles and scopes
 Config::load('services', __DIR__ . '/config/services.php');  // List of services
@@ -767,31 +767,26 @@ $register->set('pools', function () {
     return $group;
 });
 
-$register->set('influxdb', function () {
+$register->set('db', function () {
+    // This is usually for our workers or CLI commands scope
+       $dbHost = App::getEnv('_APP_DB_HOST', '');
+       $dbPort = App::getEnv('_APP_DB_PORT', '');
+       $dbUser = App::getEnv('_APP_DB_USER', '');
+       $dbPass = App::getEnv('_APP_DB_PASS', '');
+       $dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
 
- // Register DB connection
-    $host = App::getEnv('_APP_INFLUXDB_HOST', '');
-    $port = App::getEnv('_APP_INFLUXDB_PORT', '');
+       $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array(
+           PDO::ATTR_TIMEOUT => 3, // Seconds
+           PDO::ATTR_PERSISTENT => true,
+           PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+           PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
+           PDO::ATTR_EMULATE_PREPARES => true,
+           PDO::ATTR_STRINGIFY_FETCHES => true,
+       ));
 
-    if (empty($host) || empty($port)) {
-        return;
-    }
-    $driver = new InfluxDB\Driver\Curl("http://{$host}:{$port}");
-    $client = new InfluxDB\Client($host, $port, '', '', false, false, 5);
-    $client->setDriver($driver);
-
-    return $client;
+       return $pdo;
 });
-$register->set('statsd', function () {
-    // Register DB connection
-    $host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
-    $port = App::getEnv('_APP_STATSD_PORT', 8125);
 
-    $connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port);
-    $statsd = new \Domnikl\Statsd\Client($connection);
-
-    return $statsd;
-});
 $register->set('smtp', function () {
     $mail = new PHPMailer(true);
 
@@ -832,6 +827,9 @@ $register->set('passwordsDictionary', function () {
 $register->set('promiseAdapter', function () {
     return new Swoole();
 });
+$register->set('hooks', function () {
+    return new Hooks();
+});
 /*
  * Localization
  */
@@ -867,13 +865,14 @@ foreach ($locales as $locale) {
 ]);
 
 // Runtime Execution
+App::setResource('log', fn() => new Log());
 App::setResource('logger', function ($register) {
     return $register->get('logger');
 }, ['register']);
 
-App::setResource('loggerBreadcrumbs', function () {
-    return [];
-});
+App::setResource('hooks', function ($register) {
+    return $register->get('hooks');
+}, ['register']);
 
 App::setResource('register', fn() => $register);
 App::setResource('locale', fn() => new Locale(App::getEnv('_APP_LOCALE', 'en')));
@@ -910,15 +909,15 @@ App::setResource('queueForAudits', function (Connection $queue) {
 App::setResource('queueForFunctions', function (Connection $queue) {
     return new Func($queue);
 }, ['queue']);
+App::setResource('queueForUsage', function (Connection $queue) {
+    return new Usage($queue);
+}, ['queue']);
 App::setResource('queueForCertificates', function (Connection $queue) {
     return new Certificate($queue);
 }, ['queue']);
 App::setResource('queueForMigrations', function (Connection $queue) {
     return new Migration($queue);
 }, ['queue']);
-App::setResource('usage', function ($register) {
-    return new Stats($register->get('statsd'));
-}, ['register']);
 App::setResource('clients', function ($request, $console, $project) {
     $console->setAttribute('platforms', [ // Always allow current host
         '$collection' => ID::custom('platforms'),
@@ -927,6 +926,21 @@ App::setResource('clients', function ($request, $console, $project) {
         'hostname' => $request->getHostname(),
     ], Document::SET_TYPE_APPEND);
 
+    $hostnames = explode(',', App::getEnv('_APP_CONSOLE_HOSTNAMES', ''));
+    $validator = new Hostname();
+    foreach ($hostnames as $hostname) {
+        $hostname = trim($hostname);
+        if (!$validator->isValid($hostname)) {
+            continue;
+        }
+        $console->setAttribute('platforms', [
+            '$collection' => ID::custom('platforms'),
+            'type' => Origin::CLIENT_TYPE_WEB,
+            'name' => $hostname,
+            'hostname' => $hostname,
+        ], Document::SET_TYPE_APPEND);
+    }
+
     /**
      * Get All verified client URLs for both console and current projects
      * + Filter for duplicated entries

app/views/install/compose.phtml

@@ -71,7 +71,6 @@ services:
       - mariadb
       - redis
 #      - clamav
-      - influxdb
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
@@ -79,6 +78,7 @@ services:
       - _APP_CONSOLE_WHITELIST_ROOT
       - _APP_CONSOLE_WHITELIST_EMAILS
       - _APP_CONSOLE_WHITELIST_IPS
+      - _APP_CONSOLE_HOSTNAMES
       - _APP_SYSTEM_EMAIL_NAME
       - _APP_SYSTEM_EMAIL_ADDRESS
       - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
@@ -106,8 +106,6 @@ services:
       - _APP_SMTP_USERNAME
       - _APP_SMTP_PASSWORD
       - _APP_USAGE_STATS
-      - _APP_INFLUXDB_HOST
-      - _APP_INFLUXDB_PORT
       - _APP_STORAGE_LIMIT
       - _APP_STORAGE_PREVIEW_LIMIT
       - _APP_STORAGE_ANTIVIRUS
@@ -144,8 +142,6 @@ services:
       - _APP_EXECUTOR_HOST
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
-      - _APP_STATSD_HOST
-      - _APP_STATSD_PORT
       - _APP_MAINTENANCE_INTERVAL
       - _APP_MAINTENANCE_RETENTION_EXECUTION
       - _APP_MAINTENANCE_RETENTION_CACHE
@@ -273,7 +269,7 @@ services:
     depends_on:
       - redis
       - mariadb
-    volumes: 
+    volumes:
       - appwrite-uploads:/storage/uploads:rw
       - appwrite-cache:/storage/cache:rw
       - appwrite-functions:/storage/functions:rw
@@ -420,7 +416,7 @@ services:
     depends_on:
       - redis
       - mariadb
-    volumes: 
+    volumes:
       - appwrite-config:/storage/config:rw
       - appwrite-certificates:/storage/certificates:rw
     environment:
@@ -595,16 +591,15 @@ services:
       - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
       - _APP_MAINTENANCE_RETENTION_SCHEDULES
 
-  appwrite-usage:
+  appwrite-worker-usage:
     image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
     entrypoint: usage
-    container_name: appwrite-usage
+    container_name: appwrite-worker-usage
     <<: *x-logging
     restart: unless-stopped
     networks:
       - appwrite
     depends_on:
-      - influxdb
       - mariadb
     environment:
       - _APP_ENV
@@ -615,9 +610,6 @@ services:
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_INFLUXDB_HOST
-      - _APP_INFLUXDB_PORT
-      - _APP_USAGE_AGGREGATION_INTERVAL
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
@@ -652,7 +644,7 @@ services:
       - _APP_DB_PASS
 
   appwrite-assistant:
-    image: appwrite/assistant:0.2.2
+    image: appwrite/assistant:0.3.0
     container_name: appwrite-assistant
     <<: *x-logging
     restart: unless-stopped
@@ -751,27 +743,6 @@ services:
   #   volumes:
   #     - appwrite-uploads:/storage/uploads
 
-  influxdb:
-    image: appwrite/influxdb:1.5.0
-    container_name: appwrite-influxdb
-    <<: *x-logging
-    restart: unless-stopped
-    networks:
-      - appwrite
-    volumes:
-      - appwrite-influxdb:/var/lib/influxdb:rw
-
-  telegraf:
-    image: appwrite/telegraf:1.4.0
-    container_name: appwrite-telegraf
-    <<: *x-logging
-    restart: unless-stopped
-    networks:
-      - appwrite
-    environment:
-      - _APP_INFLUXDB_HOST
-      - _APP_INFLUXDB_PORT
-
 networks:
   gateway:
     name: gateway
@@ -788,5 +759,4 @@ volumes:
   appwrite-certificates:
   appwrite-functions:
   appwrite-builds:
-  appwrite-influxdb:
   appwrite-config:

app/worker.php

@@ -9,12 +9,12 @@ use Appwrite\Event\Certificate;
 use Appwrite\Event\Database as EventDatabase;
 use Appwrite\Event\Delete;
 use Appwrite\Event\Func;
+use Appwrite\Event\Hamster;
 use Appwrite\Event\Mail;
-use Appwrite\Event\Messaging;
 use Appwrite\Event\Migration;
 use Appwrite\Event\Phone;
+use Appwrite\Event\Usage;
 use Appwrite\Platform\Appwrite;
-use Appwrite\Usage\Stats;
 use Swoole\Runtime;
 use Utopia\App;
 use Utopia\Cache\Adapter\Sharding;
@@ -22,6 +22,7 @@ use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
 use Utopia\Database\Database;
+use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Platform\Service;
@@ -72,6 +73,17 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register,
     return $adapter;
 }, ['cache', 'register', 'message', 'dbForConsole']);
 
+Server::setResource('project', function (Message $message, Database $dbForConsole) {
+    $payload = $message->getPayload() ?? [];
+    $project = new Document($payload['project'] ?? []);
+
+    if ($project->getId() === 'console') {
+        return $project;
+    }
+    return $dbForConsole->getDocument('projects', $project->getId());
+    ;
+}, ['message', 'dbForConsole']);
+
 Server::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) {
     $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools
 
@@ -103,6 +115,18 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso
     };
 }, ['pools', 'dbForConsole', 'cache']);
 
+Server::setResource('abuseRetention', function () {
+    return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400));
+});
+
+Server::setResource('auditRetention', function () {
+    return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600));
+});
+
+Server::setResource('executionRetention', function () {
+    return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600));
+});
+
 Server::setResource('cache', function (Registry $register) {
     $pools = $register->get('pools');
     $list = Config::getParam('pools-cache', []);
@@ -119,9 +143,9 @@ Server::setResource('cache', function (Registry $register) {
     return new Cache(new Sharding($adapters));
 }, ['register']);
 Server::setResource('log', fn() => new Log());
-Server::setResource('usage', function ($register) {
-    return new Stats($register->get('statsd'));
-}, ['register']);
+Server::setResource('queueForUsage', function (Connection $queue) {
+    return new Usage($queue);
+}, ['queue']);
 Server::setResource('queue', function (Group $pools) {
     return $pools->get('queue')->pop()->getResource();
 }, ['pools']);
@@ -155,6 +179,9 @@ Server::setResource('queueForCertificates', function (Connection $queue) {
 Server::setResource('queueForMigrations', function (Connection $queue) {
     return new Migration($queue);
 }, ['queue']);
+Server::setResource('queueForHamster', function (Connection $queue) {
+    return new Hamster($queue);
+}, ['queue']);
 Server::setResource('logger', function (Registry $register) {
     return $register->get('logger');
 }, ['register']);
@@ -272,9 +299,12 @@ $worker
         Console::error('[Error] Line: ' . $error->getLine());
     });
 
-     $worker->workerStart()
-         ->action(function () use ($workerName) {
-             Console::info("Worker $workerName  started");
-         });
+try {
+    $workerStart = $worker->getWorkerStart();
+} catch (\Throwable $error) {
+    $worker->workerStart();
+} finally {
+    Console::info("Worker $workerName  started");
+}
 
-     $worker->start();
+$worker->start();

bin/calc-tier-stats

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-php /usr/src/code/app/cli.php calc-tier-stats $@

bin/calc-users-stats

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-php /usr/src/code/app/cli.php calc-users-stats $@

bin/clear-card-cache

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-php /usr/src/code/app/cli.php clear-card-cache $@

bin/create-inf-metric

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php create-inf-metric $@

bin/delete-orphaned-projects

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-php /usr/src/code/app/cli.php delete-orphaned-projects $@

bin/hamster

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-php /usr/src/code/app/cli.php hamster $@

bin/patch-recreate-repositories-documents

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php patch-recreate-repositories-documents $@

bin/queue-count-failed

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php queue-count --type=failed $@

bin/queue-count-processing

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php queue-count --type=processing $@

bin/queue-count-success

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php queue-count --type=success $@

bin/queue-retry

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php queue-retry $@

bin/usage

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-php /usr/src/code/app/cli.php usage $@

bin/worker-usage

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/worker.php usage $@

composer.json

@@ -43,16 +43,16 @@
         "ext-sockets": "*",
         "appwrite/php-runtimes": "0.13.*",
         "appwrite/php-clamav": "2.0.*",
-        "utopia-php/abuse": "0.32.*",
+        "utopia-php/abuse": "0.33.*",
         "utopia-php/analytics": "0.10.*",
-        "utopia-php/audit": "0.34.*",
-        "utopia-php/cache": "0.8.*",
+        "utopia-php/audit": "0.35.*",
+        "utopia-php/cache": "0.9.*",
         "utopia-php/cli": "0.15.*",
         "utopia-php/config": "0.2.*",
-        "utopia-php/database": "0.44.*",
+        "utopia-php/database": "0.45.*",
         "utopia-php/domains": "0.3.*",
         "utopia-php/dsn": "0.1.*",
-        "utopia-php/framework": "0.31.0",
+        "utopia-php/framework": "0.33.*",
         "utopia-php/image": "0.5.*",
         "utopia-php/locale": "0.4.*",
         "utopia-php/logger": "0.3.*",
@@ -62,7 +62,7 @@
         "utopia-php/platform": "0.5.*",
         "utopia-php/pools": "0.4.*",
         "utopia-php/preloader": "0.2.*",
-        "utopia-php/queue": "0.5.*",
+        "utopia-php/queue": "0.7.*",
         "utopia-php/registry": "0.5.*",
         "utopia-php/storage": "0.18.*",
         "utopia-php/swoole": "0.5.*",
@@ -70,12 +70,10 @@
         "utopia-php/websocket": "0.1.*",
         "matomo/device-detector": "6.1.*",
         "dragonmantank/cron-expression": "3.3.2",
-        "influxdb/influxdb-php": "1.15.2",
         "phpmailer/phpmailer": "6.8.0",
         "chillerlan/php-qrcode": "4.3.4",
         "adhocore/jwt": "1.1.2",
         "webonyx/graphql-php": "14.11.*",
-        "slickdeals/statsd": "3.1.0",
         "league/csv": "9.7.1"
     },
     "repositories": [
@@ -86,7 +84,7 @@
     ],
     "require-dev": {
         "ext-fileinfo": "*",
-        "appwrite/sdk-generator": "0.35.*",
+        "appwrite/sdk-generator": "0.36.*",
         "phpunit/phpunit": "9.5.20",
         "squizlabs/php_codesniffer": "^3.7",
         "swoole/ide-helper": "5.0.2",

docker-compose.yml

@@ -53,7 +53,7 @@ services:
         DEBUG: false
         TESTING: true
         VERSION: dev
-    ports: 
+    ports:
       - 9501:80
     networks:
       - appwrite
@@ -88,7 +88,7 @@ services:
       - mariadb
       - redis
       # - clamav
-    entrypoint: 
+    entrypoint:
       - php
       - -e
       - app/http.php
@@ -100,6 +100,7 @@ services:
       - _APP_CONSOLE_WHITELIST_ROOT
       - _APP_CONSOLE_WHITELIST_EMAILS
       - _APP_CONSOLE_WHITELIST_IPS
+      - _APP_CONSOLE_HOSTNAMES
       - _APP_SYSTEM_EMAIL_NAME
       - _APP_SYSTEM_EMAIL_ADDRESS
       - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
@@ -127,8 +128,6 @@ services:
       - _APP_SMTP_USERNAME
       - _APP_SMTP_PASSWORD
       - _APP_USAGE_STATS
-      - _APP_INFLUXDB_HOST
-      - _APP_INFLUXDB_PORT
       - _APP_STORAGE_LIMIT
       - _APP_STORAGE_PREVIEW_LIMIT
       - _APP_STORAGE_ANTIVIRUS
@@ -165,8 +164,6 @@ services:
       - _APP_EXECUTOR_HOST
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
-      - _APP_STATSD_HOST
-      - _APP_STATSD_PORT
       - _APP_MAINTENANCE_INTERVAL
       - _APP_MAINTENANCE_RETENTION_EXECUTION
       - _APP_MAINTENANCE_RETENTION_CACHE
@@ -195,7 +192,7 @@ services:
     container_name: appwrite-realtime
     image: appwrite-dev
     restart: unless-stopped
-    ports: 
+    ports:
       - 9505:80
     labels:
       - "traefik.enable=true"
@@ -303,7 +300,7 @@ services:
     depends_on:
       - redis
       - mariadb
-    volumes: 
+    volumes:
       - appwrite-uploads:/storage/uploads:rw
       - appwrite-cache:/storage/cache:rw
       - appwrite-functions:/storage/functions:rw
@@ -357,7 +354,7 @@ services:
     image: appwrite-dev
     networks:
       - appwrite
-    volumes: 
+    volumes:
       - ./app:/usr/src/code/app
       - ./src:/usr/src/code/src
     depends_on:
@@ -456,7 +453,7 @@ services:
     depends_on:
       - redis
       - mariadb
-    volumes: 
+    volumes:
       - appwrite-config:/storage/config:rw
       - appwrite-certificates:/storage/certificates:rw
       - ./app:/usr/src/code/app
@@ -574,6 +571,7 @@ services:
       - _APP_REDIS_PASS
       - _APP_SMS_PROVIDER
       - _APP_SMS_FROM
+      - _APP_SMS_PROJECTS_DENY_LIST
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -648,19 +646,18 @@ services:
       - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
       - _APP_MAINTENANCE_RETENTION_SCHEDULES
 
-  appwrite-usage:
-    entrypoint: usage
+  appwrite-worker-usage:
+    entrypoint: worker-usage
     <<: *x-logging
-    container_name: appwrite-usage
+    container_name: appwrite-worker-usage
     image: appwrite-dev
     networks:
       - appwrite
     volumes:
       - ./app:/usr/src/code/app
       - ./src:/usr/src/code/src
-      - ./dev:/usr/local/dev
     depends_on:
-      - influxdb
+      - redis
       - mariadb
     environment:
       - _APP_ENV
@@ -671,9 +668,6 @@ services:
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_INFLUXDB_HOST
-      - _APP_INFLUXDB_PORT
-      - _APP_USAGE_AGGREGATION_INTERVAL
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
@@ -681,6 +675,7 @@ services:
       - _APP_USAGE_STATS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
+      - _APP_USAGE_AGGREGATION_INTERVAL
 
   appwrite-schedule:
     entrypoint: schedule
@@ -711,7 +706,7 @@ services:
 
   appwrite-assistant:
     container_name: appwrite-assistant
-    image: appwrite/assistant:0.2.2
+    image: appwrite/assistant:0.3.0
     networks:
       - appwrite
     environment:
@@ -843,26 +838,6 @@ services:
   #     - appwrite
   #   volumes:
   #     - appwrite-uploads:/storage/uploads
-
-  influxdb:
-    image: appwrite/influxdb:1.5.0
-    container_name: appwrite-influxdb
-    <<: *x-logging
-    networks:
-      - appwrite
-    volumes:
-      - appwrite-influxdb:/var/lib/influxdb:rw
-
-  telegraf:
-    image: appwrite/telegraf:1.4.0
-    container_name: appwrite-telegraf
-    <<: *x-logging
-    networks:
-      - appwrite
-    environment:
-      - _APP_INFLUXDB_HOST
-      - _APP_INFLUXDB_PORT
-
   # Dev Tools Start ------------------------------------------------------------------------------------------
   #
   # The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack
@@ -872,7 +847,6 @@ services:
   # MailCatcher - An SMTP server. Catches all system emails and displays them in a nice UI.
   # RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks
   # RedisCommander - A nice UI for exploring Redis data
-  # Chronograf - A nice UI for exploring InfluxDB data
   # Webgrind - A nice UI for exploring and debugging code-level stuff
 
   maildev: # used mainly for dev tests
@@ -912,33 +886,13 @@ services:
   #   - REDIS_HOSTS=redis
   #   ports:
   #   - "8081:8081"
-
-  # chronograf:
-  #   image: chronograf:1.6
-  #   container_name: appwrite-chronograf
-  #   restart: unless-stopped
-  #   networks:
-  #   - appwrite
-  #   volumes:
-  #   - appwrite-chronograf:/var/lib/chronograf
-  #   ports:
-  #   - "8888:8888"
-  #   environment:
-  #   - INFLUXDB_URL=http://influxdb:8086
-  #   - KAPACITOR_URL=http://kapacitor:9092
-  #   - AUTH_DURATION=48h
-  #   - TOKEN_SECRET=duperduper5674829!jwt
-  #   - GH_CLIENT_ID=d86f7145a41eacfc52cc
-  #   - GH_CLIENT_SECRET=9e0081062367a2134e7f2ea95ba1a32d08b6c8ab
-  #   - GH_ORGS=appwrite
-
   # webgrind:
   #   image: 'jokkedk/webgrind:latest'
   #   volumes:
   #     - './debug:/tmp'
   #   ports:
   #     - '3001:80'
-  
+
   graphql-explorer:
     container_name: appwrite-graphql-explorer
     image: appwrite/altair:0.3.0
@@ -969,6 +923,4 @@ volumes:
   appwrite-certificates:
   appwrite-functions:
   appwrite-builds:
-  appwrite-influxdb:
   appwrite-config:
-  # appwrite-chronograf:

docs/examples/1.4.x/console-cli/examples/health/get-queue-builds.md

@@ -1 +1,2 @@
-appwrite health getQueueBuilds
+appwrite health getQueueBuilds \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-certificates.md

@@ -1 +1,2 @@
-appwrite health getQueueCertificates
+appwrite health getQueueCertificates \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-databases.md

@@ -1,2 +1,3 @@
 appwrite health getQueueDatabases \
 
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-deletes.md

@@ -1 +1,2 @@
-appwrite health getQueueDeletes
+appwrite health getQueueDeletes \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-functions.md

@@ -1 +1,2 @@
-appwrite health getQueueFunctions
+appwrite health getQueueFunctions \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-logs.md

@@ -1 +1,2 @@
-appwrite health getQueueLogs
+appwrite health getQueueLogs \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-mails.md

@@ -1 +1,2 @@
-appwrite health getQueueMails
+appwrite health getQueueMails \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-messaging.md

@@ -1 +1,2 @@
-appwrite health getQueueMessaging
+appwrite health getQueueMessaging \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-migrations.md

@@ -1 +1,2 @@
-appwrite health getQueueMigrations
+appwrite health getQueueMigrations \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-webhooks.md

@@ -1 +1,2 @@
-appwrite health getQueueWebhooks
+appwrite health getQueueWebhooks \
+

docs/references/account/delete.md

@@ -0,0 +1 @@
+Delete the currently logged in user.

docs/references/health/get-failed-queue-jobs.md

@@ -0,0 +1 @@
+Returns the amount of failed jobs in a given queue.

docs/sdks/dart/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 10.1.0
+
+* Add new queue health endpoints
+* Fix between queries
+
 ## 10.0.0
 
 * Parameter `url` is now optional in the `createMembership` endpoint

docs/sdks/flutter/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 11.0.1
+
+* Fix between queries
+
 ## 11.0.0
 
 * Parameter `url` is now optional in the `createMembership` endpoint

src/Appwrite/Auth/Validator/PasswordHistory.php

@@ -44,7 +44,7 @@ class PasswordHistory extends Password
     public function isValid($value): bool
     {
         foreach ($this->history as $hash) {
-            if (Auth::passwordVerify($value, $hash, $this->algo, $this->algoOptions)) {
+            if (!empty($hash) && Auth::passwordVerify($value, $hash, $this->algo, $this->algoOptions)) {
                 return false;
             }
         }

src/Appwrite/Event/Event.php

@@ -42,6 +42,7 @@ class Event
     public const MIGRATIONS_QUEUE_NAME = 'v1-migrations';
     public const MIGRATIONS_CLASS_NAME = 'MigrationsV1';
 
+
     protected string $queue = '';
     protected string $class = '';
     protected string $event = '';

src/Appwrite/Event/Mail.php

@@ -13,6 +13,8 @@ class Mail extends Event
     protected string $body = '';
     protected array $smtp = [];
     protected array $variables = [];
+    protected string $bodyTemplate = '';
+    protected array $attachment = [];
 
     public function __construct(protected Connection $connection)
     {
@@ -115,6 +117,29 @@ class Mail extends Event
         return $this->name;
     }
 
+    /**
+     * Sets bodyTemplate for the mail event.
+     *
+     * @param string $bodyTemplate
+     * @return self
+     */
+    public function setbodyTemplate(string $bodyTemplate): self
+    {
+        $this->bodyTemplate = $bodyTemplate;
+
+        return $this;
+    }
+
+    /**
+     * Returns subject for the mail event.
+     *
+     * @return string
+     */
+    public function getbodyTemplate(): string
+    {
+        return $this->bodyTemplate;
+    }
+
     /**
      * Set SMTP Host
      *
@@ -313,6 +338,22 @@ class Mail extends Event
         return $this;
     }
 
+    public function setAttachment(string $content, string $filename, string $encoding = 'base64', string $type = 'plain/text')
+    {
+        $this->attachment = [
+            'content' => base64_encode($content),
+            'filename' => $filename,
+            'encoding' => $encoding,
+            'type' => $type,
+        ];
+        return $this;
+    }
+
+    public function getAttachment(): array
+    {
+        return $this->attachment;
+    }
+
     /**
      * Executes the event and sends it to the mails worker.
      *
@@ -327,9 +368,11 @@ class Mail extends Event
             'recipient' => $this->recipient,
             'name' => $this->name,
             'subject' => $this->subject,
+            'bodyTemplate' => $this->bodyTemplate,
             'body' => $this->body,
             'smtp' => $this->smtp,
             'variables' => $this->variables,
+            'attachment' => $this->attachment,
             'events' => Event::generateEvents($this->getEvent(), $this->getParams())
         ]);
     }

src/Appwrite/Extend/Exception.php

@@ -84,6 +84,8 @@ class Exception extends \Exception
     public const USER_OAUTH2_BAD_REQUEST           = 'user_oauth2_bad_request';
     public const USER_OAUTH2_UNAUTHORIZED          = 'user_oauth2_unauthorized';
     public const USER_OAUTH2_PROVIDER_ERROR        = 'user_oauth2_provider_error';
+    public const USER_EMAIL_ALREADY_VERIFIED        = 'user_email_alread_verified';
+    public const USER_PHONE_ALREADY_VERIFIED        = 'user_phone_already_verified';
 
     /** Teams */
     public const TEAM_NOT_FOUND                    = 'team_not_found';
@@ -224,27 +226,28 @@ class Exception extends \Exception
     public const MIGRATION_NOT_FOUND                 = 'migration_not_found';
     public const MIGRATION_ALREADY_EXISTS            = 'migration_already_exists';
     public const MIGRATION_IN_PROGRESS               = 'migration_in_progress';
+    public const MIGRATION_PROVIDER_ERROR            = 'migration_provider_error';
 
     /** Realtime */
-    public const REALTIME_MESSAGE_FORMAT_INVALID = 'realtime_message_format_invalid';
-    public const REALTIME_TOO_MANY_MESSAGES = 'realtime_too_many_messages';
-    public const REALTIME_POLICY_VIOLATION = 'realtime_policy_violation';
+    public const REALTIME_MESSAGE_FORMAT_INVALID    = 'realtime_message_format_invalid';
+    public const REALTIME_TOO_MANY_MESSAGES         = 'realtime_too_many_messages';
+    public const REALTIME_POLICY_VIOLATION          = 'realtime_policy_violation';
+
+    /** Health */
+    public const QUEUE_SIZE_EXCEEDED                 = 'queue_size_exceeded';
 
     protected string $type = '';
     protected array $errors = [];
+    protected bool $publish;
 
     public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null)
     {
         $this->errors = Config::getParam('errors');
         $this->type = $type;
+        $this->code = $code ?? $this->errors[$type]['code'];
+        $this->message = $message ?? $this->errors[$type]['description'];
 
-        if (isset($this->errors[$type])) {
-            $this->code = $this->errors[$type]['code'];
-            $this->message = $this->errors[$type]['description'];
-        }
-
-        $this->message = $message ?? $this->message;
-        $this->code = $code ?? $this->code;
+        $this->publish = $this->errors[$type]['publish'] ?? ($this->code >= 500);
 
         parent::__construct($this->message, $this->code, $previous);
     }
@@ -270,4 +273,14 @@ class Exception extends \Exception
     {
         $this->type = $type;
     }
+
+    /**
+     * Check whether the log is publishable for the exception.
+     *
+     * @return bool
+     */
+    public function isPublishable(): bool
+    {
+        return $this->publish;
+    }
 }

src/Appwrite/Hooks/Hooks.php

@@ -0,0 +1,26 @@
+<?php
+
+namespace Appwrite\Hooks;
+
+class Hooks
+{
+    /**
+     * @var callable[] $hooks
+     */
+    private static array $hooks = [];
+
+    public static function add(string $name, callable $action)
+    {
+        self::$hooks[$name] = $action;
+    }
+
+    /**
+     * @param mixed[] $params
+     */
+    public function trigger(string $name, array $params = [])
+    {
+        if (isset(self::$hooks[$name])) {
+            call_user_func_array(self::$hooks[$name], $params);
+        }
+    }
+}

src/Appwrite/Migration/Migration.php

@@ -74,6 +74,11 @@ abstract class Migration
         '1.4.7' => 'V19',
         '1.4.8' => 'V19',
         '1.4.9' => 'V19',
+        '1.4.10' => 'V19',
+        '1.4.11' => 'V19',
+        '1.4.12' => 'V19',
+        '1.4.13' => 'V19',
+        '1.4.14' => 'V20'
     ];
 
     /**
@@ -191,17 +196,24 @@ abstract class Migration
      * @return iterable<Document>
      * @throws \Exception
      */
-    public function documentsIterator(string $collectionId): iterable
+    public function documentsIterator(string $collectionId, $queries = []): iterable
     {
         $sum = 0;
         $nextDocument = null;
         $collectionCount = $this->projectDB->count($collectionId);
+        $queries[] = Query::limit($this->limit);
 
         do {
-            $queries = [Query::limit($this->limit)];
             if ($nextDocument !== null) {
-                $queries[] = Query::cursorAfter($nextDocument);
+                $cursorQueryIndex = \array_search('cursorAfter', \array_map(fn (Query $query) => $query->getMethod(), $queries));
+
+                if ($cursorQueryIndex !== false) {
+                    $queries[$cursorQueryIndex] = Query::cursorAfter($nextDocument);
+                } else {
+                    $queries[] = Query::cursorAfter($nextDocument);
+                }
             }
+
             $documents = $this->projectDB->find($collectionId, $queries);
             $count = count($documents);
             $sum += $count;

src/Appwrite/Migration/Version/V19.php

@@ -10,6 +10,7 @@ use Utopia\Database\Database;
 use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Database\Exception;
+use Utopia\Database\Query;
 
 class V19 extends Migration
 {
@@ -41,6 +42,11 @@ class V19 extends Migration
         Console::info('Migrating Buckets');
         $this->migrateBuckets();
 
+        if ($this->project->getId() !== 'console') {
+            Console::info('Migrating Enum Attribute Size');
+            $this->migrateEnumAttributeSize();
+        }
+
         Console::info('Migrating Documents');
         $this->forEachDocument([$this, 'fixDocument']);
 
@@ -640,6 +646,22 @@ class V19 extends Migration
         return $commands;
     }
 
+    private function migrateEnumAttributeSize(): void
+    {
+        foreach (
+            $this->documentsIterator('attributes', [
+            Query::equal('format', ['enum']),
+            Query::lessThan('size', Database::LENGTH_KEY)
+            ]) as $attribute
+        ) {
+            $attribute->setAttribute('size', Database::LENGTH_KEY);
+            $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute);
+            $databaseInternalId = $attribute->getAttribute('databaseInternalId');
+            $collectionInternalId = $attribute->getAttribute('collectionInternalId');
+            $this->projectDB->updateAttribute('database_' . $databaseInternalId . '_collection_' . $collectionInternalId, $attribute->getAttribute('key'), size: 255);
+        }
+    }
+
     /**
      * Fix run on each document
      *

src/Appwrite/Migration/Version/V20.php

@@ -0,0 +1,323 @@
+<?php
+
+namespace Appwrite\Migration\Version;
+
+use Appwrite\Migration\Migration;
+use PDOException;
+use Throwable;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Database\Document;
+use Utopia\Database\Exception;
+use Utopia\Database\Exception\Authorization;
+use Utopia\Database\Exception\Duplicate;
+use Utopia\Database\Exception\Structure;
+use Utopia\Database\Query;
+
+class V20 extends Migration
+{
+    /**
+     * @throws Throwable
+     */
+    public function execute(): void
+    {
+        if ($this->project->getInternalId() == 'console') {
+            return;
+        }
+
+        /**
+         * Disable SubQueries for Performance.
+         */
+        foreach (['subQueryIndexes', 'subQueryPlatforms', 'subQueryDomains', 'subQueryKeys', 'subQueryWebhooks', 'subQuerySessions', 'subQueryTokens', 'subQueryMemberships', 'subQueryVariables'] as $name) {
+            Database::addFilter(
+                $name,
+                fn () => null,
+                fn () => []
+            );
+        }
+
+        $this->migrateUsageMetrics('project.$all.network.requests', 'network.requests');
+        $this->migrateUsageMetrics('project.$all.network.outbound', 'network.outbound');
+        $this->migrateUsageMetrics('project.$all.network.inbound', 'network.inbound');
+        $this->migrateUsageMetrics('users.$all.count.total', 'users');
+        $this->migrateSessionsMetric();
+
+        Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')');
+        $this->projectDB->setNamespace("_{$this->project->getInternalId()}");
+
+        Console::info('Migrating Functions');
+        $this->migrateFunctions();
+
+        Console::info('Migrating Databases');
+        $this->migrateDatabases();
+
+        Console::info('Migrating Collections');
+        $this->migrateCollections();
+
+        Console::info('Migrating Buckets');
+        $this->migrateBuckets();
+    }
+
+    /**
+     * @return void
+     * @throws Authorization
+     * @throws Exception
+     * @throws Structure
+     */
+    protected function migrateSessionsMetric(): void
+    {
+        /**
+         * Creating inf metric
+         */
+
+        Console::info('Migrating Sessions metric');
+
+        $sessionsCreated = $this->projectDB->sum('stats', 'value', [
+            Query::equal('metric', [
+                'sessions.email-password.requests.create',
+                'sessions.magic-url.requests.create',
+                'sessions.anonymous.requests.create',
+                'sessions.invites.requests.create',
+                'sessions.jwt.requests.create',
+                'sessions.phone.requests.create'
+            ]),
+            Query::equal('period', ['1d']),
+        ]);
+
+        $query = $this->projectDB->findOne('stats', [
+            Query::equal('metric', ['sessions.$all.requests.delete']),
+            Query::equal('period', ['1d']),
+        ]);
+
+        $sessionsDeleted =  $query['value'] ?? 0;
+        $value = $sessionsCreated - $sessionsDeleted;
+        $this->createInfMetric('sessions', $value);
+    }
+
+    /**
+     * @param string $metric
+     * @param int $value
+     * @return void
+     * @throws Exception
+     * @throws Authorization
+     * @throws Structure
+     */
+    protected function createInfMetric(string $metric, int $value): void
+    {
+
+        try {
+            /**
+             * Creating inf metric
+             */
+            console::log("Creating inf metric to {$metric}");
+            $id = \md5("_inf_{$metric}");
+            $this->projectDB->createDocument('stats', new Document([
+                '$id' => $id,
+                'metric' => $metric,
+                'period' => 'inf',
+                'value'  => $value,
+                'time'   => null,
+                'region' => 'default',
+            ]));
+        } catch (Duplicate $th) {
+            console::log("Error while creating inf metric: duplicate id {$metric}  {$id}");
+        }
+    }
+
+    /**
+     * @param string $from
+     * @param string $to
+     * @return void
+     * @throws Exception
+     */
+    protected function migrateUsageMetrics(string $from, string $to): void
+    {
+        /**
+         * inf metric
+         */
+        if (
+            str_contains($from, '$all') ||
+            str_contains($from, '.total')
+        ) {
+            $query = $this->projectDB->sum('stats', 'value', [
+                Query::equal('metric', [$from]),
+                Query::equal('period', ['1d']),
+            ]);
+
+            $value = $query ?? 0;
+            $this->createInfMetric($to, $value);
+        }
+
+        try {
+            /**
+             * Update old metric format to new
+             */
+            $limit = 1000;
+            $sum = $limit;
+            $total = 0;
+            $latestDocument = null;
+            while ($sum === $limit) {
+                $paginationQueries = [Query::limit($limit)];
+                if ($latestDocument !== null) {
+                    $paginationQueries[] =  Query::cursorAfter($latestDocument);
+                }
+                $stats = $this->projectDB->find('stats', \array_merge($paginationQueries, [
+                    Query::equal('metric', [$from]),
+                ]));
+
+                $sum = count($stats);
+                $total = $total + $sum;
+                foreach ($stats as $stat) {
+                    $format = $stat['period'] === '1d' ? 'Y-m-d 00:00' : 'Y-m-d H:00';
+                    $time = date($format, strtotime($stat['time']));
+                    $this->projectDB->deleteDocument('stats', $stat->getId());
+                    $stat->setAttribute('$id', \md5("{$time}_{$stat['period']}_{$to}"));
+                    $stat->setAttribute('metric', $to);
+                    $this->projectDB->createDocument('stats', $stat);
+                    console::log("deleting metric {$from} and creating {$to}");
+                }
+                $latestDocument = !empty(array_key_last($stats)) ? $stats[array_key_last($stats)] : null;
+            }
+        } catch (Throwable $th) {
+            Console::warning("Error while updating metric  {$from}  " . $th->getMessage());
+        }
+    }
+    /**
+     * Migrate functions.
+     *
+     * @return void
+     * @throws \Exception
+     */
+    private function migrateFunctions(): void
+    {
+
+        $this->migrateUsageMetrics('deployment.$all.storage.size', 'deployments.storage');
+        $this->migrateUsageMetrics('builds.$all.compute.total', 'builds');
+        $this->migrateUsageMetrics('builds.$all.compute.time', 'builds.compute');
+        $this->migrateUsageMetrics('executions.$all.compute.total', 'executions');
+        $this->migrateUsageMetrics('executions.$all.compute.time', 'executions.compute');
+
+        foreach ($this->documentsIterator('functions') as $function) {
+            Console::log("Migrating Functions usage stats of {$function->getId()} ({$function->getAttribute('name')})");
+
+            $functionId = $function->getId();
+            $functionInternalId = $function->getInternalId();
+
+            $this->migrateUsageMetrics("deployment.$functionId.storage.size", "function.$functionInternalId.deployments.storage");
+            $this->migrateUsageMetrics("builds.$functionId.compute.total", "$functionInternalId.builds");
+            $this->migrateUsageMetrics("builds.$functionId.compute.time", "$functionInternalId.builds.compute");
+            $this->migrateUsageMetrics("executions.$functionId.compute.total", "$functionInternalId.executions");
+            $this->migrateUsageMetrics("executions.$functionId.compute.time", "$functionInternalId.executions.compute");
+        }
+    }
+
+    /**
+     * Migrate  Databases.
+     *
+     * @return void
+     * @throws \Exception
+     */
+    private function migrateDatabases(): void
+    {
+        // Project level
+        $this->migrateUsageMetrics('databases.$all.count.total', 'databases');
+        $this->migrateUsageMetrics('collections.$all.count.total', 'collections');
+        $this->migrateUsageMetrics('documents.$all.count.total', 'documents');
+
+        foreach ($this->documentsIterator('databases') as $database) {
+            Console::log("Migrating Collections of {$database->getId()} ({$database->getAttribute('name')})");
+
+            $databaseTable = "database_{$database->getInternalId()}";
+
+            // Database level
+            $databaseId = $database->getId();
+            $databaseInternalId = $database->getInternalId();
+
+            $this->migrateUsageMetrics("collections.$databaseId.count.total", "$databaseInternalId.collections");
+            $this->migrateUsageMetrics("documents.$databaseId.count.total", "$databaseInternalId.documents");
+
+            foreach ($this->documentsIterator($databaseTable) as $collection) {
+                $collectionTable = "{$databaseTable}_collection_{$collection->getInternalId()}";
+                Console::log("Migrating Collections of {$collectionTable} {$collection->getId()} ({$collection->getAttribute('name')})");
+
+                // Collection level
+                $collectionId =  $collection->getId() ;
+                $collectionInternalId =  $collection->getInternalId();
+
+                $this->migrateUsageMetrics("documents.$databaseId/$collectionId.count.total", "$databaseInternalId.$collectionInternalId.documents");
+            }
+        }
+    }
+
+    /**
+     * Migrate Collections.
+     *
+     * @return void
+     * @throws \Exception
+     */
+    private function migrateCollections(): void
+    {
+        $internalProjectId = $this->project->getInternalId();
+        $collectionType = match ($internalProjectId) {
+            'console' => 'console',
+            default => 'projects',
+        };
+
+        $collections = $this->collections[$collectionType];
+
+        foreach ($collections as $collection) {
+            $id = $collection['$id'];
+
+            Console::log("Migrating Collection \"{$id}\"");
+
+            $this->projectDB->setNamespace("_$internalProjectId");
+
+            switch ($id) {
+                case 'stats':
+                    try {
+                        /**
+                         * Delete 'type' attribute
+                         */
+                        $this->projectDB->deleteAttribute($id, 'type');
+                        /**
+                         * Alter `signed`  internal type on `value` attr
+                         */
+                        $this->projectDB->updateAttribute($id, 'value', null, null, null, null, true);
+                        $this->projectDB->deleteCachedCollection($id);
+                    } catch (Throwable $th) {
+                        Console::warning("'type' from {$id}: {$th->getMessage()}");
+                    }
+                    break;
+            }
+        }
+    }
+
+    /**
+     * Migrating all Bucket tables.
+     *
+     * @return void
+     * @throws \Exception
+     * @throws PDOException
+     */
+    protected function migrateBuckets(): void
+    {
+        // Project level
+        $this->migrateUsageMetrics('buckets.$all.count.total', 'buckets');
+        $this->migrateUsageMetrics('files.$all.count.total', 'files');
+        $this->migrateUsageMetrics('files.$all.storage.size', 'files.storage');
+        // There is also project.$all.storage.size which is the same as  files.$all.storage.size
+
+        foreach ($this->documentsIterator('buckets') as $bucket) {
+            $id = "bucket_{$bucket->getInternalId()}";
+            Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})");
+
+            // Bucket level
+            $bucketId = $bucket->getId();
+            $bucketInternalId = $bucket->getInternalId();
+
+             $this->migrateUsageMetrics("files.$bucketId.count.total", "$bucketInternalId.files");
+            $this->migrateUsageMetrics("files.$bucketId.storage.size", "$bucketInternalId.files.storage");
+            // some stats come with $ prefix in front of the id -> files.$650c3fda307b7fec4934.storage.size;
+        }
+    }
+}

src/Appwrite/Network/Validator/CNAME.php

@@ -6,6 +6,11 @@ use Utopia\Validator;
 
 class CNAME extends Validator
 {
+    /**
+     * @var mixed
+     */
+    protected mixed $logs;
+
     /**
      * @var string
      */
@@ -27,6 +32,14 @@ class CNAME extends Validator
         return 'Invalid CNAME record';
     }
 
+    /**
+     * @return mixed
+     */
+    public function getLogs(): mixed
+    {
+        return $this->logs;
+    }
+
     /**
      * Check if CNAME record target value matches selected target
      *
@@ -42,6 +55,7 @@ class CNAME extends Validator
 
         try {
             $records = \dns_get_record($domain, DNS_CNAME);
+            $this->logs = $records;
         } catch (\Throwable $th) {
             return false;
         }

src/Appwrite/Platform/Services/Tasks.php

@@ -11,14 +11,9 @@ use Appwrite\Platform\Tasks\Schedule;
 use Appwrite\Platform\Tasks\SDKs;
 use Appwrite\Platform\Tasks\Specs;
 use Appwrite\Platform\Tasks\SSL;
-use Appwrite\Platform\Tasks\Hamster;
-use Appwrite\Platform\Tasks\Usage;
 use Appwrite\Platform\Tasks\Vars;
 use Appwrite\Platform\Tasks\Version;
-use Appwrite\Platform\Tasks\VolumeSync;
-use Appwrite\Platform\Tasks\CalcTierStats;
 use Appwrite\Platform\Tasks\Upgrade;
-use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
 
 class Tasks extends Service
 {
@@ -27,10 +22,8 @@ class Tasks extends Service
         $this->type = self::TYPE_CLI;
         $this
             ->addAction(Version::getName(), new Version())
-            ->addAction(Usage::getName(), new Usage())
             ->addAction(Vars::getName(), new Vars())
             ->addAction(SSL::getName(), new SSL())
-            ->addAction(Hamster::getName(), new Hamster())
             ->addAction(Doctor::getName(), new Doctor())
             ->addAction(Install::getName(), new Install())
             ->addAction(Upgrade::getName(), new Upgrade())
@@ -38,10 +31,7 @@ class Tasks extends Service
             ->addAction(Schedule::getName(), new Schedule())
             ->addAction(Migrate::getName(), new Migrate())
             ->addAction(SDKs::getName(), new SDKs())
-            ->addAction(VolumeSync::getName(), new VolumeSync())
             ->addAction(Specs::getName(), new Specs())
-            ->addAction(CalcTierStats::getName(), new CalcTierStats())
-            ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects())
 
         ;
     }

src/Appwrite/Platform/Services/Workers.php

@@ -12,6 +12,8 @@ use Appwrite\Platform\Workers\Databases;
 use Appwrite\Platform\Workers\Functions;
 use Appwrite\Platform\Workers\Builds;
 use Appwrite\Platform\Workers\Deletes;
+use Appwrite\Platform\Workers\Usage;
+use Appwrite\Platform\Workers\UsageHook;
 use Appwrite\Platform\Workers\Migrations;
 
 class Workers extends Service
@@ -29,6 +31,9 @@ class Workers extends Service
             ->addAction(Functions::getName(), new Functions())
             ->addAction(Builds::getName(), new Builds())
             ->addAction(Deletes::getName(), new Deletes())
+            ->addAction(Hamster::getName(), new Hamster())
+            ->addAction(UsageHook::getName(), new UsageHook())
+            ->addAction(Usage::getName(), new Usage())
             ->addAction(Migrations::getName(), new Migrations())
 
         ;

src/Appwrite/Platform/Tasks/CalcTierStats.php

@@ -1,351 +0,0 @@
-<?php
-
-namespace Appwrite\Platform\Tasks;
-
-use Exception;
-use League\Csv\CannotInsertRecord;
-use Utopia\App;
-use Utopia\Database\Validator\Authorization;
-use Utopia\Platform\Action;
-use Utopia\Cache\Cache;
-use Utopia\CLI\Console;
-use Utopia\Database\Database;
-use Utopia\Database\Query;
-use League\Csv\Writer;
-use PHPMailer\PHPMailer\PHPMailer;
-use Utopia\Pools\Group;
-use Utopia\Registry\Registry;
-
-class CalcTierStats extends Action
-{
-    /*
-     * Csv cols headers
-     */
-    private array $columns = [
-        'Project ID',
-        'Organization ID',
-        'Organization Members',
-        'Teams',
-        'Users',
-        'Requests',
-        'Bandwidth',
-        'Domains',
-        'Api keys',
-        'Webhooks',
-        'Platforms',
-        'Buckets',
-        'Files',
-        'Storage (bytes)',
-        'Max File Size (bytes)',
-        'Databases',
-        'Functions',
-        'Deployments',
-        'Executions',
-    ];
-
-    protected string $directory = '/usr/local';
-    protected string $path;
-    protected string $date;
-
-    private array $usageStats = [
-        'project.$all.network.requests'  => 'Requests',
-        'project.$all.network.bandwidth' => 'Bandwidth',
-
-    ];
-
-    public static function getName(): string
-    {
-        return 'calc-tier-stats';
-    }
-
-    public function __construct()
-    {
-
-        $this
-            ->desc('Get stats for projects')
-            ->inject('pools')
-            ->inject('cache')
-            ->inject('dbForConsole')
-            ->inject('register')
-            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
-                $this->action($pools, $cache, $dbForConsole, $register);
-            });
-    }
-
-    /**
-     * @throws \Utopia\Exception
-     * @throws CannotInsertRecord
-     */
-    public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
-    {
-        //docker compose exec -t appwrite calc-tier-stats
-
-        Console::title('Cloud free tier  stats calculation V1');
-        Console::success(APP_NAME . ' cloud free tier  stats calculation has started');
-
-        /* Initialise new Utopia app */
-        $app = new App('UTC');
-        $console = $app->getResource('console');
-
-        /** CSV stuff */
-        $this->date = date('Y-m-d');
-        $this->path = "{$this->directory}/tier_stats_{$this->date}.csv";
-        $csv = Writer::createFromPath($this->path, 'w');
-        $csv->insertOne($this->columns);
-
-        /** Database connections */
-        $totalProjects = $dbForConsole->count('projects');
-        Console::success("Found a total of: {$totalProjects} projects");
-
-        $projects = [$console];
-        $count = 0;
-        $limit = 30;
-        $sum = 30;
-        $offset = 0;
-        while (!empty($projects)) {
-            foreach ($projects as $project) {
-
-                /**
-                 * Skip user projects with id 'console'
-                 */
-                if ($project->getId() === 'console') {
-                    continue;
-                }
-
-                Console::info("Getting stats for {$project->getId()}");
-
-                try {
-                    $db = $project->getAttribute('database');
-                    $adapter = $pools
-                        ->get($db)
-                        ->pop()
-                        ->getResource();
-
-                    $dbForProject = new Database($adapter, $cache);
-                    $dbForProject->setDefaultDatabase('appwrite');
-                    $dbForProject->setNamespace('_' . $project->getInternalId());
-
-                    /** Get Project ID */
-                    $stats['Project ID'] = $project->getId();
-
-                    $stats['Organization ID']   = $project->getAttribute('teamId', null);
-
-                    /** Get Total Members */
-                    $teamInternalId = $project->getAttribute('teamInternalId', null);
-                    if ($teamInternalId) {
-                        $stats['Organization Members'] = $dbForConsole->count('memberships', [
-                            Query::equal('teamInternalId', [$teamInternalId])
-                        ]);
-                    } else {
-                        $stats['Organization Members'] = 0;
-                    }
-
-                    /** Get Total internal Teams */
-                    try {
-                        $stats['Teams'] = $dbForProject->count('teams', []);
-                    } catch (\Throwable) {
-                        $stats['Teams'] = 0;
-                    }
-
-                    /** Get Total users */
-                    try {
-                        $stats['Users'] = $dbForProject->count('users', []);
-                    } catch (\Throwable) {
-                        $stats['Users'] = 0;
-                    }
-
-                    /** Get Usage stats */
-                    $range = '30d';
-                    $periods = [
-                        '30d' => [
-                            'period' => '1d',
-                            'limit' => 30,
-                        ]
-                    ];
-
-                    $tmp = [];
-                    $metrics = $this->usageStats;
-                    Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) {
-                        foreach ($metrics as $metric => $name) {
-                            $limit = $periods[$range]['limit'];
-                            $period = $periods[$range]['period'];
-
-                            $requestDocs = $dbForProject->find('stats', [
-                                Query::equal('period', [$period]),
-                                Query::equal('metric', [$metric]),
-                                Query::limit($limit),
-                                Query::orderDesc('time'),
-                            ]);
-
-                            $tmp[$metric] = [];
-                            foreach ($requestDocs as $requestDoc) {
-                                if (empty($requestDoc)) {
-                                    continue;
-                                }
-
-                                $tmp[$metric][] = [
-                                    'value' => $requestDoc->getAttribute('value'),
-                                    'date' => $requestDoc->getAttribute('time'),
-                                ];
-                            }
-
-                            $tmp[$metric] = array_reverse($tmp[$metric]);
-                            $tmp[$metric] = array_sum(array_column($tmp[$metric], 'value'));
-                        }
-                    });
-
-                    foreach ($tmp as $key => $value) {
-                        $stats[$metrics[$key]]  = $value;
-                    }
-
-                    try {
-                        /** Get Domains */
-                        $stats['Domains'] = $dbForConsole->count('domains', [
-                            Query::equal('projectInternalId', [$project->getInternalId()]),
-                        ]);
-                    } catch (\Throwable) {
-                        $stats['Domains'] = 0;
-                    }
-
-                    try {
-                    /** Get Api keys */
-                        $stats['Api keys'] = $dbForConsole->count('keys', [
-                        Query::equal('projectInternalId', [$project->getInternalId()]),
-                        ]);
-                    } catch (\Throwable) {
-                        $stats['Api keys'] = 0;
-                    }
-
-                    try {
-                    /** Get Webhooks */
-                        $stats['Webhooks'] = $dbForConsole->count('webhooks', [
-                        Query::equal('projectInternalId', [$project->getInternalId()]),
-                        ]);
-                    } catch (\Throwable) {
-                        $stats['Webhooks'] = 0;
-                    }
-
-                    try {
-                    /** Get Platforms */
-                        $stats['Platforms'] = $dbForConsole->count('platforms', [
-                        Query::equal('projectInternalId', [$project->getInternalId()]),
-                        ]);
-                    } catch (\Throwable) {
-                        $stats['Platforms'] = 0;
-                    }
-
-                    /** Get Files & Buckets */
-                    $filesCount = 0;
-                    $filesSum = 0;
-                    $maxFileSize = 0;
-                    $counter = 0;
-                    try {
-                        $buckets = $dbForProject->find('buckets', []);
-                        foreach ($buckets as $bucket) {
-                            $file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]);
-                            if (empty($file)) {
-                                continue;
-                            }
-                            $filesSum   += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', []);
-                            $filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []);
-                            if ($file->getAttribute('sizeOriginal') > $maxFileSize) {
-                                $maxFileSize = $file->getAttribute('sizeOriginal');
-                            }
-                            $counter++;
-                        }
-                    } catch (\Throwable) {
-                        ;
-                    }
-                    $stats['Buckets'] = $counter;
-                    $stats['Files'] = $filesCount;
-                    $stats['Storage (bytes)'] = $filesSum;
-                    $stats['Max File Size (bytes)'] = $maxFileSize;
-
-
-                    try {
-                    /** Get Total Functions */
-                        $stats['Databases'] = $dbForProject->count('databases', []);
-                    } catch (\Throwable) {
-                        $stats['Databases'] = 0;
-                    }
-
-                    /** Get Total Functions */
-                    try {
-                        $stats['Functions'] = $dbForProject->count('functions', []);
-                    } catch (\Throwable) {
-                        $stats['Functions'] = 0;
-                    }
-
-                    /** Get Total Deployments */
-                    try {
-                        $stats['Deployments'] = $dbForProject->count('deployments', []);
-                    } catch (\Throwable) {
-                        $stats['Deployments'] = 0;
-                    }
-
-                    /** Get Total Executions */
-                    try {
-                        $stats['Executions'] = $dbForProject->count('executions', []);
-                    } catch (\Throwable) {
-                        $stats['Executions'] = 0;
-                    }
-
-                    $csv->insertOne(array_values($stats));
-                } catch (\Throwable $th) {
-                    Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . '  line no: ' . $th->getline() . ' with message: ' . $th->getMessage());
-                } finally {
-                    $pools
-                        ->get($db)
-                        ->reclaim();
-                }
-            }
-
-            $sum = \count($projects);
-
-            $projects = $dbForConsole->find('projects', [
-                Query::limit($limit),
-                Query::offset($offset),
-            ]);
-
-            $offset = $offset + $limit;
-            $count = $count + $sum;
-        }
-
-        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
-
-        $pools
-            ->get('console')
-            ->reclaim();
-
-        /** @var PHPMailer $mail */
-        $mail = $register->get('smtp');
-
-        $mail->clearAddresses();
-        $mail->clearAllRecipients();
-        $mail->clearReplyTos();
-        $mail->clearAttachments();
-        $mail->clearBCCs();
-        $mail->clearCCs();
-
-        try {
-            /** Addresses */
-            $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
-            $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
-
-            foreach ($recipients as $recipient) {
-                $mail->addAddress($recipient);
-            }
-
-            /** Attachments */
-            $mail->addAttachment($this->path);
-
-            /** Content */
-            $mail->Subject = "Cloud Report for {$this->date}";
-            $mail->Body = "Please find the daily cloud report atttached";
-            $mail->send();
-            Console::success('Email has been sent!');
-        } catch (Exception $e) {
-            Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
-        }
-    }
-}

src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php

@@ -1,115 +0,0 @@
-<?php
-
-namespace Appwrite\Platform\Tasks;
-
-use PHPMailer\PHPMailer\PHPMailer;
-use Utopia\App;
-use Utopia\Config\Config;
-use Utopia\Database\Query;
-use Utopia\Database\Validator\Authorization;
-use Utopia\Platform\Action;
-use Utopia\Cache\Cache;
-use Utopia\CLI\Console;
-use Utopia\Database\Database;
-use Utopia\Pools\Group;
-use Utopia\Registry\Registry;
-
-class DeleteOrphanedProjects extends Action
-{
-    public static function getName(): string
-    {
-        return 'delete-orphaned-projects';
-    }
-
-    public function __construct()
-    {
-
-        $this
-            ->desc('Get stats for projects')
-            ->inject('pools')
-            ->inject('cache')
-            ->inject('dbForConsole')
-            ->inject('register')
-            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
-                $this->action($pools, $cache, $dbForConsole, $register);
-            });
-    }
-
-
-    public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
-    {
-
-        Console::title('Delete orphaned projects V1');
-        Console::success(APP_NAME . ' Delete orphaned projects started');
-
-        /** @var array $collections */
-        $collectionsConfig = Config::getParam('collections', [])['projects'] ?? [];
-
-        /* Initialise new Utopia app */
-        $app = new App('UTC');
-        $console = $app->getResource('console');
-        $projects = [$console];
-
-        /** Database connections */
-        $totalProjects = $dbForConsole->count('projects');
-        Console::success("Found a total of: {$totalProjects} projects");
-
-        $orphans = 0;
-        $count = 0;
-        $limit = 30;
-        $sum = 30;
-        $offset = 0;
-        while (!empty($projects)) {
-            foreach ($projects as $project) {
-
-                /**
-                 * Skip user projects with id 'console'
-                 */
-                if ($project->getId() === 'console') {
-                    continue;
-                }
-
-                try {
-                    $db = $project->getAttribute('database');
-                    $adapter = $pools
-                        ->get($db)
-                        ->pop()
-                        ->getResource();
-
-                    $dbForProject = new Database($adapter, $cache);
-                    $dbForProject->setDefaultDatabase('appwrite');
-                    $dbForProject->setNamespace('_' . $project->getInternalId());
-                    $collectionsCreated = $dbForProject->count(Database::METADATA);
-                    $message = ' (' . $collectionsCreated . ') collections where found on project (' . $project->getId() . '))';
-                    if ($collectionsCreated < (count($collectionsConfig) + 2)) {
-                        Console::error($message);
-                        $orphans++;
-                    } else {
-                        Console::log($message);
-                    }
-                } catch (\Throwable $th) {
-                    //$dbForConsole->deleteDocument('projects', $project->getId());
-                    //Console::success('Deleting  project (' . $project->getId() . ')');
-                    Console::error(' (0) collections where found for project (' . $project->getId() . ')');
-                    $orphans++;
-                } finally {
-                    $pools
-                        ->get($db)
-                        ->reclaim();
-                }
-            }
-
-            $sum = \count($projects);
-
-            $projects = $dbForConsole->find('projects', [
-                Query::limit($limit),
-                Query::offset($offset),
-            ]);
-
-            $offset = $offset + $limit;
-            $count = $count + $sum;
-        }
-
-        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans . ' orphans');
-    }
-}

src/Appwrite/Platform/Tasks/Hamster.php

@@ -1,476 +0,0 @@
-<?php
-
-namespace Appwrite\Platform\Tasks;
-
-use Appwrite\Network\Validator\Origin;
-use Exception;
-use Utopia\App;
-use Utopia\Platform\Action;
-use Utopia\Cache\Cache;
-use Utopia\CLI\Console;
-use Utopia\Database\Database;
-use Utopia\Database\Query;
-use Utopia\Database\Validator\Authorization;
-use Utopia\Analytics\Adapter\Mixpanel;
-use Utopia\Analytics\Event;
-use Utopia\Config\Config;
-use Utopia\Database\Document;
-use Utopia\Pools\Group;
-
-class Hamster extends Action
-{
-    private array $metrics = [
-        'usage_files' => 'files.$all.count.total',
-        'usage_buckets' => 'buckets.$all.count.total',
-        'usage_databases' => 'databases.$all.count.total',
-        'usage_documents' => 'documents.$all.count.total',
-        'usage_collections' => 'collections.$all.count.total',
-        'usage_storage' => 'project.$all.storage.size',
-        'usage_requests' => 'project.$all.network.requests',
-        'usage_bandwidth' => 'project.$all.network.bandwidth',
-        'usage_users' => 'users.$all.count.total',
-        'usage_sessions' => 'sessions.email.requests.create',
-        'usage_executions' => 'executions.$all.compute.total',
-    ];
-
-    protected string $directory = '/usr/local';
-
-    protected string $path;
-
-    protected string $date;
-
-    protected Mixpanel $mixpanel;
-
-    public static function getName(): string
-    {
-        return 'hamster';
-    }
-
-    public function __construct()
-    {
-        $this->mixpanel = new Mixpanel(App::getEnv('_APP_MIXPANEL_TOKEN', ''));
-
-        $this
-            ->desc('Get stats for projects')
-            ->inject('pools')
-            ->inject('cache')
-            ->inject('dbForConsole')
-            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole) {
-                $this->action($pools, $cache, $dbForConsole);
-            });
-    }
-
-    private function getStatsPerProject(Group $pools, Cache $cache, Database $dbForConsole)
-    {
-        $this->calculateByGroup('projects', $dbForConsole, function (Database $dbForConsole, Document $project) use ($pools, $cache) {
-            /**
-             * Skip user projects with id 'console'
-             */
-            if ($project->getId() === 'console') {
-                Console::info("Skipping project console");
-                return;
-            }
-
-            Console::log("Getting stats for {$project->getId()}");
-
-            try {
-                $db = $project->getAttribute('database');
-                $adapter = $pools
-                    ->get($db)
-                    ->pop()
-                    ->getResource();
-
-                $dbForProject = new Database($adapter, $cache);
-                $dbForProject->setDefaultDatabase('appwrite');
-                $dbForProject->setNamespace('_' . $project->getInternalId());
-
-                $statsPerProject = [];
-
-                $statsPerProject['time'] = microtime(true);
-
-                /** Get Project ID */
-                $statsPerProject['project_id'] = $project->getId();
-
-                /** Get project created time */
-                $statsPerProject['project_created'] = $project->getAttribute('$createdAt');
-
-                /** Get Project Name */
-                $statsPerProject['project_name'] = $project->getAttribute('name');
-
-                /** Total Project Variables */
-                $statsPerProject['custom_variables'] = $dbForProject->count('variables', [], APP_LIMIT_COUNT);
-
-                /** Total Migrations */
-                $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT);
-
-                /** Get Custom SMTP */
-                $smtp = $project->getAttribute('smtp', null);
-                if ($smtp) {
-                    $statsPerProject['custom_smtp_status'] = $smtp['enabled'] === true ? 'enabled' : 'disabled';
-
-                    /** Get Custom Templates Count */
-                    $templates = array_keys($project->getAttribute('templates', []));
-                    $statsPerProject['custom_email_templates'] = array_filter($templates, function ($template) {
-                        return str_contains($template, 'email');
-                    });
-                    $statsPerProject['custom_sms_templates'] = array_filter($templates, function ($template) {
-                        return str_contains($template, 'sms');
-                    });
-                }
-
-                /** Get total relationship attributes */
-                $statsPerProject['custom_relationship_attributes'] = $dbForProject->count('attributes', [
-                    Query::equal('type', ['relationship'])
-                ], APP_LIMIT_COUNT);
-
-                /** Get Total Functions */
-                $statsPerProject['custom_functions'] = $dbForProject->count('functions', [], APP_LIMIT_COUNT);
-
-                foreach (\array_keys(Config::getParam('runtimes')) as $runtime) {
-                    $statsPerProject['custom_functions_' . $runtime] = $dbForProject->count('functions', [
-                        Query::equal('runtime', [$runtime]),
-                    ], APP_LIMIT_COUNT);
-                }
-
-                /** Get Total Deployments */
-                $statsPerProject['custom_deployments'] = $dbForProject->count('deployments', [], APP_LIMIT_COUNT);
-                $statsPerProject['custom_deployments_manual'] = $dbForProject->count('deployments', [
-                    Query::equal('type', ['manual'])
-                ], APP_LIMIT_COUNT);
-                $statsPerProject['custom_deployments_git'] = $dbForProject->count('deployments', [
-                    Query::equal('type', ['vcs'])
-                ], APP_LIMIT_COUNT);
-
-                /** Get VCS repos connected */
-                $statsPerProject['custom_vcs_repositories'] = $dbForConsole->count('repositories', [
-                    Query::equal('projectInternalId', [$project->getInternalId()])
-                ], APP_LIMIT_COUNT);
-
-                /** Get Total Teams */
-                $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT);
-
-                /** Get Total Members */
-                $teamInternalId = $project->getAttribute('teamInternalId', null);
-                if ($teamInternalId) {
-                    $statsPerProject['custom_organization_members'] = $dbForConsole->count('memberships', [
-                        Query::equal('teamInternalId', [$teamInternalId])
-                    ], APP_LIMIT_COUNT);
-                } else {
-                    $statsPerProject['custom_organization_members'] = 0;
-                }
-
-                /** Get Email and Name of the project owner */
-                if ($teamInternalId) {
-                    $membership = $dbForConsole->findOne('memberships', [
-                        Query::equal('teamInternalId', [$teamInternalId]),
-                    ]);
-
-                    if (!$membership || $membership->isEmpty()) {
-                        throw new Exception('Membership not found. Skipping project : ' . $project->getId());
-                    }
-
-                    $userId = $membership->getAttribute('userId', null);
-                    if ($userId) {
-                        $user = $dbForConsole->getDocument('users', $userId);
-                        $statsPerProject['email'] = $user->getAttribute('email', null);
-                        $statsPerProject['name'] = $user->getAttribute('name', null);
-                    }
-                }
-
-                /** Get Domains */
-                $statsPerProject['custom_domains'] = $dbForConsole->count('rules', [
-                    Query::equal('projectInternalId', [$project->getInternalId()]),
-                    Query::limit(APP_LIMIT_COUNT)
-                ]);
-
-                /** Get Platforms */
-                $platforms = $dbForConsole->find('platforms', [
-                    Query::equal('projectInternalId', [$project->getInternalId()]),
-                    Query::limit(APP_LIMIT_COUNT)
-                ]);
-
-                $statsPerProject['custom_platforms_web'] = sizeof(array_filter($platforms, function ($platform) {
-                    return $platform['type'] === 'web';
-                }));
-
-                $statsPerProject['custom_platforms_android'] = sizeof(array_filter($platforms, function ($platform) {
-                    return $platform['type'] === 'android';
-                }));
-
-                $statsPerProject['custom_platforms_apple'] = sizeof(array_filter($platforms, function ($platform) {
-                    return str_contains($platform['type'], 'apple');
-                }));
-
-                $statsPerProject['custom_platforms_flutter'] = sizeof(array_filter($platforms, function ($platform) {
-                    return str_contains($platform['type'], 'flutter');
-                }));
-
-                $flutterPlatforms = [Origin::CLIENT_TYPE_FLUTTER_ANDROID, Origin::CLIENT_TYPE_FLUTTER_IOS, Origin::CLIENT_TYPE_FLUTTER_MACOS, Origin::CLIENT_TYPE_FLUTTER_WINDOWS, Origin::CLIENT_TYPE_FLUTTER_LINUX];
-
-                foreach ($flutterPlatforms as $flutterPlatform) {
-                    $statsPerProject['custom_platforms_' . $flutterPlatform] = sizeof(array_filter($platforms, function ($platform) use ($flutterPlatform) {
-                        return $platform['type'] === $flutterPlatform;
-                    }));
-                }
-
-                $statsPerProject['custom_platforms_api_keys'] = $dbForConsole->count('keys', [
-                    Query::equal('projectInternalId', [$project->getInternalId()]),
-                    Query::limit(APP_LIMIT_COUNT)
-                ]);
-
-                /** Get Usage $statsPerProject */
-                $periods = [
-                    'infinity' => [
-                        'period' => '1d',
-                        'limit' => 90,
-                    ],
-                    '24h' => [
-                        'period' => '1h',
-                        'limit' => 24,
-                    ],
-                ];
-
-                Authorization::skip(function () use ($dbForProject, $periods, &$statsPerProject) {
-                    foreach ($this->metrics as $key => $metric) {
-                        foreach ($periods as $periodKey => $periodValue) {
-                            $limit = $periodValue['limit'];
-                            $period = $periodValue['period'];
-
-                            $requestDocs = $dbForProject->find('stats', [
-                                Query::equal('period', [$period]),
-                                Query::equal('metric', [$metric]),
-                                Query::limit($limit),
-                                Query::orderDesc('time'),
-                            ]);
-
-                            $statsPerProject[$key . '_' . $periodKey] = [];
-                            foreach ($requestDocs as $requestDoc) {
-                                $statsPerProject[$key . '_' . $periodKey][] = [
-                                    'value' => $requestDoc->getAttribute('value'),
-                                    'date' => $requestDoc->getAttribute('time'),
-                                ];
-                            }
-
-                            $statsPerProject[$key . '_' . $periodKey] = array_reverse($statsPerProject[$key . '_' . $periodKey]);
-                            // Calculate aggregate of each metric
-                            $statsPerProject[$key . '_' . $periodKey] = array_sum(array_column($statsPerProject[$key . '_' . $periodKey], 'value'));
-                        }
-                    }
-                });
-
-                if (isset($statsPerProject['email'])) {
-                    /** Send data to mixpanel */
-                    $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [
-                        'name' => $statsPerProject['name'],
-                        'email' => $statsPerProject['email']
-                    ]);
-
-                    if (!$res) {
-                        Console::error('Failed to create user profile for project: ' . $project->getId());
-                    }
-                }
-
-                $event = new Event();
-                $event
-                    ->setName('Project Daily Usage')
-                    ->setProps($statsPerProject);
-                $res = $this->mixpanel->createEvent($event);
-
-                if (!$res) {
-                    Console::error('Failed to create event for project: ' . $project->getId());
-                }
-            } catch (Exception $e) {
-                Console::error('Failed to send stats for project: ' . $project->getId());
-                Console::error($e->getMessage());
-            } finally {
-                $pools
-                    ->get($db)
-                    ->reclaim();
-            }
-        });
-    }
-
-    public function action(Group $pools, Cache $cache, Database $dbForConsole): void
-    {
-
-        Console::title('Cloud Hamster V1');
-        Console::success(APP_NAME . ' cloud hamster process has started');
-
-        $sleep = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default)
-
-        $jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes)
-        $now = new \DateTime();
-        $now->setTimezone(new \DateTimeZone(date_default_timezone_get()));
-        $next = new \DateTime($now->format("Y-m-d $jobInitTime"));
-        $next->setTimezone(new \DateTimeZone(date_default_timezone_get()));
-        $delay = $next->getTimestamp() - $now->getTimestamp();
-
-        /**
-         * If time passed for the target day.
-         */
-        if ($delay <= 0) {
-            $next->add(\DateInterval::createFromDateString('1 days'));
-            $delay = $next->getTimestamp() - $now->getTimestamp();
-        }
-
-        Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']');
-
-        Console::loop(function () use ($pools, $cache, $dbForConsole, $sleep) {
-            $now = date('d-m-Y H:i:s', time());
-            Console::info("[{$now}] Getting Cloud Usage Stats every {$sleep} seconds");
-            $loopStart = microtime(true);
-
-            /* Initialise new Utopia app */
-            $app = new App('UTC');
-
-            Console::info('Getting stats for all projects');
-            $this->getStatsPerProject($pools, $cache, $dbForConsole);
-            Console::success('Completed getting stats for all projects');
-
-            Console::info('Getting stats for all organizations');
-            $this->getStatsPerOrganization($dbForConsole);
-            Console::success('Completed getting stats for all organizations');
-
-            Console::info('Getting stats for all users');
-            $this->getStatsPerUser($dbForConsole);
-            Console::success('Completed getting stats for all users');
-
-            $pools
-                ->get('console')
-                ->reclaim();
-
-            $loopTook = microtime(true) - $loopStart;
-            $now = date('d-m-Y H:i:s', time());
-            Console::info("[{$now}] Cloud Stats took {$loopTook} seconds");
-        }, $sleep, $delay);
-    }
-
-    protected function calculateByGroup(string $collection, Database $dbForConsole, callable $callback)
-    {
-        $count = 0;
-        $chunk = 0;
-        $limit = 50;
-        $results = [];
-        $sum = $limit;
-
-        $executionStart = \microtime(true);
-
-        while ($sum === $limit) {
-            $chunk++;
-
-            $results = $dbForConsole->find($collection, \array_merge([
-                Query::limit($limit),
-                Query::offset($count)
-            ]));
-
-            $sum = count($results);
-
-            Console::log('Processing chunk #' . $chunk . '. Found ' . $sum . ' documents');
-
-            foreach ($results as $document) {
-                call_user_func($callback, $dbForConsole, $document);
-                $count++;
-            }
-        }
-
-        $executionEnd = \microtime(true);
-
-        Console::log("Processed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
-    }
-
-    protected function getStatsPerOrganization(Database $dbForConsole)
-    {
-
-        $this->calculateByGroup('teams', $dbForConsole, function (Database $dbForConsole, Document $document) {
-            try {
-                $statsPerOrganization = [];
-
-                /** Organization name */
-                $statsPerOrganization['name'] = $document->getAttribute('name');
-
-                /** Get Email and of the organization owner */
-                $membership = $dbForConsole->findOne('memberships', [
-                    Query::equal('teamInternalId', [$document->getInternalId()]),
-                ]);
-
-                if (!$membership || $membership->isEmpty()) {
-                    throw new Exception('Membership not found. Skipping organization : ' . $document->getId());
-                }
-
-                $userId = $membership->getAttribute('userId', null);
-                if ($userId) {
-                    $user = $dbForConsole->getDocument('users', $userId);
-                    $statsPerOrganization['email'] = $user->getAttribute('email', null);
-                }
-
-                /** Organization Creation Date */
-                $statsPerOrganization['created'] = $document->getAttribute('$createdAt');
-
-                /** Number of team members */
-                $statsPerOrganization['members'] = $document->getAttribute('total');
-
-                /** Number of projects in this organization */
-                $statsPerOrganization['projects'] = $dbForConsole->count('projects', [
-                    Query::equal('teamId', [$document->getId()]),
-                    Query::limit(APP_LIMIT_COUNT)
-                ]);
-
-                if (!isset($statsPerOrganization['email'])) {
-                    throw new Exception('Email not found. Skipping organization : ' . $document->getId());
-                }
-
-                $event = new Event();
-                $event
-                    ->setName('Organization Daily Usage')
-                    ->setProps($statsPerOrganization);
-                $res = $this->mixpanel->createEvent($event);
-                if (!$res) {
-                    throw new Exception('Failed to create event for organization : ' . $document->getId());
-                }
-            } catch (Exception $e) {
-                Console::error($e->getMessage());
-            }
-        });
-    }
-
-    protected function getStatsPerUser(Database $dbForConsole)
-    {
-        $this->calculateByGroup('users', $dbForConsole, function (Database $dbForConsole, Document $document) {
-            try {
-                $statsPerUser = [];
-
-                /** Organization name */
-                $statsPerUser['name'] = $document->getAttribute('name');
-
-                /** Organization ID (needs to be stored as an email since mixpanel uses the email attribute as a distinctID) */
-                $statsPerUser['email'] = $document->getAttribute('email');
-
-                /** Organization Creation Date */
-                $statsPerUser['created'] = $document->getAttribute('$createdAt');
-
-                /** Number of teams this user is a part of */
-                $statsPerUser['memberships'] = $dbForConsole->count('memberships', [
-                    Query::equal('userInternalId', [$document->getInternalId()]),
-                    Query::limit(APP_LIMIT_COUNT)
-                ]);
-
-                if (!isset($statsPerUser['email'])) {
-                    throw new Exception('User has no email: ' . $document->getId());
-                }
-
-                /** Send data to mixpanel */
-                $event = new Event();
-                $event
-                    ->setName('User Daily Usage')
-                    ->setProps($statsPerUser);
-                $res = $this->mixpanel->createEvent($event);
-
-                if (!$res) {
-                    throw new Exception('Failed to create user profile for user: ' . $document->getId());
-                }
-            } catch (Exception $e) {
-                Console::error($e->getMessage());
-            }
-        });
-    }
-}

src/Appwrite/Platform/Tasks/Install.php

@@ -8,6 +8,7 @@ use Appwrite\Docker\Env;
 use Appwrite\Utopia\View;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
+use Utopia\Validator\Boolean;
 use Utopia\Validator\Text;
 use Utopia\Platform\Action;
 
@@ -24,15 +25,16 @@ class Install extends Action
     {
         $this
             ->desc('Install Appwrite')
-            ->param('httpPort', '', new Text(4), 'Server HTTP port', true)
-            ->param('httpsPort', '', new Text(4), 'Server HTTPS port', true)
+            ->param('http-port', '', new Text(4), 'Server HTTP port', true)
+            ->param('https-port', '', new Text(4), 'Server HTTPS port', true)
             ->param('organization', 'appwrite', new Text(0), 'Docker Registry organization', true)
             ->param('image', 'appwrite', new Text(0), 'Main appwrite docker image', true)
             ->param('interactive', 'Y', new Text(1), 'Run an interactive session', true)
-            ->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive));
+            ->param('no-start', false, new Boolean(true), 'Run an interactive session', true)
+            ->callback(fn ($httpPort, $httpsPort, $organization, $image, $interactive, $noStart) => $this->action($httpPort, $httpsPort, $organization, $image, $interactive, $noStart));
     }
 
-    public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive): void
+    public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive, bool $noStart): void
     {
         $config = Config::getParam('variables');
         $defaultHTTPPort = '80';
@@ -220,9 +222,11 @@ class Install extends Action
             }
         }
 
-        Console::log("Running \"docker compose up -d --remove-orphans --renew-anon-volumes\"");
-
-        $exit = Console::execute("$env docker compose --project-directory $this->path up -d --remove-orphans --renew-anon-volumes", '', $stdout, $stderr);
+        $exit = 0;
+        if (!$noStart) {
+            Console::log("Running \"docker compose up -d --remove-orphans --renew-anon-volumes\"");
+            $exit = Console::execute("$env docker compose --project-directory $this->path up -d --remove-orphans --renew-anon-volumes", '', $stdout, $stderr);
+        }
 
         if ($exit !== 0) {
             $message = 'Failed to install Appwrite dockers';

src/Appwrite/Platform/Tasks/Maintenance.php

@@ -36,50 +36,77 @@ class Maintenance extends Action
 
         // # of days in seconds (1 day = 86400s)
         $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
-        $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600');
-        $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600');
-        $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400');
         $usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days
         $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
         $schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
 
-        Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) {
+        Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) {
             $time = DateTime::now();
 
             Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
-            $this->notifyDeleteExecutionLogs($executionLogsRetention, $queueForDeletes);
-            $this->notifyDeleteAbuseLogs($abuseLogsRetention, $queueForDeletes);
-            $this->notifyDeleteAuditLogs($auditLogRetention, $queueForDeletes);
-            $this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
+
+            $this->foreachProject($dbForConsole, function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) {
+                $queueForDeletes->setProject($project);
+
+                $this->notifyDeleteExecutionLogs($queueForDeletes);
+                $this->notifyDeleteAbuseLogs($queueForDeletes);
+                $this->notifyDeleteAuditLogs($queueForDeletes);
+                $this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
+                $this->notifyDeleteExpiredSessions($queueForDeletes);
+            });
+
             $this->notifyDeleteConnections($queueForDeletes);
-            $this->notifyDeleteExpiredSessions($queueForDeletes);
             $this->renewCertificates($dbForConsole, $queueForCertificates);
             $this->notifyDeleteCache($cacheRetention, $queueForDeletes);
             $this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
         }, $interval);
     }
 
-    private function notifyDeleteExecutionLogs(int $interval, Delete $queueForDeletes): void
+    protected function foreachProject(Database $dbForConsole, callable $callback): void
+    {
+        // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
+        $count = 0;
+        $chunk = 0;
+        $limit = 50;
+        $sum = $limit;
+        $executionStart = \microtime(true);
+
+        while ($sum === $limit) {
+            $projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
+
+            $chunk++;
+
+            /** @var string[] $projectIds */
+            $sum = count($projects);
+
+            foreach ($projects as $project) {
+                $callback($project);
+                $count++;
+            }
+        }
+
+        $executionEnd = \microtime(true);
+        Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
+    }
+
+    private function notifyDeleteExecutionLogs(Delete $queueForDeletes): void
     {
         ($queueForDeletes)
             ->setType(DELETE_TYPE_EXECUTIONS)
-            ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
             ->trigger();
     }
 
-    private function notifyDeleteAbuseLogs(int $interval, Delete $queueForDeletes): void
+    private function notifyDeleteAbuseLogs(Delete $queueForDeletes): void
     {
         ($queueForDeletes)
             ->setType(DELETE_TYPE_ABUSE)
-            ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
             ->trigger();
     }
 
-    private function notifyDeleteAuditLogs(int $interval, Delete $queueForDeletes): void
+    private function notifyDeleteAuditLogs(Delete $queueForDeletes): void
     {
         ($queueForDeletes)
             ->setType(DELETE_TYPE_AUDIT)
-            ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
             ->trigger();
     }
 

src/Appwrite/Platform/Tasks/Migrate.php

@@ -11,6 +11,7 @@ use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
+use Utopia\Registry\Registry;
 use Utopia\Validator\Text;
 
 class Migrate extends Action
@@ -29,7 +30,8 @@ class Migrate extends Action
             ->inject('cache')
             ->inject('dbForConsole')
             ->inject('getProjectDB')
-            ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB) => $this->action($version, $cache, $dbForConsole, $getProjectDB));
+            ->inject('register')
+            ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB, Registry $register) => $this->action($version, $cache, $dbForConsole, $getProjectDB, $register));
     }
 
     private function clearProjectsCache(Cache $cache, Document $project)
@@ -41,7 +43,7 @@ class Migrate extends Action
         }
     }
 
-    public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB)
+    public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register)
     {
         Authorization::disable();
         if (!array_key_exists($version, Migration::$versions)) {
@@ -89,9 +91,11 @@ class Migrate extends Action
 
                 try {
                     // TODO: Iterate through all project DBs
+                    /** @var Database $projectDB */
                     $projectDB = $getProjectDB($project);
                     $migration
                         ->setProject($project, $projectDB, $dbForConsole)
+                        ->setPDO($register->get('db', true))
                         ->execute();
                 } catch (\Throwable $th) {
                     Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());

src/Appwrite/Platform/Tasks/QueueCount.php

@@ -0,0 +1,77 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Appwrite\Event\Event;
+use Utopia\CLI\Console;
+use Utopia\Platform\Action;
+use Utopia\Queue\Client;
+use Utopia\Queue\Connection;
+use Utopia\Validator\WhiteList;
+
+class QueueCount extends Action
+{
+    public static function getName(): string
+    {
+        return 'queue-count';
+    }
+
+
+    public function __construct()
+    {
+        $this
+            ->desc('Return the number of from a specific queue identified by the name parameter with a specific type')
+            ->param('name', '', new WhiteList([
+                Event::DATABASE_QUEUE_NAME,
+                Event::DELETE_QUEUE_NAME,
+                Event::AUDITS_QUEUE_NAME,
+                Event::MAILS_QUEUE_NAME,
+                Event::FUNCTIONS_QUEUE_NAME,
+                Event::USAGE_QUEUE_NAME,
+                Event::WEBHOOK_QUEUE_NAME,
+                Event::CERTIFICATES_QUEUE_NAME,
+                Event::BUILDS_QUEUE_NAME,
+                Event::MESSAGING_QUEUE_NAME,
+                Event::MIGRATIONS_QUEUE_NAME,
+                Event::HAMSTER_QUEUE_NAME
+            ]), 'Queue name')
+            ->param('type', '', new WhiteList([
+                'success',
+                'failed',
+                'processing',
+            ]), 'Queue type')
+            ->inject('queue')
+            ->callback(fn ($name, $type, $queue) => $this->action($name, $type, $queue));
+    }
+
+    /**
+     * @param string $name The name of the queue to count the jobs from
+     * @param string $type The type of jobs to count
+     * @param Connection $queue
+     */
+    public function action(string $name, string $type, Connection $queue): void
+    {
+        if (!$name) {
+            Console::error('Missing required parameter $name');
+            return;
+        }
+
+        $queueClient = new Client($name, $queue);
+
+        $count = 0;
+
+        switch ($type) {
+            case 'success':
+                $count = $queueClient->countSuccessfulJobs();
+                break;
+            case 'failed':
+                $count = $queueClient->countFailedJobs();
+                break;
+            case 'processing':
+                $count = $queueClient->countProcessingJobs();
+                break;
+        };
+
+        Console::log("Queue: '{$name}' has {$count} {$type} jobs.");
+    }
+}

src/Appwrite/Platform/Tasks/QueueRetry.php

@@ -0,0 +1,64 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Appwrite\Event\Event;
+use Utopia\CLI\Console;
+use Utopia\Platform\Action;
+use Utopia\Queue\Client;
+use Utopia\Queue\Connection;
+use Utopia\Validator\WhiteList;
+
+class QueueRetry extends Action
+{
+    public static function getName(): string
+    {
+        return 'queue-retry';
+    }
+
+
+    public function __construct()
+    {
+        $this
+            ->desc('Retry failed jobs from a specific queue identified by the name parameter')
+            ->param('name', '', new WhiteList([
+                Event::DATABASE_QUEUE_NAME,
+                Event::DELETE_QUEUE_NAME,
+                Event::AUDITS_QUEUE_NAME,
+                Event::MAILS_QUEUE_NAME,
+                Event::FUNCTIONS_QUEUE_NAME,
+                Event::USAGE_QUEUE_NAME,
+                Event::WEBHOOK_CLASS_NAME,
+                Event::CERTIFICATES_QUEUE_NAME,
+                Event::BUILDS_QUEUE_NAME,
+                Event::MESSAGING_QUEUE_NAME,
+                Event::MIGRATIONS_QUEUE_NAME,
+                Event::HAMSTER_CLASS_NAME
+            ]), 'Queue name')
+            ->inject('queue')
+            ->callback(fn ($name, $queue) => $this->action($name, $queue));
+    }
+
+    /**
+     * @param string $name The name of the queue to retry jobs from
+     * @param Connection $queue
+     */
+    public function action(string $name, Connection $queue): void
+    {
+        if (!$name) {
+            Console::error('Missing required parameter $name');
+            return;
+        }
+
+        $queueClient = new Client($name, $queue);
+
+        if ($queueClient->countFailedJobs() === 0) {
+            Console::error('No failed jobs found.');
+            return;
+        }
+
+        Console::log('Retrying failed jobs...');
+
+        $queueClient->retry();
+    }
+}

src/Appwrite/Platform/Tasks/SDKs.php

@@ -235,6 +235,11 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
                         'X-Appwrite-Response-Format' => '1.4.0',
                     ]);
 
+                // Make sure we have a clean slate.
+                // Otherwise, all files in this dir will be pushed,
+                // regardless of whether they were just generated or not.
+                \exec('rm -rf ' . $result);
+
                 try {
                     $sdk->generate($result);
                 } catch (Exception $exception) {

src/Appwrite/Platform/Tasks/SSL.php

@@ -7,6 +7,7 @@ use Appwrite\Event\Certificate;
 use Utopia\App;
 use Utopia\CLI\Console;
 use Utopia\Database\Document;
+use Utopia\Validator\Boolean;
 use Utopia\Validator\Hostname;
 
 class SSL extends Action
@@ -21,19 +22,22 @@ class SSL extends Action
         $this
             ->desc('Validate server certificates')
             ->param('domain', App::getEnv('_APP_DOMAIN', ''), new Hostname(), 'Domain to generate certificate for. If empty, main domain will be used.', true)
+            ->param('skip-check', true, new Boolean(true), 'If DNS and renew check should be skipped. Defaults to true, and when true, all jobs will result in certificate generation attempt.', true)
             ->inject('queueForCertificates')
-            ->callback(fn (string $domain, Certificate $queueForCertificates) => $this->action($domain, $queueForCertificates));
+            ->callback(fn (string $domain, bool|string $skipCheck, Certificate $queueForCertificates) => $this->action($domain, $skipCheck, $queueForCertificates));
     }
 
-    public function action(string $domain, Certificate $queueForCertificates): void
+    public function action(string $domain, bool|string $skipCheck, Certificate $queueForCertificates): void
     {
+        $skipCheck = \strval($skipCheck) === 'true';
+
         Console::success('Scheduling a job to issue a TLS certificate for domain: ' . $domain);
 
         $queueForCertificates
             ->setDomain(new Document([
                 'domain' => $domain
             ]))
-            ->setSkipRenewCheck(true)
+            ->setSkipRenewCheck($skipCheck)
             ->trigger();
     }
 }