Merge pull request #7315 from appwrite/update-console

chore: update console

.env

@@ -50,10 +50,6 @@ _APP_STORAGE_WASABI_BUCKET=
 _APP_STORAGE_ANTIVIRUS=disabled
 _APP_STORAGE_ANTIVIRUS_HOST=clamav
 _APP_STORAGE_ANTIVIRUS_PORT=3310
-_APP_INFLUXDB_HOST=influxdb
-_APP_INFLUXDB_PORT=8086
-_APP_STATSD_HOST=telegraf
-_APP_STATSD_PORT=8125
 _APP_SMTP_HOST=maildev
 _APP_SMTP_PORT=1025
 _APP_SMTP_SECURE=
@@ -91,7 +87,7 @@ _APP_GRAPHQL_MAX_DEPTH=3
 _APP_DOCKER_HUB_USERNAME=
 _APP_DOCKER_HUB_PASSWORD=
 _APP_VCS_GITHUB_APP_NAME=
-_APP_VCS_GITHUB_PRIVATE_KEY=""
+_APP_VCS_GITHUB_PRIVATE_KEY=disabled
 _APP_VCS_GITHUB_APP_ID=
 _APP_VCS_GITHUB_CLIENT_ID=
 _APP_VCS_GITHUB_CLIENT_SECRET=

.github/workflows/linter.yml

@@ -18,6 +18,10 @@ jobs:
 
     - run: git checkout HEAD^2
 
+    - name: Validate composer.json and composer.lock
+      run: |
+        docker run --rm -v $PWD:/app composer sh -c \
+        "composer validate"
     - name: Run Linter
       run: |
         docker run --rm -v $PWD:/app composer sh -c \

.github/workflows/publish.yml

@@ -16,7 +16,6 @@ jobs:
       with:
         fetch-depth: 2
         submodules: recursive
-        ref: cl-1.4.x
 
     - name: Login to Docker Hub
       uses: docker/login-action@v2

.github/workflows/tests.yml

@@ -4,57 +4,119 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  IMAGE: appwrite-dev
+  CACHE_KEY: appwrite-dev-${{ github.event.pull_request.head.sha }}
+
 on: [pull_request]
+
 jobs:
-  tests:
-    name: Unit & E2E
+  setup:
+    name: Setup & Build Appwrite Image
     runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build Appwrite
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: false
+          tags: ${{ env.IMAGE }}
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          outputs: type=docker,dest=/tmp/${{ env.IMAGE }}.tar
+          build-args: |
+            DEBUG=false
+            TESTING=true
+            VERSION=dev
+
+      - name: Cache Docker Image
+        uses: actions/cache@v3
+        with:
+          key: ${{ env.CACHE_KEY }}
+          path: /tmp/${{ env.IMAGE }}.tar
+
+  unit_test:
+    name: Unit Test
+    runs-on: ubuntu-latest
+    needs: setup
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-      with:
-        # We must fetch at least the immediate parents so that if this is
-        # a pull request then we can checkout the head.
-        fetch-depth: 2
-        # Fetch submodules
-        submodules: recursive
+      - name: checkout
+        uses: actions/checkout@v2
 
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
+      - name: Load Cache
+        uses: actions/cache@v3
+        with:
+          key: ${{ env.CACHE_KEY }}
+          path: /tmp/${{ env.IMAGE }}.tar
+          fail-on-cache-miss: true
 
-    # This is a separate action that sets up buildx runner
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      - name: Load and Start Appwrite
+        run: |
+          docker load --input /tmp/${{ env.IMAGE }}.tar
+          docker compose up -d
+          sleep 10
 
-    - name: Build Appwrite
-      uses: docker/build-push-action@v3
-      with:
-        context: .
-        push: false
-        tags: appwrite-dev
-        load: true
-        cache-from: type=gha
-        cache-to: type=gha,mode=max
-        build-args: |
-          DEBUG=false
-          TESTING=true
-          VERSION=dev
+      - name: Doctor
+        run: docker compose exec -T appwrite doctor
 
-    - name: Start Appwrite
-      run:  |
-        docker compose up -d
-        sleep 30
+      - name: Environment Variables
+        run: docker compose exec -T appwrite vars
 
-    - name: Doctor
-      run:  |
-        docker compose logs appwrite
-        docker compose exec -T appwrite doctor
+      - name: Run Unit Tests
+        run: docker compose exec appwrite test /usr/src/code/tests/unit
 
-    - name: Environment Variables
-      run:  docker compose exec -T appwrite vars
+  e2e_test:
+    name: E2E Test
+    runs-on: ubuntu-latest
+    needs: setup
+    strategy:
+      fail-fast: false
+      matrix:
+        service:
+          [
+            Account,
+            Avatars,
+            Console,
+            Databases,
+            Functions,
+            GraphQL,
+            Health,
+            Locale,
+            Projects,
+            Realtime,
+            Storage,
+            Teams,
+            Users,
+            Webhooks,
+            VCS,
+          ]
 
-    - name: Run Tests
-      run: docker compose exec -T appwrite test --debug
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: Load Cache
+        uses: actions/cache@v3
+        with:
+          key: ${{ env.CACHE_KEY }}
+          path: /tmp/${{ env.IMAGE }}.tar
+          fail-on-cache-miss: true
+
+      - name: Load and Start Appwrite
+        run: |
+          docker load --input /tmp/${{ env.IMAGE }}.tar
+          docker compose up -d
+          sleep 10
+
+      - name: Run ${{matrix.service}} Tests
+        run: docker compose exec -T appwrite test /usr/src/code/tests/e2e/Services/${{matrix.service}} --debug

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "app/console"]
 	path = app/console
 	url = https://github.com/appwrite/console
-	branch = 3.2.3
+	branch = 3.3.8

.gitpod.yml

@@ -8,7 +8,7 @@ tasks:
     command: |
       docker run --rm --interactive --tty \
       --volume $PWD:/app \
-      composer update \
+      composer install \
       --ignore-platform-reqs \
       --optimize-autoloader \
       --no-plugins \

CHANGES.md

@@ -1,3 +1,103 @@
+# Version 1.4.14
+
+## Changes
+- New usage metrics collection flow [#7005](https://github.com/appwrite/appwrite/pull/7005)
+  - Deprecated influxdb, telegraf containers and removed all of their occurrences from the code.
+  - Removed _APP_INFLUXDB_HOST, _APP_INFLUXDB_PORT, _APP_STATSD_HOST, _APP_STATSD_PORT env variables.
+  - Removed usage labels dependency.
+  - Dropped type attribute from stats collection.
+  - Usage metrics are processed via new usage worker.
+  - updated Metric names.
+  
+# Version 1.4.13
+
+## Notable changes
+
+* Change enum size validation in update controller [#7164](https://github.com/appwrite/appwrite/pull/7164)
+* Bump console to version 3.2.8 in [#7167](https://github.com/appwrite/appwrite/pull/7167)
+
+## Bug fixes
+
+* Fix error after adding bigger enum [#7162](https://github.com/appwrite/appwrite/pull/7162)
+* Add chunkId to abuse key to prevent rate limit for SDKs [#7154](https://github.com/appwrite/appwrite/pull/7154)
+
+## Miscellaneous
+
+* Fix enum test case [#7163](https://github.com/appwrite/appwrite/pull/7163)
+* Add flag to send logs to logger [#7155](https://github.com/appwrite/appwrite/pull/7155)
+* Add a CI task to validate composer file and lock [#7142](https://github.com/appwrite/appwrite/pull/7142)
+
+# Version 1.4.12
+
+## Miscellaneous
+* Bump console to version 3.2.7 [#7148](https://github.com/appwrite/appwrite/pull/7148)
+* Chore update database to 0.45.2 [#7138](https://github.com/appwrite/appwrite/pull/7138)
+* Implement queue thresholds for the health API [#7123](https://github.com/appwrite/appwrite/pull/7123)
+* Add Authorization::skip to the usage worker [#7124](https://github.com/appwrite/appwrite/pull/7124)
+
+## Bug fixes
+* fix: use queueForDeletes in git installation delete endpoint [#7140](https://github.com/appwrite/appwrite/pull/7140)
+* fix: patch script, make errors silent [#7134](https://github.com/appwrite/appwrite/pull/7134)
+* fix: repositories recreation script [#7133](https://github.com/appwrite/appwrite/pull/7133)
+* fix: Only delete repositories linked to the particular project [#7131](https://github.com/appwrite/appwrite/pull/7131)
+
+# Version 1.4.11
+
+## Miscellaneous
+
+* Update database by @abnegate in [#7111](https://github.com/appwrite/appwrite/pull/7111)
+
+# Version 1.4.10
+
+## Bug fixes
+* Handle cases where password history could contain NULLs [#7092](https://github.com/appwrite/appwrite/pull/7092)
+* Missing functionId error on create execution [#7091](https://github.com/appwrite/appwrite/pull/7091)
+* Ensure usage endpoints don't throw 500 when usage is disabled [#7087](https://github.com/appwrite/appwrite/pull/7087)
+* Missing sessionId error when deleting all user sessions [#7085](https://github.com/appwrite/appwrite/pull/7085)
+* Domain validation in Create Proxy rule results in 500 error [#7084](https://github.com/appwrite/appwrite/pull/7084)
+* Fix optional services [#7078](https://github.com/appwrite/appwrite/pull/7078)
+* Fix regression from worker refactor [#7074](https://github.com/appwrite/appwrite/pull/7074)
+* Use getQueueSize() in the Health service's get X queue endpoints [#7073](https://github.com/appwrite/appwrite/pull/7073)
+* Delete linked VCS repos and comments [#7066](https://github.com/appwrite/appwrite/pull/7066)
+
+
+# Version 1.4.9
+
+## Bug fixes
+
+* Fix 400 error on function domain execution in [#7059](https://github.com/appwrite/appwrite/pull/7059)
+
+# Version 1.4.8
+
+## Notable changes
+
+* Fix certificate emails and add support for variables in email template subject in [#6495](https://github.com/appwrite/appwrite/)pull/6495
+* Bump console to version 3.2.5 in [#7027](https://github.com/appwrite/appwrite/pull/7027)
+* Bump utopia database and storage versions in [#7002](https://github.com/appwrite/appwrite/pull/7002)
+
+## Bug fixes
+
+* Fixes cookie headers not being passed properly by router in [#7024](https://github.com/appwrite/appwrite/pull/7024)
+* Fix permission problem in deletes worker in [#7013](https://github.com/appwrite/appwrite/pull/7013)
+
+## Miscellaneous
+
+* Improve error handling in the realtime service in [#6998](https://github.com/appwrite/appwrite/pull/6998)
+* Update the error code for unsupported protocol in [#7006](https://github.com/appwrite/appwrite/pull/7006)
+* Improve CI tests by executing them in parallel in [#6198](https://github.com/appwrite/appwrite/pull/6198)
+* Update README.md to add links to orchestration tools in [#7011](https://github.com/appwrite/appwrite/pull/7011)
+* Update gitpod setup to install instead of update dependencies in [#6938](https://github.com/appwrite/appwrite/pull/6938)
+* Remove analytics from install script in [#7017](https://github.com/appwrite/appwrite/pull/7017)
+* Improve database logging in [#7003](https://github.com/appwrite/appwrite/pull/7003)
+* Add VCS tests in [#6894](https://github.com/appwrite/appwrite/pull/6894)
+* Improve error messages in [#6487](https://github.com/appwrite/appwrite/pull/6487)
+* Add command to delete orphaned projects in [#7015](https://github.com/appwrite/appwrite/pull/7015)
+
+# Version 1.4.7
+
+## Fixes
+- Fix missing body in async function execution in [#6988](https://github.com/appwrite/appwrite/pull/6988)
+
 # Version 1.4.6
 
 ## Changes

CONTRIBUTING.md

@@ -89,7 +89,7 @@ $ git push origin [name_of_your_new_branch]
 
 To set up a working **development environment**, just fork the project git repository and install the backend and frontend dependencies using the proper package manager and create run the docker-compose stack.
 
-> If you just want to install Appwrite for day-to-day use and not as a contributor, you can reference the [installation guide](https://github.com/appwrite/appwrite#installation), the [getting started guide](https://appwrite.io/docs/getting-started-for-web), or the main [README](README.md) file.
+> If you just want to install Appwrite for day-to-day use and not as a contributor, you can reference the [installation guide](https://github.com/appwrite/appwrite#installation), the [getting started guide](https://appwrite.io/docs/quick-starts), or the main [README](README.md) file.
 
 ```bash
 git clone git@github.com:[YOUR_FORK_HERE]/appwrite.git
@@ -210,7 +210,6 @@ Appwrite's current structure is a combination of both [Monolithic](https://en.wi
 │       ├── Task
 │       ├── Template
 │       ├── URL
-│       ├── Usage
 │       └── Utopia
 └── tests # End to end & unit tests
     ├── e2e
@@ -454,7 +453,7 @@ From time to time, our team will add tutorials that will help contributors find
 
 - [Adding Support for a New OAuth2 Provider](./docs/tutorials/add-oauth2-provider.md)
 - [Appwrite Environment Variables](./docs/tutorials/add-environment-variable.md)
-- [Running in Production](https://appwrite.io/docs/production)
+- [Running in Production](https://appwrite.io/docs/advanced/self-hosting/production)
 - [Adding Storage Adapter](./docs/tutorials/add-storage-adapter.md)
 
 ## Other Ways to Help

Dockerfile

@@ -74,7 +74,6 @@ RUN mkdir -p /storage/uploads && \
 # Executables
 RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/maintenance &&  \
-    chmod +x /usr/local/bin/usage && \
     chmod +x /usr/local/bin/install && \
     chmod +x /usr/local/bin/upgrade && \
     chmod +x /usr/local/bin/migrate && \
@@ -94,16 +93,21 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/worker-mails && \
     chmod +x /usr/local/bin/worker-messaging && \
     chmod +x /usr/local/bin/worker-webhooks && \
+    chmod +x /usr/local/bin/worker-usage && \
     chmod +x /usr/local/bin/worker-migrations
 
 # Cloud Executabless
 RUN chmod +x /usr/local/bin/hamster && \
     chmod +x /usr/local/bin/volume-sync && \
     chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \
+    chmod +x /usr/local/bin/patch-recreate-repositories-documents && \
     chmod +x /usr/local/bin/patch-delete-project-collections && \
+    chmod +x /usr/local/bin/delete-orphaned-projects && \
     chmod +x /usr/local/bin/clear-card-cache && \
     chmod +x /usr/local/bin/calc-users-stats && \
-    chmod +x /usr/local/bin/calc-tier-stats
+    chmod +x /usr/local/bin/calc-tier-stats && \
+    chmod +x /usr/local/bin/get-migration-stats && \
+    chmod +x /usr/local/bin/create-inf-metric
 
 # Letsencrypt Permissions
 RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/

README-CN.md

@@ -55,7 +55,7 @@ Appwrite 可以提供给开发者用户验证，外部授权，用户数据读
 
 ## 安装
 
-Appwrite 的容器化服务器只需要一行指令就可以运行。您可以使用 docker-compose 在本地主机上运行 Appwrite，也可以在任何其他容器化工具（如 Kubernetes、Docker Swarm 或 Rancher）上运行 Appwrite。
+Appwrite 的容器化服务器只需要一行指令就可以运行。您可以使用 docker-compose 在本地主机上运行 Appwrite，也可以在任何其他容器化工具（如 [Kubernetes](https://kubernetes.io/docs/home/)、[Docker Swarm](https://docs.docker.com/engine/swarm/) 或 [Rancher](https://rancher.com/docs/)）上运行 Appwrite。
 
 启动 Appwrite 服务器的最简单方法是运行我们的 docker-compose 文件。在运行安装命令之前，请确保您的机器上安装了 [Docker](https://dockerdocs.cn/get-docker/index.html)：
 
@@ -66,7 +66,7 @@ docker run -it --rm \
     --volume /var/run/docker.sock:/var/run/docker.sock \
     --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \
     --entrypoint="install" \
-    appwrite/appwrite:1.4.6
+    appwrite/appwrite:1.4.13
 ```
 
 ### Windows
@@ -78,7 +78,7 @@ docker run -it --rm ^
     --volume //var/run/docker.sock:/var/run/docker.sock ^
     --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^
     --entrypoint="install" ^
-    appwrite/appwrite:1.4.6
+    appwrite/appwrite:1.4.13
 ```
 
 #### PowerShell
@@ -88,7 +88,7 @@ docker run -it --rm `
     --volume /var/run/docker.sock:/var/run/docker.sock `
     --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw `
     --entrypoint="install" `
-    appwrite/appwrite:1.4.6
+    appwrite/appwrite:1.4.13
 ```
 
 运行后，可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后，服务器可能需要几分钟才能启动。

README.md

@@ -65,7 +65,7 @@ Table of Contents:
 
 ## Installation
 
-Appwrite is designed to run in a containerized environment. Running your server is as easy as running one command from your terminal. You can either run Appwrite on your localhost using docker-compose or on any other container orchestration tool, such as Kubernetes, Docker Swarm, or Rancher.
+Appwrite is designed to run in a containerized environment. Running your server is as easy as running one command from your terminal. You can either run Appwrite on your localhost using docker-compose or on any other container orchestration tool, such as [Kubernetes](https://kubernetes.io/docs/home/), [Docker Swarm](https://docs.docker.com/engine/swarm/), or [Rancher](https://rancher.com/docs/).
 
 The easiest way to start running your Appwrite server is by running our docker-compose file. Before running the installation command, make sure you have [Docker](https://www.docker.com/products/docker-desktop) installed on your machine:
 
@@ -76,7 +76,7 @@ docker run -it --rm \
     --volume /var/run/docker.sock:/var/run/docker.sock \
     --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \
     --entrypoint="install" \
-    appwrite/appwrite:1.4.6
+    appwrite/appwrite:1.4.13
 ```
 
 ### Windows
@@ -88,7 +88,7 @@ docker run -it --rm ^
     --volume //var/run/docker.sock:/var/run/docker.sock ^
     --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^
     --entrypoint="install" ^
-    appwrite/appwrite:1.4.6
+    appwrite/appwrite:1.4.13
 ```
 
 #### PowerShell
@@ -98,7 +98,7 @@ docker run -it --rm `
     --volume /var/run/docker.sock:/var/run/docker.sock `
     --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw `
     --entrypoint="install" `
-    appwrite/appwrite:1.4.6
+    appwrite/appwrite:1.4.13
 ```
 
 Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation.

app/cli.php

@@ -61,7 +61,11 @@ CLI::setResource('dbForConsole', function ($pools, $cache) {
                 ->getResource();
 
             $dbForConsole = new Database($dbAdapter, $cache);
-            $dbForConsole->setNamespace('_console');
+
+            $dbForConsole
+                ->setNamespace('_console')
+                ->setMetadata('host', \gethostname())
+                ->setMetadata('project', 'console');
 
             // Ensure tables exist
             $collections = Config::getParam('collections', [])['console'];
@@ -111,36 +115,15 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
 
         $databases[$databaseName] = $database;
 
-        $database->setNamespace('_' . $project->getInternalId());
+        $database
+            ->setNamespace('_' . $project->getInternalId())
+            ->setMetadata('host', \gethostname())
+            ->setMetadata('project', $project->getId());
 
         return $database;
     };
 }, ['pools', 'dbForConsole', 'cache']);
 
-CLI::setResource('influxdb', function (Registry $register) {
-    $client = $register->get('influxdb'); /** @var InfluxDB\Client $client */
-    $attempts = 0;
-    $max = 10;
-    $sleep = 1;
-
-    do { // check if telegraf database is ready
-        try {
-            $attempts++;
-            $database = $client->selectDB('telegraf');
-            if (in_array('telegraf', $client->listDatabases())) {
-                break; // leave the do-while if successful
-            }
-        } catch (\Throwable $th) {
-            Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
-            if ($attempts >= $max) {
-                throw new \Exception('InfluxDB database not ready yet');
-            }
-            sleep($sleep);
-        }
-    } while ($attempts < $max);
-    return $database;
-}, ['register']);
-
 CLI::setResource('queue', function (Group $pools) {
     return $pools->get('queue')->pop()->getResource();
 }, ['pools']);

app/config/collections.php

@@ -18,6 +18,63 @@ $auth = Config::getParam('auth', []);
  */
 
 $commonCollections = [
+    'cache' => [
+        '$collection' => Database::METADATA,
+        '$id' => 'cache',
+        'name' => 'Cache',
+        'attributes' => [
+            [
+                '$id' => 'resource',
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => 255,
+                'signed' => true,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
+            [
+                '$id' => 'accessedAt',
+                'type' => Database::VAR_DATETIME,
+                'format' => '',
+                'size' => 0,
+                'signed' => false,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => ['datetime'],
+            ],
+            [
+                '$id' => 'signature',
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => 255,
+                'signed' => true,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
+        ],
+        'indexes' => [
+            [
+                '$id' => '_key_accessedAt',
+                'type' => Database::INDEX_KEY,
+                'attributes' => ['accessedAt'],
+                'lengths' => [],
+                'orders' => [],
+            ],
+            [
+                '$id' => '_key_resource',
+                'type' => Database::INDEX_KEY,
+                'attributes' => ['resource'],
+                'lengths' => [],
+                'orders' => [],
+            ],
+        ],
+    ],
+
     'users' => [
         '$collection' => ID::custom(Database::METADATA),
         '$id' => ID::custom('users'),
@@ -1270,10 +1327,10 @@ $commonCollections = [
         ]
     ],
 
-    'stats' => [
+    'stats_v2' => [
         '$collection' => ID::custom(Database::METADATA),
-        '$id' => ID::custom('stats'),
-        'name' => 'Stats',
+        '$id' => ID::custom('stats_v2'),
+        'name' => 'stats_v2',
         'attributes' => [
             [
                 '$id' => ID::custom('metric'),
@@ -1302,7 +1359,7 @@ $commonCollections = [
                 'type' => Database::VAR_INTEGER,
                 'format' => '',
                 'size' => 8,
-                'signed' => false,
+                'signed' => true,
                 'required' => true,
                 'default' => null,
                 'array' => false,
@@ -1330,17 +1387,6 @@ $commonCollections = [
                 'array' => false,
                 'filters' => [],
             ],
-            [
-                '$id' => ID::custom('type'),
-                'type' => Database::VAR_INTEGER,
-                'format' => '',
-                'size' => 1,
-                'signed' => false,
-                'required' => true,
-                'default' => 0, // 0 -> count, 1 -> sum
-                'array' => false,
-                'filters' => [],
-            ],
         ],
         'indexes' => [
             [
@@ -1359,7 +1405,7 @@ $commonCollections = [
             ],
             [
                 '$id' => ID::custom('_key_metric_period_time'),
-                'type' => Database::INDEX_KEY,
+                'type' => Database::INDEX_UNIQUE,
                 'attributes' => ['metric', 'period', 'time'],
                 'lengths' => [],
                 'orders' => [Database::ORDER_DESC],
@@ -2883,63 +2929,6 @@ $projectCollections = array_merge([
         ],
     ],
 
-    'cache' => [
-        '$collection' => Database::METADATA,
-        '$id' => 'cache',
-        'name' => 'Cache',
-        'attributes' => [
-            [
-                '$id' => 'resource',
-                'type' => Database::VAR_STRING,
-                'format' => '',
-                'size' => 255,
-                'signed' => true,
-                'required' => false,
-                'default' => null,
-                'array' => false,
-                'filters' => [],
-            ],
-            [
-                '$id' => 'accessedAt',
-                'type' => Database::VAR_DATETIME,
-                'format' => '',
-                'size' => 0,
-                'signed' => false,
-                'required' => false,
-                'default' => null,
-                'array' => false,
-                'filters' => ['datetime'],
-            ],
-            [
-                '$id' => 'signature',
-                'type' => Database::VAR_STRING,
-                'format' => '',
-                'size' => 255,
-                'signed' => true,
-                'required' => false,
-                'default' => null,
-                'array' => false,
-                'filters' => [],
-            ],
-        ],
-        'indexes' => [
-            [
-                '$id' => '_key_accessedAt',
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['accessedAt'],
-                'lengths' => [],
-                'orders' => [],
-            ],
-            [
-                '$id' => '_key_resource',
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['resource'],
-                'lengths' => [],
-                'orders' => [],
-            ],
-        ],
-    ],
-
     'variables' => [
         '$collection' => Database::METADATA,
         '$id' => 'variables',

app/config/errors.php

@@ -86,7 +86,7 @@ return [
     Exception::GENERAL_PROTOCOL_UNSUPPORTED => [
         'name' => Exception::GENERAL_PROTOCOL_UNSUPPORTED,
         'description' => 'The request cannot be fulfilled with the current protocol. Please check the value of the _APP_OPTIONS_FORCE_HTTPS environment variable.',
-        'code' => 500,
+        'code' => 426,
     ],
     Exception::GENERAL_CODES_DISABLED => [
         'name' => Exception::GENERAL_CODES_DISABLED,
@@ -209,6 +209,7 @@ return [
         'name' => Exception::USER_AUTH_METHOD_UNSUPPORTED,
         'description' => 'The requested authentication method is either disabled or unsupported. Please check the supported authentication methods in the Appwrite console.',
         'code' => 501,
+        'publish' => false,
     ],
     Exception::USER_PHONE_ALREADY_EXISTS => [
         'name' => Exception::USER_PHONE_ALREADY_EXISTS,
@@ -240,6 +241,16 @@ return [
         'description' => 'OAuth2 provider returned some error.',
         'code' => 424,
     ],
+    Exception::USER_EMAIL_ALREADY_VERIFIED => [
+        'name' => Exception::USER_EMAIL_ALREADY_VERIFIED,
+        'description' => 'User email is already verified',
+        'code' => 409,
+    ],
+    Exception::USER_PHONE_ALREADY_VERIFIED => [
+        'name' => Exception::USER_PHONE_ALREADY_VERIFIED,
+        'description' => 'User phone is already verified',
+        'code' => 409
+    ],
 
     /** Teams */
     Exception::TEAM_NOT_FOUND => [
@@ -355,7 +366,7 @@ return [
     ],
     Exception::STORAGE_BUCKET_ALREADY_EXISTS => [
         'name' => Exception::STORAGE_BUCKET_ALREADY_EXISTS,
-        'description' => 'A storage bucket with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.',
+        'description' => 'A storage bucket with the requested ID already exists. Try again with a different ID or use ID.unique() to generate a unique ID.',
         'code' => 409,
     ],
     Exception::STORAGE_BUCKET_NOT_FOUND => [
@@ -479,7 +490,7 @@ return [
     ],
     Exception::COLLECTION_ALREADY_EXISTS => [
         'name' => Exception::COLLECTION_ALREADY_EXISTS,
-        'description' => 'A collection with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.',
+        'description' => 'A collection with the requested ID already exists. Try again with a different ID or use ID.unique() to generate a unique ID.',
         'code' => 409,
     ],
     Exception::COLLECTION_LIMIT_EXCEEDED => [
@@ -511,7 +522,7 @@ return [
     ],
     Exception::DOCUMENT_ALREADY_EXISTS => [
         'name' => Exception::DOCUMENT_ALREADY_EXISTS,
-        'description' => 'Document with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.',
+        'description' => 'Document with the requested ID already exists. Try again with a different ID or use ID.unique() to generate a unique ID.',
         'code' => 409,
     ],
     Exception::DOCUMENT_UPDATE_CONFLICT => [
@@ -553,7 +564,7 @@ return [
     ],
     Exception::ATTRIBUTE_ALREADY_EXISTS => [
         'name' => Exception::ATTRIBUTE_ALREADY_EXISTS,
-        'description' => 'Attribute with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.',
+        'description' => 'Attribute with the requested key already exists. Attribute keys must be unique, try again with a different key.',
         'code' => 409,
     ],
     Exception::ATTRIBUTE_LIMIT_EXCEEDED => [
@@ -585,7 +596,7 @@ return [
     ],
     Exception::INDEX_ALREADY_EXISTS => [
         'name' => Exception::INDEX_ALREADY_EXISTS,
-        'description' => 'Index with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.',
+        'description' => 'Index with the requested key already exists. Try again with a different key.',
         'code' => 409,
     ],
     Exception::INDEX_INVALID => [
@@ -602,7 +613,7 @@ return [
     ],
     Exception::PROJECT_ALREADY_EXISTS => [
         'name' => Exception::PROJECT_ALREADY_EXISTS,
-        'description' => 'Project with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.',
+        'description' => 'Project with the requested ID already exists. Try again with a different ID or use ID.unique() to generate a unique ID.',
         'code' => 409,
     ],
     Exception::PROJECT_UNKNOWN => [
@@ -732,4 +743,34 @@ return [
         'description' => 'Migration is already in progress. You can check the status of the migration in your Appwrite Console\'s "Settings" > "Migrations".',
         'code' => 409,
     ],
+
+    /** Realtime */
+    Exception::REALTIME_MESSAGE_FORMAT_INVALID => [
+        'name' => Exception::REALTIME_MESSAGE_FORMAT_INVALID,
+        'description' => 'Message format is not valid.',
+        'code' => 1003,
+    ],
+    Exception::REALTIME_POLICY_VIOLATION => [
+        'name' => Exception::REALTIME_POLICY_VIOLATION,
+        'description' => 'Policy violation.',
+        'code' => 1008,
+    ],
+    Exception::REALTIME_TOO_MANY_MESSAGES => [
+        'name' => Exception::REALTIME_TOO_MANY_MESSAGES,
+        'description' => 'Too many messages.',
+        'code' => 1013,
+    ],
+    Exception::MIGRATION_PROVIDER_ERROR => [
+        'name' => Exception::MIGRATION_PROVIDER_ERROR,
+        'description' => 'An error occurred on the provider\'s side. Please try again later.',
+        'code' => 400,
+    ],
+
+    /** Health */
+    Exception::QUEUE_SIZE_EXCEEDED => [
+        'name' => Exception::QUEUE_SIZE_EXCEEDED,
+        'description' => 'Queue size threshold hit.',
+        'code' => 503,
+        'publish' => false
+    ],
 ];

app/config/locale/templates/email-inner-base.tpl

@@ -1,15 +1,9 @@
-<p>{{hello}}</p>
-
-<br>
-
+<p>{{hello}},</p>
 <p>{{body}}</p>
-
-<a href="{{redirect}}" target="_blank">{{redirect}}</a>
-
+<p><a href="{{redirect}}" target="_blank">{{redirect}}</a></p>
 <p>{{footer}}</p>
-
-<br>
-
-<p>{{thanks}}</p>
-
-<p>{{signature}}</p>
+<p style="margin-bottom: 32px">
+    {{thanks}},
+    <br/>
+    {{signature}}
+</p>

app/config/platforms.php

@@ -15,7 +15,7 @@ return [
             [
                 'key' => 'web',
                 'name' => 'Web',
-                'version' => '13.0.0',
+                'version' => '13.0.1',
                 'url' => 'https://github.com/appwrite/sdk-for-web',
                 'package' => 'https://www.npmjs.com/package/appwrite',
                 'enabled' => true,
@@ -63,7 +63,7 @@ return [
             [
                 'key' => 'flutter',
                 'name' => 'Flutter',
-                'version' => '11.0.0',
+                'version' => '11.0.1',
                 'url' => 'https://github.com/appwrite/sdk-for-flutter',
                 'package' => 'https://pub.dev/packages/appwrite',
                 'enabled' => true,
@@ -81,7 +81,7 @@ return [
             [
                 'key' => 'apple',
                 'name' => 'Apple',
-                'version' => '4.0.1',
+                'version' => '4.0.2',
                 'url' => 'https://github.com/appwrite/sdk-for-apple',
                 'package' => 'https://github.com/appwrite/sdk-for-apple',
                 'enabled' => true,
@@ -116,7 +116,7 @@ return [
             [
                 'key' => 'android',
                 'name' => 'Android',
-                'version' => '4.0.0',
+                'version' => '4.0.1',
                 'url' => 'https://github.com/appwrite/sdk-for-android',
                 'package' => 'https://search.maven.org/artifact/io.appwrite/sdk-for-android',
                 'enabled' => true,
@@ -185,7 +185,7 @@ return [
             [
                 'key' => 'web',
                 'name' => 'Console',
-                'version' => '0.3.0',
+                'version' => '0.4.2',
                 'url' => 'https://github.com/appwrite/sdk-for-console',
                 'package' => '',
                 'enabled' => true,
@@ -195,7 +195,7 @@ return [
                 'family' => APP_PLATFORM_CONSOLE,
                 'prism' => 'javascript',
                 'source' => \realpath(__DIR__ . '/../sdks/console-web'),
-                'gitUrl' => 'git@github.com:appwrite/sdk-for-console.git',
+                'gitUrl' => 'https://github.com/appwrite/sdk-for-console.git',
                 'gitBranch' => 'dev',
                 'gitRepoName' => 'sdk-for-console',
                 'gitUserName' => 'appwrite',
@@ -203,7 +203,7 @@ return [
             [
                 'key' => 'cli',
                 'name' => 'Command Line',
-                'version' => '4.1.0',
+                'version' => '4.2.0',
                 'url' => 'https://github.com/appwrite/sdk-for-cli',
                 'package' => 'https://www.npmjs.com/package/appwrite-cli',
                 'enabled' => true,
@@ -231,7 +231,7 @@ return [
             [
                 'key' => 'nodejs',
                 'name' => 'Node.js',
-                'version' => '11.0.0',
+                'version' => '11.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-node',
                 'package' => 'https://www.npmjs.com/package/node-appwrite',
                 'enabled' => true,
@@ -249,7 +249,7 @@ return [
             [
                 'key' => 'deno',
                 'name' => 'Deno',
-                'version' => '9.0.0',
+                'version' => '9.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-deno',
                 'package' => 'https://deno.land/x/appwrite',
                 'enabled' => true,
@@ -267,7 +267,7 @@ return [
             [
                 'key' => 'php',
                 'name' => 'PHP',
-                'version' => '10.0.0',
+                'version' => '10.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-php',
                 'package' => 'https://packagist.org/packages/appwrite/appwrite',
                 'enabled' => true,
@@ -285,7 +285,7 @@ return [
             [
                 'key' => 'python',
                 'name' => 'Python',
-                'version' => '4.0.0',
+                'version' => '4.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-python',
                 'package' => 'https://pypi.org/project/appwrite/',
                 'enabled' => true,
@@ -303,7 +303,7 @@ return [
             [
                 'key' => 'ruby',
                 'name' => 'Ruby',
-                'version' => '10.0.0',
+                'version' => '10.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-ruby',
                 'package' => 'https://rubygems.org/gems/appwrite',
                 'enabled' => true,
@@ -321,7 +321,7 @@ return [
             [
                 'key' => 'go',
                 'name' => 'Go',
-                'version' => '3.0.0',
+                'version' => '3.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-go',
                 'package' => '',
                 'enabled' => false,
@@ -339,7 +339,7 @@ return [
             [
                 'key' => 'java',
                 'name' => 'Java',
-                'version' => '3.0.0',
+                'version' => '3.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-java',
                 'package' => '',
                 'enabled' => false,
@@ -357,7 +357,7 @@ return [
             [
                 'key' => 'dotnet',
                 'name' => '.NET',
-                'version' => '0.6.0',
+                'version' => '0.7.0',
                 'url' => 'https://github.com/appwrite/sdk-for-dotnet',
                 'package' => 'https://www.nuget.org/packages/Appwrite',
                 'enabled' => true,
@@ -375,7 +375,7 @@ return [
             [
                 'key' => 'dart',
                 'name' => 'Dart',
-                'version' => '10.0.0',
+                'version' => '10.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-dart',
                 'package' => 'https://pub.dev/packages/dart_appwrite',
                 'enabled' => true,
@@ -393,7 +393,7 @@ return [
             [
                 'key' => 'kotlin',
                 'name' => 'Kotlin',
-                'version' => '4.0.0',
+                'version' => '4.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-kotlin',
                 'package' => 'https://search.maven.org/artifact/io.appwrite/sdk-for-kotlin',
                 'enabled' => true,
@@ -415,7 +415,7 @@ return [
             [
                 'key' => 'swift',
                 'name' => 'Swift',
-                'version' => '4.0.1',
+                'version' => '4.1.0',
                 'url' => 'https://github.com/appwrite/sdk-for-swift',
                 'package' => 'https://github.com/appwrite/sdk-for-swift',
                 'enabled' => true,

app/config/services.php

@@ -170,7 +170,7 @@ return [
         'docs' => false,
         'docsUrl' => '',
         'tests' => false,
-        'optional' => true,
+        'optional' => false,
         'icon' => '',
     ],
     'functions' => [
@@ -196,7 +196,7 @@ return [
         'docs' => true,
         'docsUrl' => 'https://appwrite.io/docs/proxy',
         'tests' => false,
-        'optional' => true,
+        'optional' => false,
         'icon' => '/images/services/proxy.png',
     ],
     'mock' => [
@@ -248,7 +248,7 @@ return [
         'docs' => true,
         'docsUrl' => 'https://appwrite.io/docs/migrations',
         'tests' => true,
-        'optional' => true,
+        'optional' => false,
         'icon' => '/images/services/migrations.png',
     ],
 ];

app/config/variables.php

@@ -337,7 +337,7 @@ return [
     ],
     [
         'category' => 'InfluxDB',
-        'description' => 'Appwrite uses an InfluxDB server for managing time-series data and server stats. The InfluxDB env vars are used to allow Appwrite server to connect to the InfluxDB container.',
+        'description' => 'Deprecated since 1.4.8.',
         'variables' => [
             [
                 'name' => '_APP_INFLUXDB_HOST',
@@ -361,7 +361,7 @@ return [
     ],
     [
         'category' => 'StatsD',
-        'description' => 'Appwrite uses a StatsD server for aggregating and sending stats data over a fast UDP connection. The StatsD env vars are used to allow Appwrite server to connect to the StatsD container.',
+        'description' => 'Deprecated since 1.4.8.',
         'variables' => [
             [
                 'name' => '_APP_STATSD_HOST',

app/controllers/api/account.php

@@ -45,6 +45,7 @@ use Utopia\Validator\WhiteList;
 use Appwrite\Auth\Validator\PasswordHistory;
 use Appwrite\Auth\Validator\PasswordDictionary;
 use Appwrite\Auth\Validator\PersonalData;
+use Appwrite\Hooks\Hooks;
 
 $oauthDefaultSuccess = '/auth/oauth2/success';
 $oauthDefaultFailure = '/auth/oauth2/failure';
@@ -58,7 +59,6 @@ App::post('/v1/account')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'create')
@@ -77,7 +77,8 @@ App::post('/v1/account')
     ->inject('project')
     ->inject('dbForProject')
     ->inject('queueForEvents')
-    ->action(function (string $userId, string $email, string $password, string $name, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents) {
+    ->inject('hooks')
+    ->action(function (string $userId, string $email, string $password, string $name, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks) {
 
         $email = \strtolower($email);
         if ('console' === $project->getId()) {
@@ -118,6 +119,8 @@ App::post('/v1/account')
             }
         }
 
+        $hooks->trigger('passwordValidator', [$project, $password, $user]);
+
         $passwordHistory = $project->getAttribute('auths', [])['passwordHistory'] ?? 0;
         $password = Auth::passwordHash($password, Auth::DEFAULT_ALGO, Auth::DEFAULT_ALGO_OPTIONS);
         try {
@@ -174,8 +177,6 @@ App::post('/v1/account/sessions/email')
     ->label('audits.event', 'session.create')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
-    ->label('usage.metric', 'sessions.{scope}.requests.create')
-    ->label('usage.params', ['provider:email'])
     ->label('sdk.auth', [])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createEmailSession')
@@ -428,8 +429,6 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
     ->label('abuse-limit', 50)
     ->label('abuse-key', 'ip:{ip}')
     ->label('docs', false)
-    ->label('usage.metric', 'sessions.{scope}.requests.create')
-    ->label('usage.params', ['provider:{request.provider}'])
     ->param('provider', '', new WhiteList(\array_keys(Config::getParam('providers')), true), 'OAuth2 provider.')
     ->param('code', '', new Text(2048, 0), 'OAuth2 code. This is a temporary code that the will be later exchanged for an access token.', true)
     ->param('state', '', new Text(2048), 'OAuth2 state params.', true)
@@ -794,7 +793,6 @@ App::get('/v1/account/identities')
     ->desc('List Identities')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'listIdentities')
@@ -849,7 +847,6 @@ App::delete('/v1/account/identities/:identityId')
     ->label('audits.event', 'identity.delete')
     ->label('audits.resource', 'identity/{request.$identityId}')
     ->label('audits.userId', '{user.$id}')
-    ->label('usage.metric', 'identities.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'deleteIdentity')
@@ -1000,7 +997,12 @@ App::post('/v1/account/sessions/magic-url')
         $customTemplate = $project->getAttribute('templates', [])['email.magicSession-' . $locale->default] ?? [];
 
         $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
-        $message->setParam('{{body}}', $body);
+        $message
+            ->setParam('{{body}}', $body)
+            ->setParam('{{hello}}', $locale->getText("emails.magicSession.hello"))
+            ->setParam('{{footer}}', $locale->getText("emails.magicSession.footer"))
+            ->setParam('{{thanks}}', $locale->getText("emails.magicSession.thanks"))
+            ->setParam('{{signature}}', $locale->getText("emails.magicSession.signature"));
         $body = $message->render();
 
         $smtp = $project->getAttribute('smtp', []);
@@ -1050,16 +1052,7 @@ App::post('/v1/account/sessions/magic-url')
         }
 
         $emailVariables = [
-            'subject' => $subject,
-            'hello' => $locale->getText("emails.magicSession.hello"),
-            'body' => $body,
-            'footer' => $locale->getText("emails.magicSession.footer"),
-            'thanks' => $locale->getText("emails.magicSession.thanks"),
-            'signature' => $locale->getText("emails.magicSession.signature"),
             'direction' => $locale->getText('settings.direction'),
-            'bg-body' => '#f7f7f7',
-            'bg-content' => '#ffffff',
-            'text-content' => '#000000',
             /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
             'user' => '',
             'team' => '',
@@ -1098,8 +1091,6 @@ App::put('/v1/account/sessions/magic-url')
     ->label('audits.event', 'session.update')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
-    ->label('usage.metric', 'sessions.{scope}.requests.create')
-    ->label('usage.params', ['provider:magic-url'])
     ->label('sdk.auth', [])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateMagicURLSession')
@@ -1473,8 +1464,6 @@ App::post('/v1/account/sessions/anonymous')
     ->label('audits.event', 'session.create')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
-    ->label('usage.metric', 'sessions.{scope}.requests.create')
-    ->label('usage.params', ['provider:anonymous'])
     ->label('sdk.auth', [])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createAnonymousSession')
@@ -1652,7 +1641,6 @@ App::get('/v1/account')
     ->desc('Get account')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'get')
@@ -1673,7 +1661,6 @@ App::get('/v1/account/prefs')
     ->desc('Get account preferences')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'getPrefs')
@@ -1696,7 +1683,6 @@ App::get('/v1/account/sessions')
     ->desc('List sessions')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'listSessions')
@@ -1735,7 +1721,6 @@ App::get('/v1/account/logs')
     ->desc('List logs')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'listLogs')
@@ -1796,7 +1781,6 @@ App::get('/v1/account/sessions/:sessionId')
     ->desc('Get session')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'getSession')
@@ -1844,7 +1828,6 @@ App::patch('/v1/account/name')
     ->label('scope', 'account')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateName')
@@ -1879,7 +1862,6 @@ App::patch('/v1/account/password')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updatePassword')
@@ -1945,7 +1927,6 @@ App::patch('/v1/account/email')
     ->label('scope', 'account')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateEmail')
@@ -2015,7 +1996,6 @@ App::patch('/v1/account/phone')
     ->label('scope', 'account')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updatePhone')
@@ -2074,7 +2054,6 @@ App::patch('/v1/account/prefs')
     ->label('scope', 'account')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updatePrefs')
@@ -2108,7 +2087,6 @@ App::patch('/v1/account/status')
     ->label('scope', 'account')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateStatus')
@@ -2152,7 +2130,6 @@ App::delete('/v1/account/sessions/:sessionId')
     ->label('event', 'users.[userId].sessions.[sessionId].delete')
     ->label('audits.event', 'session.delete')
     ->label('audits.resource', 'user/{user.$id}')
-    ->label('usage.metric', 'sessions.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'deleteSession')
@@ -2229,7 +2206,6 @@ App::patch('/v1/account/sessions/:sessionId')
     ->label('audits.event', 'session.update')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
-    ->label('usage.metric', 'sessions.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateSession')
@@ -2314,7 +2290,6 @@ App::delete('/v1/account/sessions')
     ->label('event', 'users.[userId].sessions.[sessionId].delete')
     ->label('audits.event', 'session.delete')
     ->label('audits.resource', 'user/{user.$id}')
-    ->label('usage.metric', 'sessions.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'deleteSessions')
@@ -2376,7 +2351,6 @@ App::post('/v1/account/recovery')
     ->label('audits.event', 'recovery.create')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createRecovery')
@@ -2457,7 +2431,12 @@ App::post('/v1/account/recovery')
         $customTemplate = $project->getAttribute('templates', [])['email.recovery-' . $locale->default] ?? [];
 
         $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
-        $message->setParam('{{body}}', $body);
+        $message
+            ->setParam('{{body}}', $body)
+            ->setParam('{{hello}}', $locale->getText("emails.recovery.hello"))
+            ->setParam('{{footer}}', $locale->getText("emails.recovery.footer"))
+            ->setParam('{{thanks}}', $locale->getText("emails.recovery.thanks"))
+            ->setParam('{{signature}}', $locale->getText("emails.recovery.signature"));
         $body = $message->render();
 
         $smtp = $project->getAttribute('smtp', []);
@@ -2507,16 +2486,7 @@ App::post('/v1/account/recovery')
         }
 
         $emailVariables = [
-            'subject' => $subject,
-            'hello' => $locale->getText("emails.recovery.hello"),
-            'body' => $body,
-            'footer' => $locale->getText("emails.recovery.footer"),
-            'thanks' => $locale->getText("emails.recovery.thanks"),
-            'signature' => $locale->getText("emails.recovery.signature"),
             'direction' => $locale->getText('settings.direction'),
-            'bg-body' => '#f7f7f7',
-            'bg-content' => '#ffffff',
-            'text-content' => '#000000',
             /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
             'user' => $profile->getAttribute('name'),
             'team' => '',
@@ -2524,7 +2494,6 @@ App::post('/v1/account/recovery')
             'redirect' => $url
         ];
 
-
         $queueForMails
             ->setRecipient($profile->getAttribute('email', ''))
             ->setName($profile->getAttribute('name'))
@@ -2559,7 +2528,6 @@ App::put('/v1/account/recovery')
     ->label('audits.event', 'recovery.update')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateRecovery')
@@ -2646,7 +2614,6 @@ App::post('/v1/account/verification')
     ->label('event', 'users.[userId].verification.[tokenId].create')
     ->label('audits.event', 'verification.create')
     ->label('audits.resource', 'user/{response.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createVerification')
@@ -2671,6 +2638,10 @@ App::post('/v1/account/verification')
             throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP Disabled');
         }
 
+        if ($user->getAttribute('emailVerification')) {
+            throw new Exception(Exception::USER_EMAIL_ALREADY_VERIFIED);
+        }
+
         $roles = Authorization::getRoles();
         $isPrivilegedUser = Auth::isPrivilegedUser($roles);
         $isAppUser = Auth::isAppUser($roles);
@@ -2709,7 +2680,12 @@ App::post('/v1/account/verification')
         $customTemplate = $project->getAttribute('templates', [])['email.verification-' . $locale->default] ?? [];
 
         $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
-        $message->setParam('{{body}}', $body);
+        $message
+            ->setParam('{{body}}', $body)
+            ->setParam('{{hello}}', $locale->getText("emails.verification.hello"))
+            ->setParam('{{footer}}', $locale->getText("emails.verification.footer"))
+            ->setParam('{{thanks}}', $locale->getText("emails.verification.thanks"))
+            ->setParam('{{signature}}', $locale->getText("emails.verification.signature"));
         $body = $message->render();
 
         $smtp = $project->getAttribute('smtp', []);
@@ -2759,16 +2735,7 @@ App::post('/v1/account/verification')
         }
 
         $emailVariables = [
-            'subject' => $subject,
-            'hello' => $locale->getText("emails.verification.hello"),
-            'body' => $body,
-            'footer' => $locale->getText("emails.verification.footer"),
-            'thanks' => $locale->getText("emails.verification.thanks"),
-            'signature' => $locale->getText("emails.verification.signature"),
             'direction' => $locale->getText('settings.direction'),
-            'bg-body' => '#f7f7f7',
-            'bg-content' => '#ffffff',
-            'text-content' => '#000000',
             /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
             'user' => $user->getAttribute('name'),
             'team' => '',
@@ -2807,7 +2774,6 @@ App::put('/v1/account/verification')
     ->label('event', 'users.[userId].verification.[tokenId].update')
     ->label('audits.event', 'verification.update')
     ->label('audits.resource', 'user/{response.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateVerification')
@@ -2868,7 +2834,6 @@ App::post('/v1/account/verification/phone')
     ->label('event', 'users.[userId].verification.[tokenId].create')
     ->label('audits.event', 'verification.create')
     ->label('audits.resource', 'user/{response.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createPhoneVerification')
@@ -2896,6 +2861,10 @@ App::post('/v1/account/verification/phone')
             throw new Exception(Exception::USER_PHONE_NOT_FOUND);
         }
 
+        if ($user->getAttribute('phoneVerification')) {
+            throw new Exception(Exception::USER_PHONE_ALREADY_VERIFIED);
+        }
+
         $roles = Authorization::getRoles();
         $isPrivilegedUser = Auth::isPrivilegedUser($roles);
         $isAppUser = Auth::isAppUser($roles);
@@ -2964,7 +2933,6 @@ App::put('/v1/account/verification/phone')
     ->label('event', 'users.[userId].verification.[tokenId].update')
     ->label('audits.event', 'verification.update')
     ->label('audits.resource', 'user/{response.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updatePhoneVerification')

app/controllers/api/databases.php

@@ -402,7 +402,6 @@ App::post('/v1/databases')
     ->label('scope', 'databases.write')
     ->label('audits.event', 'database.create')
     ->label('audits.resource', 'database/{response.$id}')
-    ->label('usage.metric', 'databases.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'create')
@@ -476,7 +475,6 @@ App::get('/v1/databases')
     ->desc('List databases')
     ->groups(['api', 'database'])
     ->label('scope', 'databases.read')
-    ->label('usage.metric', 'databases.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'list')
@@ -524,7 +522,6 @@ App::get('/v1/databases/:databaseId')
     ->desc('Get database')
     ->groups(['api', 'database'])
     ->label('scope', 'databases.read')
-    ->label('usage.metric', 'databases.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'get')
@@ -639,7 +636,6 @@ App::put('/v1/databases/:databaseId')
     ->label('event', 'databases.[databaseId].update')
     ->label('audits.event', 'database.update')
     ->label('audits.resource', 'database/{response.$id}')
-    ->label('usage.metric', 'databases.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'update')
@@ -684,7 +680,6 @@ App::delete('/v1/databases/:databaseId')
     ->label('event', 'databases.[databaseId].delete')
     ->label('audits.event', 'database.delete')
     ->label('audits.resource', 'database/{request.databaseId}')
-    ->label('usage.metric', 'databases.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'delete')
@@ -729,8 +724,6 @@ App::post('/v1/databases/:databaseId/collections')
     ->label('scope', 'collections.write')
     ->label('audits.event', 'collection.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{response.$id}')
-    ->label('usage.metric', 'collections.{scope}.requests.create')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'createCollection')
@@ -796,8 +789,6 @@ App::get('/v1/databases/:databaseId/collections')
     ->desc('List collections')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
-    ->label('usage.metric', 'collections.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listCollections')
@@ -855,8 +846,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId')
     ->desc('Get collection')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
-    ->label('usage.metric', 'collections.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'getCollection')
@@ -891,8 +880,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs')
     ->desc('List collection logs')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
-    ->label('usage.metric', 'collections.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listCollectionLogs')
@@ -990,8 +977,6 @@ App::put('/v1/databases/:databaseId/collections/:collectionId')
     ->label('event', 'databases.[databaseId].collections.[collectionId].update')
     ->label('audits.event', 'collection.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateCollection')
@@ -1060,8 +1045,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId')
     ->label('event', 'databases.[databaseId].collections.[collectionId].delete')
     ->label('audits.event', 'collection.delete')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.delete')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'deleteCollection')
@@ -1117,8 +1100,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/string
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'createStringAttribute')
@@ -1175,8 +1156,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/email'
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createEmailAttribute')
@@ -1219,8 +1198,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createEnumAttribute')
@@ -1231,7 +1208,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
     ->param('databaseId', '', new UID(), 'Database ID.')
     ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).')
     ->param('key', '', new Key(), 'Attribute Key.')
-    ->param('elements', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE, min: 0), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.')
+    ->param('elements', [], new ArrayList(new Text(DATABASE::LENGTH_KEY), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . DATABASE::LENGTH_KEY . ' characters long.')
     ->param('required', null, new Boolean(), 'Is attribute required?')
     ->param('default', null, new Text(0), 'Default value for attribute when not provided. Cannot be set when attribute is required.', true)
     ->param('array', false, new Boolean(), 'Is attribute an array?', true)
@@ -1240,16 +1217,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
     ->inject('queueForDatabase')
     ->inject('queueForEvents')
     ->action(function (string $databaseId, string $collectionId, string $key, array $elements, ?bool $required, ?string $default, bool $array, Response $response, Database $dbForProject, EventDatabase $queueForDatabase, Event $queueForEvents) {
-        // use length of longest string as attribute size
-        $size = 0;
-        foreach ($elements as $element) {
-            $length = \strlen($element);
-            if ($length === 0) {
-                throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Each enum element must not be empty');
-            }
-            $size = ($length > $size) ? $length : $size;
-        }
-
         if (!is_null($default) && !in_array($default, $elements)) {
             throw new Exception(Exception::ATTRIBUTE_VALUE_INVALID, 'Default value not found in elements');
         }
@@ -1257,7 +1224,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
         $attribute = createAttribute($databaseId, $collectionId, new Document([
             'key' => $key,
             'type' => Database::VAR_STRING,
-            'size' => $size,
+            'size' => Database::LENGTH_KEY,
             'required' => $required,
             'default' => $default,
             'array' => $array,
@@ -1278,8 +1245,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/ip')
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createIpAttribute')
@@ -1322,8 +1287,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/url')
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createUrlAttribute')
@@ -1366,8 +1329,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/intege
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createIntegerAttribute')
@@ -1439,8 +1400,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/float'
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createFloatAttribute')
@@ -1515,8 +1474,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/boolea
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createBooleanAttribute')
@@ -1558,8 +1515,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/dateti
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createDatetimeAttribute')
@@ -1604,8 +1559,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/relati
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createRelationshipAttribute')
@@ -1683,8 +1636,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes')
     ->desc('List attributes')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
-    ->label('usage.metric', 'collections.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listAttributes')
@@ -1758,8 +1709,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes/:key')
     ->desc('Get attribute')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
-    ->label('usage.metric', 'collections.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'getAttribute')
@@ -1837,8 +1786,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/strin
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateStringAttribute')
@@ -1878,8 +1825,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/email
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateEmailAttribute')
@@ -1919,8 +1864,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/enum/
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateEnumAttribute')
@@ -1930,7 +1873,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/enum/
     ->param('databaseId', '', new UID(), 'Database ID.')
     ->param('collectionId', '', new UID(), 'Collection ID. You can create a new collection using the Database service [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection).')
     ->param('key', '', new Key(), 'Attribute Key.')
-    ->param('elements', null, new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.')
+    ->param('elements', null, new ArrayList(new Text(DATABASE::LENGTH_KEY), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of elements in enumerated type. Uses length of longest element to determine size. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' elements are allowed, each ' . DATABASE::LENGTH_KEY . ' characters long.')
     ->param('required', null, new Boolean(), 'Is attribute required?')
     ->param('default', null, new Nullable(new Text(0)), 'Default value for attribute when not provided. Cannot be set when attribute is required.')
     ->inject('response')
@@ -1962,8 +1905,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/ip/:k
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateIpAttribute')
@@ -2003,8 +1944,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/url/:
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateUrlAttribute')
@@ -2044,8 +1983,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/integ
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateIntegerAttribute')
@@ -2095,8 +2032,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/float
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateFloatAttribute')
@@ -2146,8 +2081,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/boole
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateBooleanAttribute')
@@ -2186,8 +2119,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/datet
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'documents.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateDatetimeAttribute')
@@ -2226,8 +2157,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/:key/
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'documents.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateRelationshipAttribute')
@@ -2282,8 +2211,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.delete')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'deleteAttribute')
@@ -2393,8 +2320,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes')
     ->label('scope', 'collections.write')
     ->label('audits.event', 'index.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'createIndex')
@@ -2553,8 +2478,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes')
     ->desc('List indexes')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
-    ->label('usage.metric', 'collections.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listIndexes')
@@ -2618,8 +2541,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes/:key')
     ->desc('Get index')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
-    ->label('usage.metric', 'collections.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'getIndex')
@@ -2662,8 +2583,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/indexes/:key')
     ->label('event', 'databases.[databaseId].collections.[collectionId].indexes.[indexId].update')
     ->label('audits.event', 'index.delete')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'collections.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'deleteIndex')
@@ -2728,8 +2647,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents')
     ->label('scope', 'documents.write')
     ->label('audits.event', 'document.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
-    ->label('usage.metric', 'documents.{scope}.requests.create')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -2966,8 +2883,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents')
     ->desc('List documents')
     ->groups(['api', 'database'])
     ->label('scope', 'documents.read')
-    ->label('usage.metric', 'documents.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listDocuments')
@@ -3093,8 +3008,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
     ->desc('Get document')
     ->groups(['api', 'database'])
     ->label('scope', 'documents.read')
-    ->label('usage.metric', 'documents.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'getDocument')
@@ -3188,8 +3101,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
     ->desc('List document logs')
     ->groups(['api', 'database'])
     ->label('scope', 'documents.read')
-    ->label('usage.metric', 'documents.{scope}.requests.read')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listDocumentLogs')
@@ -3292,8 +3203,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum
     ->label('scope', 'documents.write')
     ->label('audits.event', 'document.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}')
-    ->label('usage.metric', 'documents.{scope}.requests.update')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -3522,8 +3431,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
     ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].delete')
     ->label('audits.event', 'document.delete')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{request.documentId}')
-    ->label('usage.metric', 'documents.{scope}.requests.delete')
-    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -3632,7 +3539,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
 
 App::get('/v1/databases/usage')
     ->desc('Get usage stats for the database')
-    ->groups(['api', 'database'])
+    ->groups(['api', 'database', 'usage'])
     ->label('scope', 'collections.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'databases')
@@ -3640,117 +3547,77 @@ App::get('/v1/databases/usage')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_DATABASES)
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), '`Date range.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), '`Date range.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (string $range, Response $response, Database $dbForProject) {
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            METRIC_DATABASES,
+            METRIC_COLLECTIONS,
+            METRIC_DOCUMENTS,
+        ];
 
-            $metrics = [
-                'databases.$all.count.total',
-                'documents.$all.count.total',
-                'collections.$all.count.total',
-                'databases.$all.requests.create',
-                'databases.$all.requests.read',
-                'databases.$all.requests.update',
-                'databases.$all.requests.delete',
-                'collections.$all.requests.create',
-                'collections.$all.requests.read',
-                'collections.$all.requests.update',
-                'collections.$all.requests.delete',
-                'documents.$all.requests.create',
-                'documents.$all.requests.read',
-                'documents.$all.requests.update',
-                'documents.$all.requests.delete'
-            ];
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
+            foreach ($metrics as $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
 
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    // Added 3'rd level to Index [period, metric, time] because of order by.
-                    $stats[$metric] = array_reverse($stats[$metric]);
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'databasesCount' => $stats['databases.$all.count.total'] ?? [],
-                'documentsCount' => $stats['documents.$all.count.total'] ?? [],
-                'collectionsCount' => $stats['collections.$all.count.total'] ?? [],
-                'documentsCreate' =>  $stats['documents.$all.requests.create'] ?? [],
-                'documentsRead' =>  $stats['documents.$all.requests.read'] ?? [],
-                'documentsUpdate' => $stats['documents.$all.requests.update'] ?? [],
-                'documentsDelete' => $stats['documents.$all.requests.delete'] ?? [],
-                'collectionsCreate' => $stats['collections.$all.requests.create'] ?? [],
-                'collectionsRead' =>  $stats['collections.$all.requests.read'] ?? [],
-                'collectionsUpdate' => $stats['collections.$all.requests.update'] ?? [],
-                'collectionsDelete' => $stats['collections.$all.requests.delete'] ?? [],
-                'databasesCreate' => $stats['databases.$all.requests.create'] ?? [],
-                'databasesRead' =>  $stats['databases.$all.requests.read'] ?? [],
-                'databasesUpdate' => $stats['databases.$all.requests.update'] ?? [],
-                'databasesDelete' => $stats['databases.$all.requests.delete'] ?? [],
-            ]);
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
         }
-
-        $response->dynamic($usage, Response::MODEL_USAGE_DATABASES);
+    }
+        $response->dynamic(new Document([
+            'range' => $range,
+            'databasesTotal'   => $usage[$metrics[0]]['total'],
+            'collectionsTotal' => $usage[$metrics[1]]['total'],
+            'documentsTotal'   => $usage[$metrics[2]]['total'],
+            'databases'   => $usage[$metrics[0]]['data'],
+            'collections' => $usage[$metrics[1]]['data'],
+            'documents'   => $usage[$metrics[2]]['data'],
+        ]), Response::MODEL_USAGE_DATABASES);
     });
 
 App::get('/v1/databases/:databaseId/usage')
     ->desc('Get usage stats for the database')
-    ->groups(['api', 'database'])
+    ->groups(['api', 'database', 'usage'])
     ->label('scope', 'collections.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'databases')
@@ -3759,108 +3626,82 @@ App::get('/v1/databases/:databaseId/usage')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_DATABASE)
     ->param('databaseId', '', new UID(), 'Database ID.')
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), '`Date range.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), '`Date range.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (string $databaseId, string $range, Response $response, Database $dbForProject) {
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $database =  $dbForProject->getDocument('databases', $databaseId);
 
-            $metrics = [
-                'collections.' . $databaseId . '.count.total',
-                'collections.' . $databaseId . '.requests.create',
-                'collections.' . $databaseId . '.requests.read',
-                'collections.' . $databaseId . '.requests.update',
-                'collections.' . $databaseId . '.requests.delete',
-                'documents.' . $databaseId . '.count.total',
-                'documents.' . $databaseId . '.requests.create',
-                'documents.' . $databaseId . '.requests.read',
-                'documents.' . $databaseId . '.requests.update',
-                'documents.' . $databaseId . '.requests.delete'
-            ];
-
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    // TODO@kodumbeats explore performance if query is ordered by time ASC
-                    $stats[$metric] = array_reverse($stats[$metric]);
-                }
-            });
-
-            $usage = new Document([
-                'range' => $range,
-                'collectionsCount' => $stats["collections.{$databaseId}.count.total"] ?? [],
-                'collectionsCreate' => $stats["collections.{$databaseId}.requests.create"] ?? [],
-                'collectionsRead' =>  $stats["collections.{$databaseId}.requests.read"] ?? [],
-                'collectionsUpdate' => $stats["collections.{$databaseId}.requests.update"] ?? [],
-                'collectionsDelete' => $stats["collections.{$databaseId}.requests.delete"] ?? [],
-                'documentsCount' => $stats["documents.{$databaseId}.count.total"] ?? [],
-                'documentsCreate' =>  $stats["documents.{$databaseId}.requests.create"] ?? [],
-                'documentsRead' =>  $stats["documents.{$databaseId}.requests.read"] ?? [],
-                'documentsUpdate' => $stats["documents.{$databaseId}.requests.update"] ?? [],
-                'documentsDelete' => $stats["documents.{$databaseId}.requests.delete"] ?? [],
-            ]);
+        if ($database->isEmpty()) {
+            throw new Exception(Exception::DATABASE_NOT_FOUND);
         }
 
-        $response->dynamic($usage, Response::MODEL_USAGE_DATABASE);
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS),
+            str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS),
+        ];
+
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
+            foreach ($metrics as $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
+
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
+                }
+            }
+        });
+
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
+        }
+    }
+
+        $response->dynamic(new Document([
+            'range' => $range,
+            'collectionsTotal'   => $usage[$metrics[0]]['total'],
+            'documentsTotal'   => $usage[$metrics[1]]['total'],
+            'collections'   => $usage[$metrics[0]]['data'],
+            'documents'   => $usage[$metrics[1]]['data'],
+        ]), Response::MODEL_USAGE_DATABASE);
     });
 
 App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
     ->alias('/v1/database/:collectionId/usage', ['databaseId' => 'default'])
     ->desc('Get usage stats for a collection')
-    ->groups(['api', 'database'])
+    ->groups(['api', 'database', 'usage'])
     ->label('scope', 'collections.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'databases')
@@ -3869,7 +3710,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_COLLECTION)
     ->param('databaseId', '', new UID(), 'Database ID.')
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true)
     ->param('collectionId', '', new UID(), 'Collection ID.')
     ->inject('response')
     ->inject('dbForProject')
@@ -3883,84 +3724,60 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
             throw new Exception(Exception::COLLECTION_NOT_FOUND);
         }
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collectionDocument->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS),
+        ];
 
-            $metrics = [
-                "documents.{$databaseId}/{$collectionId}.count.total",
-                "documents.{$databaseId}/{$collectionId}.requests.create",
-                "documents.{$databaseId}/{$collectionId}.requests.read",
-                "documents.{$databaseId}/{$collectionId}.requests.update",
-                "documents.{$databaseId}/{$collectionId}.requests.delete",
-            ];
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
+            foreach ($metrics as $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
 
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' =>  $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'documentsCount' => $stats["documents.{$databaseId}/{$collectionId}.count.total"] ?? [],
-                'documentsCreate' => $stats["documents.{$databaseId}/{$collectionId}.requests.create"] ?? [],
-                'documentsRead' => $stats["documents.{$databaseId}/{$collectionId}.requests.read"] ?? [],
-                'documentsUpdate' =>  $stats["documents.{$databaseId}/{$collectionId}.requests.update"] ?? [],
-                'documentsDelete' =>  $stats["documents.{$databaseId}/{$collectionId}.requests.delete" ?? []]
-            ]);
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
         }
+    }
 
-        $response->dynamic($usage, Response::MODEL_USAGE_COLLECTION);
+        $response->dynamic(new Document([
+            'range' => $range,
+            'documentsTotal'   => $usage[$metrics[0]]['total'],
+            'documents'   =>  $usage[$metrics[0]]['data'],
+        ]), Response::MODEL_USAGE_COLLECTION);
     });

app/controllers/api/functions.php

@@ -6,13 +6,13 @@ use Appwrite\Event\Build;
 use Appwrite\Event\Delete;
 use Appwrite\Event\Event;
 use Appwrite\Event\Func;
+use Appwrite\Event\Usage;
 use Appwrite\Event\Validator\FunctionEvent;
 use Appwrite\Utopia\Response\Model\Rule;
 use Appwrite\Extend\Exception;
 use Appwrite\Utopia\Database\Validator\CustomId;
 use Appwrite\Messaging\Adapter\Realtime;
 use Utopia\Validator\Assoc;
-use Appwrite\Usage\Stats;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Helpers\Permission;
 use Utopia\Database\Helpers\Role;
@@ -46,6 +46,7 @@ use Utopia\Validator\Boolean;
 use Utopia\Database\Exception\Duplicate as DuplicateException;
 use MaxMind\Db\Reader;
 use Utopia\VCS\Adapter\Git\GitHub;
+use Utopia\VCS\Exception\RepositoryNotFound;
 
 include_once __DIR__ . '/../shared/api.php';
 
@@ -58,7 +59,14 @@ $redeployVcs = function (Request $request, Document $function, Document $project
     $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId);
     $owner = $github->getOwnerName($providerInstallationId);
     $providerRepositoryId = $function->getAttribute('providerRepositoryId', '');
-    $repositoryName = $github->getRepositoryName($providerRepositoryId);
+    try {
+        $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? '';
+        if (empty($repositoryName)) {
+            throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
+        }
+    } catch (RepositoryNotFound $e) {
+        throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
+    }
     $providerBranch = $function->getAttribute('providerBranch', 'main');
     $authorUrl = "https://github.com/$owner";
     $repositoryUrl = "https://github.com/$owner/$repositoryName";
@@ -234,12 +242,16 @@ App::post('/v1/functions')
 
         // Git connect logic
         if (!empty($providerRepositoryId)) {
+            $teamId = $project->getAttribute('teamId', '');
+
             $repository = $dbForConsole->createDocument('repositories', new Document([
                 '$id' => ID::unique(),
                 '$permissions' => [
-                    Permission::read(Role::any()),
-                    Permission::update(Role::any()),
-                    Permission::delete(Role::any()),
+                    Permission::read(Role::team(ID::custom($teamId))),
+                    Permission::update(Role::team(ID::custom($teamId), 'owner')),
+                    Permission::update(Role::team(ID::custom($teamId), 'developer')),
+                    Permission::delete(Role::team(ID::custom($teamId), 'owner')),
+                    Permission::delete(Role::team(ID::custom($teamId), 'developer')),
                 ],
                 'installationId' => $installation->getId(),
                 'installationInternalId' => $installation->getInternalId(),
@@ -287,9 +299,8 @@ App::post('/v1/functions')
             $ruleModel = new Rule();
             $ruleCreate =
                 $queueForEvents
-                     ->setClass(Event::WEBHOOK_CLASS_NAME)
-                     ->setQueue(Event::WEBHOOK_QUEUE_NAME)
-                ;
+                ->setClass(Event::WEBHOOK_CLASS_NAME)
+                ->setQueue(Event::WEBHOOK_QUEUE_NAME);
 
             $ruleCreate
                 ->setProject($project)
@@ -447,7 +458,7 @@ App::get('/v1/functions/:functionId/usage')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS)
     ->param('functionId', '', new UID(), 'Function ID.')
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d']), 'Date range.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (string $functionId, string $range, Response $response, Database $dbForProject) {
@@ -458,97 +469,85 @@ App::get('/v1/functions/:functionId/usage')
             throw new Exception(Exception::FUNCTION_NOT_FOUND);
         }
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS),
+            str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $function->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE),
+            str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS),
+            str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE),
+            str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE),
+            str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS),
+            str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE),
+        ];
 
-            $metrics = [
-                "executions.$functionId.compute.total",
-                "executions.$functionId.compute.success",
-                "executions.$functionId.compute.failure",
-                "executions.$functionId.compute.time",
-                "builds.$functionId.compute.total",
-                "builds.$functionId.compute.success",
-                "builds.$functionId.compute.failure",
-                "builds.$functionId.compute.time",
-            ];
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
+            foreach ($metrics as $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
 
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'executionsTotal' => $stats["executions.$functionId.compute.total"] ?? [],
-                'executionsFailure' => $stats["executions.$functionId.compute.failure"] ?? [],
-                'executionsSuccess' => $stats["executions.$functionId.compute.success"] ?? [],
-                'executionsTime' => $stats["executions.$functionId.compute.time"] ?? [],
-                'buildsTotal' => $stats["builds.$functionId.compute.total"] ?? [],
-                'buildsFailure' => $stats["builds.$functionId.compute.failure"] ?? [],
-                'buildsSuccess' => $stats["builds.$functionId.compute.success"] ?? [],
-                'buildsTime' => $stats["builds.$functionId.compute.time" ?? []]
-            ]);
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
         }
+    }
 
-        $response->dynamic($usage, Response::MODEL_USAGE_FUNCTION);
+        $response->dynamic(new Document([
+            'range' => $range,
+            'deploymentsTotal' => $usage[$metrics[0]]['total'],
+            'deploymentsStorageTotal' => $usage[$metrics[1]]['total'],
+            'buildsTotal' => $usage[$metrics[2]]['total'],
+            'buildsStorageTotal' => $usage[$metrics[3]]['total'],
+            'buildsTimeTotal' => $usage[$metrics[4]]['total'],
+            'executionsTotal' => $usage[$metrics[5]]['total'],
+            'executionsTimeTotal' => $usage[$metrics[6]]['total'],
+            'deployments' => $usage[$metrics[0]]['data'],
+            'deploymentsStorage' => $usage[$metrics[1]]['data'],
+            'builds' => $usage[$metrics[2]]['data'],
+            'buildsStorage' => $usage[$metrics[3]]['data'],
+            'buildsTime' => $usage[$metrics[4]]['data'],
+            'executions' => $usage[$metrics[5]]['data'],
+            'executionsTime' => $usage[$metrics[6]]['data'],
+        ]), Response::MODEL_USAGE_FUNCTION);
     });
 
 App::get('/v1/functions/usage')
     ->desc('Get functions usage')
-    ->groups(['api', 'functions', 'usage'])
+    ->groups(['api', 'functions'])
     ->label('scope', 'functions.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'functions')
@@ -556,97 +555,87 @@ App::get('/v1/functions/usage')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS)
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d']), 'Date range.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (string $range, Response $response, Database $dbForProject) {
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            METRIC_FUNCTIONS,
+            METRIC_DEPLOYMENTS,
+            METRIC_DEPLOYMENTS_STORAGE,
+            METRIC_BUILDS,
+            METRIC_BUILDS_STORAGE,
+            METRIC_BUILDS_COMPUTE,
+            METRIC_EXECUTIONS,
+            METRIC_EXECUTIONS_COMPUTE,
+        ];
 
-            $metrics = [
-                'executions.$all.compute.total',
-                'executions.$all.compute.failure',
-                'executions.$all.compute.success',
-                'executions.$all.compute.time',
-                'builds.$all.compute.total',
-                'builds.$all.compute.failure',
-                'builds.$all.compute.success',
-                'builds.$all.compute.time',
-            ];
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
+            foreach ($metrics as $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
 
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'executionsTotal' => $stats[$metrics[0]] ?? [],
-                'executionsFailure' => $stats[$metrics[1]] ?? [],
-                'executionsSuccess' => $stats[$metrics[2]] ?? [],
-                'executionsTime' => $stats[$metrics[3]] ?? [],
-                'buildsTotal' => $stats[$metrics[4]] ?? [],
-                'buildsFailure' => $stats[$metrics[5]] ?? [],
-                'buildsSuccess' => $stats[$metrics[6]] ?? [],
-                'buildsTime' => $stats[$metrics[7]] ?? [],
-            ]);
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
         }
-
-        $response->dynamic($usage, Response::MODEL_USAGE_FUNCTIONS);
+    }
+        $response->dynamic(new Document([
+            'range' => $range,
+            'functionsTotal' => $usage[$metrics[0]]['total'],
+            'deploymentsTotal' => $usage[$metrics[1]]['total'],
+            'deploymentsStorageTotal' => $usage[$metrics[2]]['total'],
+            'buildsTotal' => $usage[$metrics[3]]['total'],
+            'buildsStorageTotal' => $usage[$metrics[4]]['total'],
+            'buildsTimeTotal' => $usage[$metrics[5]]['total'],
+            'executionsTotal' => $usage[$metrics[6]]['total'],
+            'executionsTimeTotal' => $usage[$metrics[7]]['total'],
+            'functions' => $usage[$metrics[0]]['data'],
+            'deployments' => $usage[$metrics[1]]['data'],
+            'deploymentsStorage' => $usage[$metrics[2]]['data'],
+            'builds' => $usage[$metrics[3]]['data'],
+            'buildsStorage' => $usage[$metrics[4]]['data'],
+            'buildsTime' => $usage[$metrics[5]]['data'],
+            'executions' => $usage[$metrics[6]]['data'],
+            'executionsTime' => $usage[$metrics[7]]['data'],
+        ]), Response::MODEL_USAGE_FUNCTIONS);
     });
 
 App::put('/v1/functions/:functionId')
@@ -870,8 +859,7 @@ App::get('/v1/functions/:functionId/deployments/:deploymentId/download')
             ->setContentType('application/gzip')
             ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache
             ->addHeader('X-Peak', \memory_get_peak_usage())
-            ->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '.tar.gz"')
-        ;
+            ->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '.tar.gz"');
 
         $size = $deviceFunctions->getFileSize($path);
         $rangeHeader = $request->getHeader('range');
@@ -899,7 +887,6 @@ App::get('/v1/functions/:functionId/deployments/:deploymentId/download')
         }
 
         if ($size > APP_STORAGE_READ_BUFFER) {
-            $response->addHeader('Content-Length', $deviceFunctions->getFileSize($path));
             for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) {
                 $response->chunk(
                     $deviceFunctions->read(
@@ -1511,11 +1498,11 @@ App::post('/v1/functions/:functionId/executions')
     ->inject('dbForProject')
     ->inject('user')
     ->inject('queueForEvents')
-    ->inject('usage')
+    ->inject('queueForUsage')
     ->inject('mode')
     ->inject('queueForFunctions')
     ->inject('geodb')
-    ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $queueForEvents, Stats $usage, string $mode, Func $queueForFunctions, Reader $geodb) {
+    ->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, Response $response, Document $project, Database $dbForProject, Document $user, Event $queueForEvents, Usage $queueForUsage, string $mode, Func $queueForFunctions, Reader $geodb) {
 
         $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId));
 
@@ -1653,6 +1640,8 @@ App::post('/v1/functions/:functionId/executions')
                 ->setJWT($jwt)
                 ->setProject($project)
                 ->setUser($user)
+                ->setParam('functionId', $function->getId())
+                ->setParam('executionId', $execution->getId())
                 ->trigger();
 
             return $response
@@ -1741,6 +1730,13 @@ App::post('/v1/functions/:functionId/executions')
                 ->setAttribute('responseStatusCode', 500)
                 ->setAttribute('errors', $th->getMessage() . '\nError Code: ' . $th->getCode());
             Console::error($th->getMessage());
+        } finally {
+            $queueForUsage
+                ->addMetric(METRIC_EXECUTIONS, 1)
+                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
+                ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project
+                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function
+            ;
         }
 
         if ($function->getAttribute('logging')) {
@@ -1748,14 +1744,6 @@ App::post('/v1/functions/:functionId/executions')
             $execution = Authorization::skip(fn () => $dbForProject->createDocument('executions', $execution));
         }
 
-        // TODO revise this later using route label
-        $usage
-            ->setParam('functionId', $function->getId())
-            ->setParam('executions.{scope}.compute', 1)
-            ->setParam('executionStatus', $execution->getAttribute('status', ''))
-            ->setParam('executionTime', $execution->getAttribute('duration')); // ms
-
-
         $roles = Authorization::getRoles();
         $isPrivilegedUser = Auth::isPrivilegedUser($roles);
         $isAppUser = Auth::isAppUser($roles);

app/controllers/api/health.php

@@ -14,6 +14,7 @@ use Utopia\Registry\Registry;
 use Utopia\Storage\Device;
 use Utopia\Storage\Device\Local;
 use Utopia\Storage\Storage;
+use Utopia\Validator\Integer;
 use Utopia\Validator\Text;
 
 App::get('/v1/health')
@@ -87,22 +88,18 @@ App::get('/v1/health/db')
                             'ping' => \round((\microtime(true) - $checkStart) / 1000)
                         ]);
                     } else {
-                        $output[] = new Document([
-                            'name' => $key . " ($database)",
-                            'status' => 'fail',
-                            'ping' => \round((\microtime(true) - $checkStart) / 1000)
-                        ]);
+                        $failure[] = $database;
                     }
                 } catch (\Throwable $th) {
-                    $output[] = new Document([
-                        'name' => $key . " ($database)",
-                        'status' => 'fail',
-                        'ping' => \round((\microtime(true) - $checkStart) / 1000)
-                    ]);
+                    $failure[] = $database;
                 }
             }
         }
 
+        if (!empty($failure)) {
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'DB failure on: ' . implode(", ", $failure));
+        }
+
         $response->dynamic(new Document([
             'statuses' => $output,
             'total' => count($output),
@@ -348,11 +345,20 @@ App::get('/v1/health/queue/webhooks')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::WEBHOOK_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/logs')
@@ -366,11 +372,20 @@ App::get('/v1/health/queue/logs')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::AUDITS_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/certificates')
@@ -384,11 +399,20 @@ App::get('/v1/health/queue/certificates')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::CERTIFICATES_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/builds')
@@ -402,11 +426,20 @@ App::get('/v1/health/queue/builds')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::BUILDS_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/databases')
@@ -421,11 +454,20 @@ App::get('/v1/health/queue/databases')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
     ->param('name', 'database_db_main', new Text(256), 'Queue name for which to check the queue size', true)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (string $name, Connection $queue, Response $response) {
+    ->action(function (string $name, int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client($name, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/deletes')
@@ -439,11 +481,20 @@ App::get('/v1/health/queue/deletes')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::DELETE_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/mails')
@@ -457,11 +508,20 @@ App::get('/v1/health/queue/mails')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::MAILS_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/messaging')
@@ -475,11 +535,20 @@ App::get('/v1/health/queue/messaging')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::MESSAGING_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/migrations')
@@ -493,11 +562,20 @@ App::get('/v1/health/queue/migrations')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::MIGRATIONS_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/queue/functions')
@@ -511,11 +589,20 @@ App::get('/v1/health/queue/functions')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
+    ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
     ->inject('queue')
     ->inject('response')
-    ->action(function (Connection $queue, Response $response) {
+    ->action(function (int|string $threshold, Connection $queue, Response $response) {
+        $threshold = \intval($threshold);
+
         $client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queue);
-        $response->dynamic(new Document([ 'size' => $client->sumProcessingJobs() ]), Response::MODEL_HEALTH_QUEUE);
+        $size = $client->getQueueSize();
+
+        if ($size >= $threshold) {
+            throw new Exception(Exception::QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
+        }
+
+        $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
     }, ['response']);
 
 App::get('/v1/health/storage/local')
@@ -536,10 +623,10 @@ App::get('/v1/health/storage/local')
 
         foreach (
             [
-            'Uploads' => APP_STORAGE_UPLOADS,
-            'Cache' => APP_STORAGE_CACHE,
-            'Config' => APP_STORAGE_CONFIG,
-            'Certs' => APP_STORAGE_CERTIFICATES
+                'Uploads' => APP_STORAGE_UPLOADS,
+                'Cache' => APP_STORAGE_CACHE,
+                'Config' => APP_STORAGE_CONFIG,
+                'Certs' => APP_STORAGE_CERTIFICATES
             ] as $key => $volume
         ) {
             $device = new Local($volume);
@@ -601,7 +688,7 @@ App::get('/v1/health/anti-virus')
     });
 
 App::get('/v1/health/stats') // Currently only used internally
-    ->desc('Get system stats')
+->desc('Get system stats')
     ->groups(['api', 'health'])
     ->label('scope', 'root')
     // ->label('sdk.auth', [APP_AUTH_TYPE_KEY])

app/controllers/api/migrations.php

@@ -208,6 +208,16 @@ App::post('/v1/migrations/firebase')
     ->inject('queueForEvents')
     ->inject('queueForMigrations')
     ->action(function (array $resources, string $serviceAccount, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) {
+        $serviceAccountData = json_decode($serviceAccount, true);
+
+        if (empty($serviceAccountData)) {
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Invalid Service Account JSON');
+        }
+
+        if (!isset($serviceAccountData['project_id']) || !isset($serviceAccountData['client_email']) || !isset($serviceAccountData['private_key'])) {
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Invalid Service Account JSON');
+        }
+
         $migration = $dbForProject->createDocument('migrations', new Document([
             '$id' => ID::unique(),
             'status' => 'pending',
@@ -449,15 +459,26 @@ App::get('/v1/migrations/appwrite/report')
     ->inject('project')
     ->inject('user')
     ->action(function (array $resources, string $endpoint, string $projectID, string $key, Response $response) {
-        try {
-            $appwrite = new Appwrite($projectID, $endpoint, $key);
+        $appwrite = new Appwrite($projectID, $endpoint, $key);
 
-            $response
-                ->setStatusCode(Response::STATUS_CODE_OK)
-                ->dynamic(new Document($appwrite->report($resources)), Response::MODEL_MIGRATION_REPORT);
+        try {
+            $report = $appwrite->report($resources);
         } catch (\Throwable $e) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage());
+            switch ($e->getCode()) {
+                case 401:
+                    throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, 'Source Error: ' . $e->getMessage());
+                case 429:
+                    throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Source Error: Rate Limit Exceeded, Is your Cloud Provider blocking Appwrite\'s IP?');
+                case 500:
+                    throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
+            }
+
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
         }
+
+        $response
+            ->setStatusCode(Response::STATUS_CODE_OK)
+            ->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT);
     });
 
 App::get('/v1/migrations/firebase/report')
@@ -475,15 +496,36 @@ App::get('/v1/migrations/firebase/report')
     ->param('serviceAccount', '', new Text(65536), 'JSON of the Firebase service account credentials')
     ->inject('response')
     ->action(function (array $resources, string $serviceAccount, Response $response) {
-        try {
-            $firebase = new Firebase(json_decode($serviceAccount, true));
+        $serviceAccount = json_decode($serviceAccount, true);
 
-            $response
-                ->setStatusCode(Response::STATUS_CODE_OK)
-                ->dynamic(new Document($firebase->report($resources)), Response::MODEL_MIGRATION_REPORT);
-        } catch (\Exception $e) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage());
+        if (empty($serviceAccount)) {
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Invalid Service Account JSON');
         }
+
+        if (!isset($serviceAccount['project_id']) || !isset($serviceAccount['client_email']) || !isset($serviceAccount['private_key'])) {
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Invalid Service Account JSON');
+        }
+
+        $firebase = new Firebase($serviceAccount);
+
+        try {
+            $report = $firebase->report($resources);
+        } catch (\Throwable $e) {
+            switch ($e->getCode()) {
+                case 401:
+                    throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, 'Source Error: ' . $e->getMessage());
+                case 429:
+                    throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Source Error: Rate Limit Exceeded, Is your Cloud Provider blocking Appwrite\'s IP?');
+                case 500:
+                    throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
+            }
+
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
+        }
+
+        $response
+            ->setStatusCode(Response::STATUS_CODE_OK)
+            ->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT);
     });
 
 App::get('/v1/migrations/firebase/report/oauth')
@@ -869,15 +911,26 @@ App::get('/v1/migrations/supabase/report')
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (array $resources, string $endpoint, string $apiKey, string $databaseHost, string $username, string $password, int $port, Response $response) {
-        try {
-            $supabase = new Supabase($endpoint, $apiKey, $databaseHost, 'postgres', $username, $password, $port);
+        $supabase = new Supabase($endpoint, $apiKey, $databaseHost, 'postgres', $username, $password, $port);
 
-            $response
-                ->setStatusCode(Response::STATUS_CODE_OK)
-                ->dynamic(new Document($supabase->report($resources)), Response::MODEL_MIGRATION_REPORT);
-        } catch (\Exception $e) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage());
+        try {
+            $report = $supabase->report($resources);
+        } catch (\Throwable $e) {
+            switch ($e->getCode()) {
+                case 401:
+                    throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, 'Source Error: ' . $e->getMessage());
+                case 429:
+                    throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Source Error: Rate Limit Exceeded, Is your Cloud Provider blocking Appwrite\'s IP?');
+                case 500:
+                    throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
+            }
+
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
         }
+
+        $response
+            ->setStatusCode(Response::STATUS_CODE_OK)
+            ->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT);
     });
 
 App::get('/v1/migrations/nhost/report')
@@ -901,15 +954,26 @@ App::get('/v1/migrations/nhost/report')
     ->param('port', 5432, new Integer(true), 'Source\'s Database Port.', true)
     ->inject('response')
     ->action(function (array $resources, string $subdomain, string $region, string $adminSecret, string $database, string $username, string $password, int $port, Response $response) {
-        try {
-            $nhost = new NHost($subdomain, $region, $adminSecret, $database, $username, $password, $port);
+        $nhost = new NHost($subdomain, $region, $adminSecret, $database, $username, $password, $port);
 
-            $response
-                ->setStatusCode(Response::STATUS_CODE_OK)
-                ->dynamic(new Document($nhost->report($resources)), Response::MODEL_MIGRATION_REPORT);
-        } catch (\Exception $e) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage());
+        try {
+            $report = $nhost->report($resources);
+        } catch (\Throwable $e) {
+            switch ($e->getCode()) {
+                case 401:
+                    throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, 'Source Error: ' . $e->getMessage());
+                case 429:
+                    throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Source Error: Rate Limit Exceeded, Is your Cloud Provider blocking Appwrite\'s IP?');
+                case 500:
+                    throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
+            }
+
+            throw new Exception(Exception::MIGRATION_PROVIDER_ERROR, 'Source Error: ' . $e->getMessage());
         }
+
+        $response
+            ->setStatusCode(Response::STATUS_CODE_OK)
+            ->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT);
     });
 
 App::patch('/v1/migrations/:migrationId')

app/controllers/api/project.php

@@ -3,8 +3,10 @@
 use Appwrite\Extend\Exception;
 use Appwrite\Utopia\Response;
 use Utopia\App;
+use Utopia\Config\Config;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
+use Utopia\Database\Validator\Datetime as DateTimeValidator;
 use Utopia\Database\Exception\Duplicate as DuplicateException;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Helpers\Permission;
@@ -14,11 +16,10 @@ use Utopia\Database\Validator\Authorization;
 use Utopia\Database\Validator\UID;
 use Utopia\Validator\Text;
 use Utopia\Validator\WhiteList;
-use Utopia\Database\DateTime;
 
 App::get('/v1/project/usage')
     ->desc('Get usage stats for a project')
-    ->groups(['api'])
+    ->groups(['api', 'usage'])
     ->label('scope', 'projects.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'project')
@@ -26,100 +27,141 @@ App::get('/v1/project/usage')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_PROJECT)
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
+    ->param('startDate', '', new DateTimeValidator(), 'Starting date for the usage')
+    ->param('endDate', '', new DateTimeValidator(), 'End date for the usage')
+    ->param('period', '1d', new WhiteList(['1h', '1d']), 'Period used', true)
     ->inject('response')
     ->inject('dbForProject')
-    ->action(function (string $range, Response $response, Database $dbForProject) {
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+    ->action(function (string $startDate, string $endDate, string $period, Response $response, Database $dbForProject) {
+        $stats = $total = $usage = [];
+        $format = 'Y-m-d 00:00:00';
+        $firstDay = (new DateTime($startDate))->format($format);
+        $lastDay = (new DateTime($endDate))->format($format);
 
-            $metrics = [
-                'project.$all.network.requests',
-                'project.$all.network.bandwidth',
-                'project.$all.storage.size',
-                'users.$all.count.total',
-                'databases.$all.count.total',
-                'documents.$all.count.total',
-                'executions.$all.compute.total',
-                'buckets.$all.count.total'
-            ];
+        $metrics = [
+           'total' => [
+                METRIC_EXECUTIONS,
+                METRIC_DOCUMENTS,
+                METRIC_DATABASES,
+                METRIC_USERS,
+                METRIC_BUCKETS,
+                METRIC_FILES_STORAGE
+           ],
+           'period' => [
+                METRIC_NETWORK_REQUESTS,
+                METRIC_NETWORK_INBOUND,
+                METRIC_NETWORK_OUTBOUND,
+                METRIC_USERS,
+                METRIC_EXECUTIONS
+             ]
+        ];
 
-            $stats = [];
+        $factor = match ($period) {
+            '1h' => 3600,
+            '1d' => 86400,
+        };
 
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
+        $limit = match ($period) {
+            '1h' => (new DateTime($startDate))->diff(new DateTime($endDate))->days * 24,
+            '1d' => (new DateTime($startDate))->diff(new DateTime($endDate))->days
+        };
 
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
+        $format = match ($period) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
 
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
+        Authorization::skip(function () use ($dbForProject, $firstDay, $lastDay, $period, $metrics, &$total, &$stats) {
+            foreach ($metrics['total'] as $metric) {
+                $result = $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
+                $total[$metric] = $result['value'] ?? 0;
+            }
 
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
+            foreach ($metrics['period'] as $metric) {
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::greaterThanEqual('time', $firstDay),
+                    Query::lessThan('time', $lastDay),
+                    Query::orderDesc('time'),
+                ]);
+
+                $stats[$metric] = [];
+                foreach ($results as $result) {
+                    $stats[$metric][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'requests' => $stats[$metrics[0]] ?? [],
-                'network' => $stats[$metrics[1]] ?? [],
-                'storage' => $stats[$metrics[2]] ?? [],
-                'users' => $stats[$metrics[3]] ?? [],
-                'databases' => $stats[$metrics[4]] ?? [],
-                'documents' => $stats[$metrics[5]] ?? [],
-                'executions' => $stats[$metrics[6]] ?? [],
-                'buckets' => $stats[$metrics[7]] ?? [],
-            ]);
+        $now = time();
+        foreach ($metrics['period'] as $metric) {
+            $usage[$metric] = [];
+            $leap = $now - ($limit * $factor);
+            while ($leap < $now) {
+                $leap += $factor;
+                $formatDate = date($format, $leap);
+                $usage[$metric][] = [
+                    'value' => $stats[$metric][$formatDate]['value'] ?? 0,
+                    'date' => $formatDate,
+                ];
+            }
         }
 
-        $response->dynamic($usage, Response::MODEL_USAGE_PROJECT);
+        $executionsBreakdown = array_map(function ($function) use ($dbForProject) {
+            $id = $function->getId();
+            $name = $function->getAttribute('name');
+            $metric = str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS);
+            $value = $dbForProject->findOne('stats_v2', [
+                Query::equal('metric', [$metric]),
+                Query::equal('period', ['inf'])
+            ]);
+
+            return [
+                'resourceId' => $id,
+                'name' => $name,
+                'value' => $value['value'] ?? 0,
+            ];
+        }, $dbForProject->find('functions'));
+
+        $bucketsBreakdown = array_map(function ($bucket) use ($dbForProject) {
+            $id = $bucket->getId();
+            $name = $bucket->getAttribute('name');
+            $metric = str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE);
+            $value = $dbForProject->findOne('stats_v2', [
+                Query::equal('metric', [$metric]),
+                Query::equal('period', ['inf'])
+            ]);
+
+            return [
+                'resourceId' => $id,
+                'name' => $name,
+                'value' => $value['value'] ?? 0,
+            ];
+        }, $dbForProject->find('buckets'));
+
+        $response->dynamic(new Document([
+            'requests' => ($usage[METRIC_NETWORK_REQUESTS]),
+            'network' => ($usage[METRIC_NETWORK_INBOUND] + $usage[METRIC_NETWORK_OUTBOUND]),
+            'users' => ($usage[METRIC_USERS]),
+            'executions' => ($usage[METRIC_EXECUTIONS]),
+            'executionsTotal' => $total[METRIC_EXECUTIONS],
+            'documentsTotal' => $total[METRIC_DOCUMENTS],
+            'databasesTotal' => $total[METRIC_DATABASES],
+            'usersTotal' => $total[METRIC_USERS],
+            'bucketsTotal' => $total[METRIC_BUCKETS],
+            'filesStorageTotal' => $total[METRIC_FILES_STORAGE],
+            'executionsBreakdown' => $executionsBreakdown,
+            'bucketsBreakdown' => $bucketsBreakdown
+        ]), Response::MODEL_USAGE_PROJECT);
     });
 
-// Variables
 
+// Variables
 App::post('/v1/project/variables')
     ->desc('Create Variable')
     ->groups(['api'])

app/controllers/api/projects.php

@@ -92,12 +92,12 @@ App::post('/v1/projects')
 
         $projectId = ($projectId == 'unique()') ? ID::unique() : $projectId;
 
-        $backups['database_db_fra1_v14x_02'] = ['from' => '7:30', 'to' => '8:15'];
-        $backups['database_db_fra1_v14x_03'] = ['from' => '10:30', 'to' => '11:15'];
-        $backups['database_db_fra1_v14x_04'] = ['from' => '13:30', 'to' => '14:15'];
-        $backups['database_db_fra1_v14x_05'] = ['from' => '4:30', 'to' => '5:15'];
-        $backups['database_db_fra1_v14x_06'] = ['from' => '16:30', 'to' => '17:15'];
-        $backups['database_db_fra1_v14x_07'] = ['from' => '19:30', 'to' => '20:15'];
+        $backups['database_db_fra1_v14x_02'] = ['from' => '03:00', 'to' => '05:00'];
+        $backups['database_db_fra1_v14x_03'] = ['from' => '00:00', 'to' => '02:00'];
+        $backups['database_db_fra1_v14x_04'] = ['from' => '00:00', 'to' => '02:00'];
+        $backups['database_db_fra1_v14x_05'] = ['from' => '00:00', 'to' => '02:00'];
+        $backups['database_db_fra1_v14x_06'] = ['from' => '00:00', 'to' => '02:00'];
+        $backups['database_db_fra1_v14x_07'] = ['from' => '00:00', 'to' => '02:00'];
 
         $databases = Config::getParam('pools-database', []);
 
@@ -123,7 +123,7 @@ App::post('/v1/projects')
 
         $databaseOverride = App::getEnv('_APP_DATABASE_OVERRIDE', null);
         $index = array_search($databaseOverride, $databases);
-        if ($index) {
+        if ($index !== false) {
             $database = $databases[$index];
         } else {
             $database = $databases[array_rand($databases)];
@@ -293,120 +293,6 @@ App::get('/v1/projects/:projectId')
         $response->dynamic($project, Response::MODEL_PROJECT);
     });
 
-App::get('/v1/projects/:projectId/usage')
-    ->desc('Get usage stats for a project')
-    ->groups(['api', 'projects', 'usage'])
-    ->label('scope', 'projects.read')
-    ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
-    ->label('sdk.namespace', 'projects')
-    ->label('sdk.method', 'getUsage')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_USAGE_PROJECT)
-    ->param('projectId', '', new UID(), 'Project unique ID.')
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
-    ->inject('response')
-    ->inject('dbForConsole')
-    ->inject('dbForProject')
-    ->inject('register')
-    ->action(function (string $projectId, string $range, Response $response, Database $dbForConsole, Database $dbForProject, Registry $register) {
-
-        $project = $dbForConsole->getDocument('projects', $projectId);
-
-        if ($project->isEmpty()) {
-            throw new Exception(Exception::PROJECT_NOT_FOUND);
-        }
-
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
-
-            $dbForProject->setNamespace("_{$project->getInternalId()}");
-
-            $metrics = [
-                'project.$all.network.requests',
-                'project.$all.network.bandwidth',
-                'project.$all.storage.size',
-                'users.$all.count.total',
-                'databases.$all.count.total',
-                'documents.$all.count.total',
-                'executions.$all.compute.total',
-                'buckets.$all.count.total'
-            ];
-
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
-                }
-            });
-
-            $usage = new Document([
-                'range' => $range,
-                'requests' => $stats[$metrics[0]] ?? [],
-                'network' => $stats[$metrics[1]] ?? [],
-                'storage' => $stats[$metrics[2]] ?? [],
-                'users' => $stats[$metrics[3]] ?? [],
-                'databases' => $stats[$metrics[4]] ?? [],
-                'documents' => $stats[$metrics[5]] ?? [],
-                'executions' => $stats[$metrics[6]] ?? [],
-                'buckets' => $stats[$metrics[7]] ?? [],
-            ]);
-        }
-
-        $response->dynamic($usage, Response::MODEL_USAGE_PROJECT);
-    });
-
 App::patch('/v1/projects/:projectId')
     ->desc('Update project')
     ->groups(['api', 'projects'])
@@ -1703,7 +1589,6 @@ App::get('/v1/projects/:projectId/templates/email/:type/:locale')
             $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
             $message
                 ->setParam('{{hello}}', $localeObj->getText("emails.{$type}.hello"))
-                ->setParam('{{user}}', '')
                 ->setParam('{{footer}}', $localeObj->getText("emails.{$type}.footer"))
                 ->setParam('{{body}}', $localeObj->getText('emails.' . $type . '.body'))
                 ->setParam('{{thanks}}', $localeObj->getText("emails.{$type}.thanks"))

app/controllers/api/proxy.php

@@ -37,10 +37,11 @@ App::post('/v1/proxy/rules')
     ->param('resourceId', '', new UID(), 'ID of resource for the action type. If resourceType is "api", leave empty. If resourceType is "function", provide ID of the function.', true)
     ->inject('response')
     ->inject('project')
-    ->inject('events')
+    ->inject('queueForCertificates')
+    ->inject('queueForEvents')
     ->inject('dbForConsole')
     ->inject('dbForProject')
-    ->action(function (string $domain, string $resourceType, string $resourceId, Response $response, Document $project, Event $events, Database $dbForConsole, Database $dbForProject) {
+    ->action(function (string $domain, string $resourceType, string $resourceId, Response $response, Document $project, Certificate $queueForCertificates, Event $queueForEvents, Database $dbForConsole, Database $dbForProject) {
         $mainDomain = App::getEnv('_APP_DOMAIN', '');
         if ($domain === $mainDomain) {
             throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'You cannot assign your main domain to specific resource. Please use subdomain or a different domain.');
@@ -86,7 +87,11 @@ App::post('/v1/proxy/rules')
             $resourceInternalId = $function->getInternalId();
         }
 
-        $domain = new Domain($domain);
+        try {
+            $domain = new Domain($domain);
+        } catch (\Exception) {
+            throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Domain may not start with http:// or https://.');
+        }
 
         $ruleId = ID::unique();
         $rule = new Document([
@@ -113,8 +118,7 @@ App::post('/v1/proxy/rules')
             if ($validator->isValid($domain->get())) {
                 $status = 'verifying';
 
-                $event = new Certificate();
-                $event
+                $queueForCertificates
                     ->setDomain(new Document([
                         'domain' => $rule->getAttribute('domain')
                     ]))
@@ -125,7 +129,7 @@ App::post('/v1/proxy/rules')
         $rule->setAttribute('status', $status);
         $rule = $dbForConsole->createDocument('rules', $rule);
 
-        $events->setParam('ruleId', $rule->getId());
+        $queueForEvents->setParam('ruleId', $rule->getId());
 
         $rule->setAttribute('logs', '');
 
@@ -235,9 +239,9 @@ App::delete('/v1/proxy/rules/:ruleId')
     ->inject('response')
     ->inject('project')
     ->inject('dbForConsole')
-    ->inject('deletes')
-    ->inject('events')
-    ->action(function (string $ruleId, Response $response, Document $project, Database $dbForConsole, Delete $deletes, Event $events) {
+    ->inject('queueForDeletes')
+    ->inject('queueForEvents')
+    ->action(function (string $ruleId, Response $response, Document $project, Database $dbForConsole, Delete $queueForDeletes, Event $queueForEvents) {
         $rule = $dbForConsole->getDocument('rules', $ruleId);
 
         if ($rule->isEmpty() || $rule->getAttribute('projectInternalId') !== $project->getInternalId()) {
@@ -246,11 +250,11 @@ App::delete('/v1/proxy/rules/:ruleId')
 
         $dbForConsole->deleteDocument('rules', $rule->getId());
 
-        $deletes
+        $queueForDeletes
             ->setType(DELETE_TYPE_DOCUMENT)
             ->setDocument($rule);
 
-        $events->setParam('ruleId', $rule->getId());
+        $queueForEvents->setParam('ruleId', $rule->getId());
 
         $response->noContent();
     });
@@ -270,10 +274,11 @@ App::patch('/v1/proxy/rules/:ruleId/verification')
     ->label('sdk.response.model', Response::MODEL_PROXY_RULE)
     ->param('ruleId', '', new UID(), 'Rule ID.')
     ->inject('response')
-    ->inject('events')
+    ->inject('queueForCertificates')
+    ->inject('queueForEvents')
     ->inject('project')
     ->inject('dbForConsole')
-    ->action(function (string $ruleId, Response $response, Event $events, Document $project, Database $dbForConsole) {
+    ->action(function (string $ruleId, Response $response, Certificate $queueForCertificates, Event $queueForEvents, Document $project, Database $dbForConsole) {
         $rule = $dbForConsole->getDocument('rules', $ruleId);
 
         if ($rule->isEmpty() || $rule->getAttribute('projectInternalId') !== $project->getInternalId()) {
@@ -300,14 +305,13 @@ App::patch('/v1/proxy/rules/:ruleId/verification')
         $dbForConsole->updateDocument('rules', $rule->getId(), $rule->setAttribute('status', 'verifying'));
 
         // Issue a TLS certificate when domain is verified
-        $event = new Certificate();
-        $event
+        $queueForCertificates
             ->setDomain(new Document([
                 'domain' => $rule->getAttribute('domain')
             ]))
             ->trigger();
 
-        $events->setParam('ruleId', $rule->getId());
+        $queueForEvents->setParam('ruleId', $rule->getId());
 
         $certificate = $dbForConsole->getDocument('certificates', $rule->getAttribute('certificateId', ''));
         $rule->setAttribute('logs', $certificate->getAttribute('logs', ''));

app/controllers/api/storage.php

@@ -52,7 +52,6 @@ App::post('/v1/storage/buckets')
     ->label('event', 'buckets.[bucketId].create')
     ->label('audits.event', 'bucket.create')
     ->label('audits.resource', 'bucket/{response.$id}')
-    ->label('usage.metric', 'buckets.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'createBucket')
@@ -148,7 +147,6 @@ App::get('/v1/storage/buckets')
     ->desc('List buckets')
     ->groups(['api', 'storage'])
     ->label('scope', 'buckets.read')
-    ->label('usage.metric', 'buckets.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'listBuckets')
@@ -197,7 +195,6 @@ App::get('/v1/storage/buckets/:bucketId')
     ->desc('Get bucket')
     ->groups(['api', 'storage'])
     ->label('scope', 'buckets.read')
-    ->label('usage.metric', 'buckets.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'getBucket')
@@ -226,7 +223,6 @@ App::put('/v1/storage/buckets/:bucketId')
     ->label('event', 'buckets.[bucketId].update')
     ->label('audits.event', 'bucket.update')
     ->label('audits.resource', 'bucket/{response.$id}')
-    ->label('usage.metric', 'buckets.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'updateBucket')
@@ -294,7 +290,6 @@ App::delete('/v1/storage/buckets/:bucketId')
     ->label('audits.event', 'bucket.delete')
     ->label('event', 'buckets.[bucketId].delete')
     ->label('audits.resource', 'bucket/{request.bucketId}')
-    ->label('usage.metric', 'buckets.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'deleteBucket')
@@ -337,8 +332,6 @@ App::post('/v1/storage/buckets/:bucketId/files')
     ->label('audits.event', 'file.create')
     ->label('event', 'buckets.[bucketId].files.[fileId].create')
     ->label('audits.resource', 'file/{response.$id}')
-    ->label('usage.metric', 'files.{scope}.requests.create')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -689,8 +682,6 @@ App::get('/v1/storage/buckets/:bucketId/files')
     ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('usage.metric', 'files.{scope}.requests.read')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'listFiles')
     ->label('sdk.description', '/docs/references/storage/list-files.md')
@@ -770,8 +761,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId')
     ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('usage.metric', 'files.{scope}.requests.read')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'getFile')
     ->label('sdk.description', '/docs/references/storage/get-file.md')
@@ -821,8 +810,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
     ->label('cache', true)
     ->label('cache.resourceType', 'bucket/{request.bucketId}')
     ->label('cache.resource', 'file/{request.fileId}')
-    ->label('usage.metric', 'files.{scope}.requests.read')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'getFilePreview')
@@ -987,8 +974,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
     ->desc('Get file for download')
     ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
-    ->label('usage.metric', 'files.{scope}.requests.read')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'getFileDownload')
@@ -1109,7 +1094,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
         }
 
         if ($size > APP_STORAGE_READ_BUFFER) {
-            $response->addHeader('Content-Length', $deviceFiles->getFileSize($path));
             for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) {
                 $response->chunk(
                     $deviceFiles->read(
@@ -1130,8 +1114,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
     ->desc('Get file for view')
     ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
-    ->label('usage.metric', 'files.{scope}.requests.read')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'getFileView')
@@ -1262,7 +1244,6 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
 
         $size = $deviceFiles->getFileSize($path);
         if ($size > APP_STORAGE_READ_BUFFER) {
-            $response->addHeader('Content-Length', $deviceFiles->getFileSize($path));
             for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) {
                 $response->chunk(
                     $deviceFiles->read(
@@ -1286,8 +1267,6 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
     ->label('event', 'buckets.[bucketId].files.[fileId].update')
     ->label('audits.event', 'file.update')
     ->label('audits.resource', 'file/{response.$id}')
-    ->label('usage.metric', 'files.{scope}.requests.update')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -1396,8 +1375,6 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
     ->label('event', 'buckets.[bucketId].files.[fileId].delete')
     ->label('audits.event', 'file.delete')
     ->label('audits.resource', 'file/{request.fileId}')
-    ->label('usage.metric', 'files.{scope}.requests.delete')
-    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -1489,7 +1466,7 @@ App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
 
 App::get('/v1/storage/usage')
     ->desc('Get usage stats for storage')
-    ->groups(['api', 'storage', 'usage'])
+    ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'storage')
@@ -1497,109 +1474,78 @@ App::get('/v1/storage/usage')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_STORAGE)
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (string $range, Response $response, Database $dbForProject) {
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            METRIC_BUCKETS,
+            METRIC_FILES,
+            METRIC_FILES_STORAGE,
+        ];
 
-            $metrics = [
-                'project.$all.storage.size',
-                'buckets.$all.count.total',
-                'buckets.$all.requests.create',
-                'buckets.$all.requests.read',
-                'buckets.$all.requests.update',
-                'buckets.$all.requests.delete',
-                'files.$all.storage.size',
-                'files.$all.count.total',
-                'files.$all.requests.create',
-                'files.$all.requests.read',
-                'files.$all.requests.update',
-                'files.$all.requests.delete',
-            ];
+        $total = [];
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
+            foreach ($metrics as $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
 
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'bucketsCount' => $stats['buckets.$all.count.total'],
-                'bucketsCreate' => $stats['buckets.$all.requests.create'],
-                'bucketsRead' => $stats['buckets.$all.requests.read'],
-                'bucketsUpdate' => $stats['buckets.$all.requests.update'],
-                'bucketsDelete' => $stats['buckets.$all.requests.delete'],
-                'storage' => $stats['project.$all.storage.size'],
-                'filesCount' => $stats['files.$all.count.total'],
-                'filesCreate' => $stats['files.$all.requests.create'],
-                'filesRead' => $stats['files.$all.requests.read'],
-                'filesUpdate' => $stats['files.$all.requests.update'],
-                'filesDelete' => $stats['files.$all.requests.delete'],
-            ]);
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
         }
-
-        $response->dynamic($usage, Response::MODEL_USAGE_STORAGE);
+    }
+        $response->dynamic(new Document([
+            'range' => $range,
+            'bucketsTotal' => $usage[$metrics[0]]['total'],
+            'filesTotal' => $usage[$metrics[1]]['total'],
+            'filesStorageTotal' => $usage[$metrics[2]]['total'],
+            'buckets' => $usage[$metrics[0]]['data'],
+            'files' =>  $usage[$metrics[1]]['data'],
+            'storage' =>  $usage[$metrics[2]]['data'],
+        ]), Response::MODEL_USAGE_STORAGE);
     });
 
 App::get('/v1/storage/:bucketId/usage')
-    ->desc('Get usage stats for a storage bucket')
-    ->groups(['api', 'storage', 'usage'])
+    ->desc('Get usage stats for storage bucket')
+    ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'storage')
@@ -1608,7 +1554,7 @@ App::get('/v1/storage/:bucketId/usage')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_BUCKETS)
     ->param('bucketId', '', new UID(), 'Bucket ID.')
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (string $bucketId, string $range, Response $response, Database $dbForProject) {
@@ -1619,86 +1565,65 @@ App::get('/v1/storage/:bucketId/usage')
             throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
         }
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES),
+            str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE),
+        ];
 
-            $metrics = [
-                "files.{$bucketId}.count.total",
-                "files.{$bucketId}.storage.size",
-                "files.{$bucketId}.requests.create",
-                "files.{$bucketId}.requests.read",
-                "files.{$bucketId}.requests.update",
-                "files.{$bucketId}.requests.delete",
-            ];
 
-            $stats = [];
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats, &$total) {
+            foreach ($metrics as $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
 
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'filesCount' => $stats[$metrics[0]],
-                'filesStorage' => $stats[$metrics[1]],
-                'filesCreate' => $stats[$metrics[2]],
-                'filesRead' => $stats[$metrics[3]],
-                'filesUpdate' => $stats[$metrics[4]],
-                'filesDelete' => $stats[$metrics[5]],
-            ]);
+
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
         }
+    }
 
-        $response->dynamic($usage, Response::MODEL_USAGE_BUCKETS);
+        $response->dynamic(new Document([
+            'range' => $range,
+            'filesTotal' => $usage[$metrics[0]]['total'],
+            'filesStorageTotal' => $usage[$metrics[1]]['total'],
+            'files' => $usage[$metrics[0]]['data'],
+            'storage' => $usage[$metrics[1]]['data'],
+        ]), Response::MODEL_USAGE_BUCKETS);
     });

app/controllers/api/teams.php

@@ -555,7 +555,12 @@ App::post('/v1/teams/:teamId/memberships')
                 $customTemplate = $project->getAttribute('templates', [])['email.invitation-' . $locale->default] ?? [];
 
                 $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
-                $message->setParam('{{body}}', $body);
+                $message
+                    ->setParam('{{body}}', $body)
+                    ->setParam('{{hello}}', $locale->getText("emails.invitation.hello"))
+                    ->setParam('{{footer}}', $locale->getText("emails.invitation.footer"))
+                    ->setParam('{{thanks}}', $locale->getText("emails.invitation.thanks"))
+                    ->setParam('{{signature}}', $locale->getText("emails.invitation.signature"));
                 $body = $message->render();
 
                 $smtp = $project->getAttribute('smtp', []);
@@ -606,16 +611,7 @@ App::post('/v1/teams/:teamId/memberships')
 
                 $emailVariables = [
                     'owner' => $user->getAttribute('name'),
-                    'subject' => $subject,
-                    'hello' => $locale->getText("emails.invitation.hello"),
-                    'body' => $body,
-                    'footer' => $locale->getText("emails.invitation.footer"),
-                    'thanks' => $locale->getText("emails.invitation.thanks"),
-                    'signature' => $locale->getText("emails.invitation.signature"),
                     'direction' => $locale->getText('settings.direction'),
-                    'bg-body' => '#f7f7f7',
-                    'bg-content' => '#ffffff',
-                    'text-content' => '#000000',
                     /* {{user}} ,{{team}}, {{project}} and {{redirect}} are required in the templates */
                     'user' => $user->getAttribute('name'),
                     'team' => $team->getAttribute('name'),

app/controllers/api/users.php

@@ -85,7 +85,7 @@ function createUser(string $hash, mixed $hashOptions, string $userId, ?string $e
             'status' => true,
             'labels' => [],
             'password' => $password,
-            'passwordHistory' => is_null($password) && $passwordHistory === 0 ? [] : [$password],
+            'passwordHistory' => is_null($password) || $passwordHistory === 0 ? [] : [$password],
             'passwordUpdate' => (!empty($password)) ? DateTime::now() : null,
             'hash' => $hash === 'plaintext' ? Auth::DEFAULT_ALGO : $hash,
             'hashOptions' => $hash === 'plaintext' ? Auth::DEFAULT_ALGO_OPTIONS : $hashOptionsObject + ['type' => $hash],
@@ -114,7 +114,6 @@ App::post('/v1/users')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'create')
@@ -146,7 +145,6 @@ App::post('/v1/users/bcrypt')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createBcryptUser')
@@ -177,7 +175,6 @@ App::post('/v1/users/md5')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createMD5User')
@@ -208,7 +205,6 @@ App::post('/v1/users/argon2')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createArgon2User')
@@ -239,7 +235,6 @@ App::post('/v1/users/sha')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createSHAUser')
@@ -277,7 +272,6 @@ App::post('/v1/users/phpass')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createPHPassUser')
@@ -308,7 +302,6 @@ App::post('/v1/users/scrypt')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createScryptUser')
@@ -352,7 +345,6 @@ App::post('/v1/users/scrypt-modified')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createScryptModifiedUser')
@@ -383,7 +375,6 @@ App::get('/v1/users')
     ->desc('List users')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'list')
@@ -432,7 +423,6 @@ App::get('/v1/users/:userId')
     ->desc('Get user')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'get')
@@ -458,7 +448,6 @@ App::get('/v1/users/:userId/prefs')
     ->desc('Get user preferences')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'getPrefs')
@@ -486,7 +475,6 @@ App::get('/v1/users/:userId/sessions')
     ->desc('List user sessions')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'listSessions')
@@ -528,7 +516,6 @@ App::get('/v1/users/:userId/memberships')
     ->desc('List user memberships')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'listMemberships')
@@ -568,7 +555,6 @@ App::get('/v1/users/:userId/logs')
     ->desc('List user logs')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'listLogs')
@@ -650,7 +636,6 @@ App::get('/v1/users/identities')
     ->desc('List Identities')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
-    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'listIdentities')
@@ -703,7 +688,6 @@ App::patch('/v1/users/:userId/status')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updateStatus')
@@ -739,7 +723,6 @@ App::put('/v1/users/:userId/labels')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updateLabels')
@@ -777,7 +760,6 @@ App::patch('/v1/users/:userId/verification/phone')
     ->label('scope', 'users.write')
     ->label('audits.event', 'verification.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updatePhoneVerification')
@@ -814,7 +796,6 @@ App::patch('/v1/users/:userId/name')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updateName')
@@ -852,7 +833,6 @@ App::patch('/v1/users/:userId/password')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updatePassword')
@@ -917,7 +897,6 @@ App::patch('/v1/users/:userId/email')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updateEmail')
@@ -973,7 +952,6 @@ App::patch('/v1/users/:userId/phone')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updatePhone')
@@ -1018,7 +996,6 @@ App::patch('/v1/users/:userId/verification')
     ->label('audits.event', 'verification.update')
     ->label('audits.resource', 'user/{request.userId}')
     ->label('audits.userId', '{request.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updateEmailVerification')
@@ -1051,7 +1028,6 @@ App::patch('/v1/users/:userId/prefs')
     ->groups(['api', 'users'])
     ->label('event', 'users.[userId].update.prefs')
     ->label('scope', 'users.write')
-    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updatePrefs')
@@ -1087,7 +1063,6 @@ App::delete('/v1/users/:userId/sessions/:sessionId')
     ->label('scope', 'users.write')
     ->label('audits.event', 'session.delete')
     ->label('audits.resource', 'user/{request.userId}')
-    ->label('usage.metric', 'sessions.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'deleteSession')
@@ -1127,11 +1102,10 @@ App::delete('/v1/users/:userId/sessions/:sessionId')
 App::delete('/v1/users/:userId/sessions')
     ->desc('Delete user sessions')
     ->groups(['api', 'users'])
-    ->label('event', 'users.[userId].sessions.[sessionId].delete')
+    ->label('event', 'users.[userId].sessions.delete')
     ->label('scope', 'users.write')
     ->label('audits.event', 'session.delete')
     ->label('audits.resource', 'user/{user.$id}')
-    ->label('usage.metric', 'sessions.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'deleteSessions')
@@ -1174,7 +1148,6 @@ App::delete('/v1/users/:userId')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.delete')
     ->label('audits.resource', 'user/{request.userId}')
-    ->label('usage.metric', 'users.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'delete')
@@ -1217,7 +1190,6 @@ App::delete('/v1/users/identities/:identityId')
     ->label('scope', 'users.write')
     ->label('audits.event', 'identity.delete')
     ->label('audits.resource', 'identity/{request.$identityId}')
-    ->label('usage.metric', 'users.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'deleteIdentity')
@@ -1242,7 +1214,7 @@ App::delete('/v1/users/identities/:identityId')
 
 App::get('/v1/users/usage')
     ->desc('Get usage stats for the users API')
-    ->groups(['api', 'users', 'usage'])
+    ->groups(['api', 'users'])
     ->label('scope', 'users.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'users')
@@ -1250,97 +1222,69 @@ App::get('/v1/users/usage')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_USERS)
-    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
-    ->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(fn ($value) => "oauth-" . $value, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true)
+    ->param('range', '30d', new WhiteList(['24h', '30d', '90d'], true), 'Date range.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->inject('register')
-    ->action(function (string $range, string $provider, Response $response, Database $dbForProject) {
+    ->action(function (string $range, Response $response, Database $dbForProject) {
 
-        $usage = [];
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
-            $periods = [
-                '24h' => [
-                    'period' => '1h',
-                    'limit' => 24,
-                ],
-                '7d' => [
-                    'period' => '1d',
-                    'limit' => 7,
-                ],
-                '30d' => [
-                    'period' => '1d',
-                    'limit' => 30,
-                ],
-                '90d' => [
-                    'period' => '1d',
-                    'limit' => 90,
-                ],
-            ];
+        $periods = Config::getParam('usage', []);
+        $stats = $usage = [];
+        $days = $periods[$range];
+        $metrics = [
+            METRIC_USERS,
+            METRIC_SESSIONS,
+        ];
 
-            $metrics = [
-                'users.$all.count.total',
-                'users.$all.requests.create',
-                'users.$all.requests.read',
-                'users.$all.requests.update',
-                'users.$all.requests.delete',
-                'sessions.$all.requests.create',
-                'sessions.$all.requests.delete',
-                "sessions.$provider.requests.create",
-            ];
+        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
+            foreach ($metrics as $count => $metric) {
+                $result =  $dbForProject->findOne('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', ['inf'])
+                ]);
 
-            $stats = [];
-
-            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
-                foreach ($metrics as $metric) {
-                    $limit = $periods[$range]['limit'];
-                    $period = $periods[$range]['period'];
-
-                    $requestDocs = $dbForProject->find('stats', [
-                        Query::equal('period', [$period]),
-                        Query::equal('metric', [$metric]),
-                        Query::limit($limit),
-                        Query::orderDesc('time'),
-                    ]);
-
-                    $stats[$metric] = [];
-                    foreach ($requestDocs as $requestDoc) {
-                        $stats[$metric][] = [
-                            'value' => $requestDoc->getAttribute('value'),
-                            'date' => $requestDoc->getAttribute('time'),
-                        ];
-                    }
-
-                    // backfill metrics with empty values for graphs
-                    $backfill = $limit - \count($requestDocs);
-                    while ($backfill > 0) {
-                        $last = $limit - $backfill - 1; // array index of last added metric
-                        $diff = match ($period) { // convert period to seconds for unix timestamp math
-                            '1h' => 3600,
-                            '1d' => 86400,
-                        };
-                        $stats[$metric][] = [
-                            'value' => 0,
-                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
-                        ];
-                        $backfill--;
-                    }
-                    $stats[$metric] = array_reverse($stats[$metric]);
+                $stats[$metric]['total'] = $result['value'] ?? 0;
+                $limit = $days['limit'];
+                $period = $days['period'];
+                $results = $dbForProject->find('stats_v2', [
+                    Query::equal('metric', [$metric]),
+                    Query::equal('period', [$period]),
+                    Query::limit($limit),
+                    Query::orderDesc('time'),
+                ]);
+                $stats[$metric]['data'] = [];
+                foreach ($results as $result) {
+                    $stats[$metric]['data'][$result->getAttribute('time')] = [
+                        'value' => $result->getAttribute('value'),
+                    ];
                 }
-            });
+            }
+        });
 
-            $usage = new Document([
-                'range' => $range,
-                'usersCount' => $stats['users.$all.count.total'] ?? [],
-                'usersCreate' => $stats['users.$all.requests.create'] ?? [],
-                'usersRead' => $stats['users.$all.requests.read'] ?? [],
-                'usersUpdate' => $stats['users.$all.requests.update'] ?? [],
-                'usersDelete' => $stats['users.$all.requests.delete'] ?? [],
-                'sessionsCreate' => $stats['sessions.$all.requests.create'] ?? [],
-                'sessionsProviderCreate' => $stats["sessions.$provider.requests.create"] ?? [],
-                'sessionsDelete' => $stats['sessions.$all.requests.delete' ?? []]
-            ]);
+        $format = match ($days['period']) {
+            '1h' => 'Y-m-d\TH:00:00.000P',
+            '1d' => 'Y-m-d\T00:00:00.000P',
+        };
+
+    foreach ($metrics as $metric) {
+        $usage[$metric]['total'] =  $stats[$metric]['total'];
+        $usage[$metric]['data'] = [];
+        $leap = time() - ($days['limit'] * $days['factor']);
+        while ($leap < time()) {
+            $leap += $days['factor'];
+            $formatDate = date($format, $leap);
+            $usage[$metric]['data'][] = [
+                'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0,
+                'date' => $formatDate,
+            ];
         }
+    }
 
-        $response->dynamic($usage, Response::MODEL_USAGE_USERS);
+        $response->dynamic(new Document([
+            'range' => $range,
+            'usersTotal'   => $usage[$metrics[0]]['total'],
+            'sessionsTotal' => $usage[$metrics[1]]['total'],
+            'users'   => $usage[$metrics[0]]['data'],
+            'sessions' => $usage[$metrics[1]]['data'],
+        ]), Response::MODEL_USAGE_USERS);
     });

app/controllers/api/vcs.php

@@ -22,7 +22,6 @@ use Utopia\Database\Helpers\Permission;
 use Utopia\Database\Helpers\Role;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
-use Utopia\Database\Validator\UID;
 use Utopia\Detector\Adapter\Bun;
 use Utopia\Detector\Adapter\CPP;
 use Utopia\Detector\Adapter\Dart;
@@ -36,6 +35,7 @@ use Utopia\Detector\Adapter\Ruby;
 use Utopia\Detector\Adapter\Swift;
 use Utopia\Detector\Detector;
 use Utopia\Validator\Boolean;
+use Utopia\VCS\Exception\RepositoryNotFound;
 
 use function Swoole\Coroutine\batch;
 
@@ -66,7 +66,14 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId
             }
 
             $owner = $github->getOwnerName($providerInstallationId) ?? '';
-            $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? '';
+            try {
+                $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? '';
+                if (empty($repositoryName)) {
+                    throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
+                }
+            } catch (RepositoryNotFound $e) {
+                throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
+            }
 
             if (empty($repositoryName)) {
                 throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
@@ -154,7 +161,14 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId
                 $message = 'Authorization required for external contributor.';
 
                 $providerRepositoryId = $resource->getAttribute('providerRepositoryId');
-                $repositoryName = $github->getRepositoryName($providerRepositoryId);
+                try {
+                    $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? '';
+                    if (empty($repositoryName)) {
+                        throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
+                    }
+                } catch (RepositoryNotFound $e) {
+                    throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
+                }
                 $owner = $github->getOwnerName($providerInstallationId);
                 $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'failure', $message, $authorizeUrl, $name);
                 continue;
@@ -206,7 +220,14 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId
                 $message = 'Starting...';
 
                 $providerRepositoryId = $resource->getAttribute('providerRepositoryId');
-                $repositoryName = $github->getRepositoryName($providerRepositoryId);
+                try {
+                    $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? '';
+                    if (empty($repositoryName)) {
+                        throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
+                    }
+                } catch (RepositoryNotFound $e) {
+                    throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
+                }
                 $owner = $github->getOwnerName($providerInstallationId);
 
                 $providerTargetUrl = $request->getProtocol() . '://' . $request->getHostname() . "/console/project-$projectId/functions/function-$functionId";
@@ -273,7 +294,7 @@ App::get('/v1/vcs/github/callback')
     ->label('scope', 'public')
     ->label('error', __DIR__ . '/../../views/general/error.phtml')
     ->param('installation_id', '', new Text(256, 0), 'GitHub installation ID', true)
-    ->param('setup_action', '', new Text(256, 0), 'GitHub setup actuon type', true)
+    ->param('setup_action', '', new Text(256, 0), 'GitHub setup action type', true)
     ->param('state', '', new Text(2048), 'GitHub state. Contains info sent when starting authorization flow.', true)
     ->param('code', '', new Text(2048, 0), 'OAuth2 code. This is a temporary code that the will be later exchanged for an access token.', true)
     ->inject('gitHub')
@@ -458,9 +479,12 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories/:pr
         $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId);
 
         $owner = $github->getOwnerName($providerInstallationId);
-        $repositoryName = $github->getRepositoryName($providerRepositoryId);
-
-        if (empty($repositoryName)) {
+        try {
+            $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? '';
+            if (empty($repositoryName)) {
+                throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
+            }
+        } catch (RepositoryNotFound $e) {
             throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
         }
 
@@ -720,9 +744,12 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories/:pro
         $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId);
 
         $owner = $github->getOwnerName($providerInstallationId) ?? '';
-        $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? '';
-
-        if (empty($repositoryName)) {
+        try {
+            $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? '';
+            if (empty($repositoryName)) {
+                throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
+            }
+        } catch (RepositoryNotFound $e) {
             throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
         }
 
@@ -766,9 +793,12 @@ App::get('/v1/vcs/github/installations/:installationId/providerRepositories/:pro
         $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId);
 
         $owner = $github->getOwnerName($providerInstallationId) ?? '';
-        $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? '';
-
-        if (empty($repositoryName)) {
+        try {
+            $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? '';
+            if (empty($repositoryName)) {
+                throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
+            }
+        } catch (RepositoryNotFound $e) {
             throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
         }
 
@@ -827,10 +857,10 @@ App::post('/v1/vcs/github/events')
                 $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId);
 
                 //find functionId from functions table
-                $repositories = $dbForConsole->find('repositories', [
+                $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
                     Query::equal('providerRepositoryId', [$providerRepositoryId]),
                     Query::limit(100),
-                ]);
+                ]));
 
                 // create new deployment only on push and not when branch is created
                 if (!$providerBranchCreated) {
@@ -847,13 +877,13 @@ App::post('/v1/vcs/github/events')
                     ]);
 
                     foreach ($installations as $installation) {
-                        $repositories = $dbForConsole->find('repositories', [
+                        $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
                             Query::equal('installationInternalId', [$installation->getInternalId()]),
                             Query::limit(1000)
-                        ]);
+                        ]));
 
                         foreach ($repositories as $repository) {
-                            $dbForConsole->deleteDocument('repositories', $repository->getId());
+                            Authorization::skip(fn () => $dbForConsole->deleteDocument('repositories', $repository->getId()));
                         }
 
                         $dbForConsole->deleteDocument('installations', $installation->getId());
@@ -885,10 +915,10 @@ App::post('/v1/vcs/github/events')
                     $providerCommitAuthor = $commitDetails["commitAuthor"] ?? '';
                     $providerCommitMessage = $commitDetails["commitMessage"] ?? '';
 
-                    $repositories = $dbForConsole->find('repositories', [
+                    $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
                         Query::equal('providerRepositoryId', [$providerRepositoryId]),
                         Query::orderDesc('$createdAt')
-                    ]);
+                    ]));
 
                     $createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $external, $dbForConsole, $queueForBuilds, $getProjectDB, $request);
                 } elseif ($parsedPayload["action"] == "closed") {
@@ -899,10 +929,10 @@ App::post('/v1/vcs/github/events')
                     $external = $parsedPayload["external"] ?? true;
 
                     if ($external) {
-                        $repositories = $dbForConsole->find('repositories', [
+                        $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
                             Query::equal('providerRepositoryId', [$providerRepositoryId]),
                             Query::orderDesc('$createdAt')
-                        ]);
+                        ]));
 
                         foreach ($repositories as $repository) {
                             $providerPullRequestIds = $repository->getAttribute('providerPullRequestIds', []);
@@ -1016,8 +1046,8 @@ App::delete('/v1/vcs/installations/:installationId')
     ->inject('response')
     ->inject('project')
     ->inject('dbForConsole')
-    ->inject('deletes')
-    ->action(function (string $installationId, Response $response, Document $project, Database $dbForConsole, Delete $deletes) {
+    ->inject('queueForDeletes')
+    ->action(function (string $installationId, Response $response, Document $project, Database $dbForConsole, Delete $queueForDeletes) {
         $installation = $dbForConsole->getDocument('installations', $installationId);
 
         if ($installation->isEmpty()) {
@@ -1028,7 +1058,7 @@ App::delete('/v1/vcs/installations/:installationId')
             throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove installation from DB');
         }
 
-        $deletes
+        $queueForDeletes
             ->setType(DELETE_TYPE_DOCUMENT)
             ->setDocument($installation);
 
@@ -1062,9 +1092,9 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor
             throw new Exception(Exception::INSTALLATION_NOT_FOUND);
         }
 
-        $repository = $dbForConsole->getDocument('repositories', $repositoryId, [
+        $repository = Authorization::skip(fn () => $dbForConsole->getDocument('repositories', $repositoryId, [
             Query::equal('projectInternalId', [$project->getInternalId()])
-        ]);
+        ]));
 
         if ($repository->isEmpty()) {
             throw new Exception(Exception::REPOSITORY_NOT_FOUND);
@@ -1079,7 +1109,7 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor
 
         // TODO: Delete from array when PR is closed
 
-        $repository = $dbForConsole->updateDocument('repositories', $repository->getId(), $repository);
+        $repository = Authorization::skip(fn () => $dbForConsole->updateDocument('repositories', $repository->getId(), $repository));
 
         $privateKey = App::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY');
         $githubAppId = App::getEnv('_APP_VCS_GITHUB_APP_ID');
@@ -1090,7 +1120,14 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor
         $providerRepositoryId = $repository->getAttribute('providerRepositoryId');
 
         $owner = $github->getOwnerName($providerInstallationId);
-        $repositoryName = $github->getRepositoryName($providerRepositoryId);
+        try {
+            $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? '';
+            if (empty($repositoryName)) {
+                throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
+            }
+        } catch (RepositoryNotFound $e) {
+            throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND);
+        }
         $pullRequestResponse = $github->getPullRequest($owner, $repositoryName, $providerPullRequestId);
 
         $providerBranch = \explode(':', $pullRequestResponse['head']['label'])[1] ?? '';

app/controllers/general.php

@@ -117,12 +117,14 @@ function router(App $utopia, Database $dbForConsole, SwooleRequest $swooleReques
             $path .= '?' . $query;
         }
 
+        $requestHeaders = $request->getHeaders();
+
         $body = \json_encode([
             'async' => false,
             'body' => $swooleRequest->getContent() ?? '',
             'method' => $swooleRequest->server['request_method'],
             'path' => $path,
-            'headers' => $swooleRequest->header
+            'headers' => $requestHeaders
         ]);
 
         $headers = [
@@ -606,8 +608,13 @@ App::error()
 
         $version = App::getEnv('_APP_VERSION', 'UNKNOWN');
         $route = $utopia->getRoute();
+        $publish = true;
 
-        if ($logger) {
+        if ($error instanceof AppwriteException) {
+            $publish = $error->isPublishable();
+        }
+
+        if ($logger && $publish) {
             if ($error->getCode() >= 500 || $error->getCode() === 0) {
                 try {
                     /** @var Utopia\Database\Document $user */

app/controllers/mock.php

@@ -9,506 +9,13 @@ use Appwrite\Utopia\Request;
 use Appwrite\Utopia\Response;
 use Utopia\App;
 use Utopia\Database\Database;
-use Utopia\Validator\ArrayList;
-use Utopia\Validator\Integer;
 use Utopia\Validator\Text;
-use Utopia\Storage\Validator\File;
 use Utopia\Validator\WhiteList;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Validator\UID;
-use Utopia\Validator\Nullable;
-
-App::get('/v1/mock/tests/foo')
-    ->desc('Get Foo')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'foo')
-    ->label('sdk.method', 'get')
-    ->label('sdk.description', 'Mock a get request.')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->param('x', '', new Text(100), 'Sample string param')
-    ->param('y', '', new Integer(true), 'Sample numeric param')
-    ->param('z', null, new ArrayList(new Text(256), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Sample array param')
-    ->action(function ($x, $y, $z) {
-    });
-
-App::post('/v1/mock/tests/foo')
-    ->desc('Post Foo')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'foo')
-    ->label('sdk.method', 'post')
-    ->label('sdk.description', 'Mock a post request.')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->param('x', '', new Text(100), 'Sample string param')
-    ->param('y', '', new Integer(true), 'Sample numeric param')
-    ->param('z', null, new ArrayList(new Text(256), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Sample array param')
-    ->action(function ($x, $y, $z) {
-    });
-
-App::patch('/v1/mock/tests/foo')
-    ->desc('Patch Foo')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'foo')
-    ->label('sdk.method', 'patch')
-    ->label('sdk.description', 'Mock a patch request.')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->param('x', '', new Text(100), 'Sample string param')
-    ->param('y', '', new Integer(true), 'Sample numeric param')
-    ->param('z', null, new ArrayList(new Text(256), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Sample array param')
-    ->action(function ($x, $y, $z) {
-    });
-
-App::put('/v1/mock/tests/foo')
-    ->desc('Put Foo')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'foo')
-    ->label('sdk.method', 'put')
-    ->label('sdk.description', 'Mock a put request.')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->param('x', '', new Text(100), 'Sample string param')
-    ->param('y', '', new Integer(true), 'Sample numeric param')
-    ->param('z', null, new ArrayList(new Text(256), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Sample array param')
-    ->action(function ($x, $y, $z) {
-    });
-
-App::delete('/v1/mock/tests/foo')
-    ->desc('Delete Foo')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'foo')
-    ->label('sdk.method', 'delete')
-    ->label('sdk.description', 'Mock a delete request.')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->param('x', '', new Text(100), 'Sample string param')
-    ->param('y', '', new Integer(true), 'Sample numeric param')
-    ->param('z', null, new ArrayList(new Text(256), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Sample array param')
-    ->action(function ($x, $y, $z) {
-    });
-
-App::get('/v1/mock/tests/bar')
-    ->desc('Get Bar')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'bar')
-    ->label('sdk.method', 'get')
-    ->label('sdk.description', 'Mock a get request.')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->param('required', '', new Text(100), 'Sample string param')
-    ->param('default', '', new Integer(true), 'Sample numeric param')
-    ->param('z', null, new ArrayList(new Text(256), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Sample array param')
-    ->action(function ($required, $default, $z) {
-    });
-
-App::post('/v1/mock/tests/bar')
-    ->desc('Post Bar')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'bar')
-    ->label('sdk.method', 'post')
-    ->label('sdk.description', 'Mock a post request.')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.offline.model', '/mock/tests/bar')
-    ->label('sdk.offline.key', '{required}')
-    ->label('sdk.mock', true)
-    ->param('required', '', new Text(100), 'Sample string param')
-    ->param('default', '', new Integer(true), 'Sample numeric param')
-    ->param('z', null, new ArrayList(new Text(256), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Sample array param')
-    ->action(function ($required, $default, $z) {
-    });
-
-App::patch('/v1/mock/tests/bar')
-    ->desc('Patch Bar')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'bar')
-    ->label('sdk.method', 'patch')
-    ->label('sdk.description', 'Mock a patch request.')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->param('required', '', new Text(100), 'Sample string param')
-    ->param('default', '', new Integer(true), 'Sample numeric param')
-    ->param('z', null, new ArrayList(new Text(256), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Sample array param')
-    ->action(function ($required, $default, $z) {
-    });
-
-App::put('/v1/mock/tests/bar')
-    ->desc('Put Bar')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'bar')
-    ->label('sdk.method', 'put')
-    ->label('sdk.description', 'Mock a put request.')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->param('required', '', new Text(100), 'Sample string param')
-    ->param('default', '', new Integer(true), 'Sample numeric param')
-    ->param('z', null, new ArrayList(new Text(256), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Sample array param')
-    ->action(function ($required, $default, $z) {
-    });
-
-App::delete('/v1/mock/tests/bar')
-    ->desc('Delete Bar')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'bar')
-    ->label('sdk.method', 'delete')
-    ->label('sdk.description', 'Mock a delete request.')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->param('required', '', new Text(100), 'Sample string param')
-    ->param('default', '', new Integer(true), 'Sample numeric param')
-    ->param('z', null, new ArrayList(new Text(256), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Sample array param')
-    ->action(function ($required, $default, $z) {
-    });
-
-App::get('/v1/mock/tests/general/headers')
-    ->desc('Get headers')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'general')
-    ->label('sdk.method', 'headers')
-    ->label('sdk.description', 'Return headers from the request')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->inject('request')
-    ->inject('response')
-    ->action(function (Request $request, Response $response) {
-        $res = [
-            'x-sdk-name' => $request->getHeader('x-sdk-name'),
-            'x-sdk-platform' => $request->getHeader('x-sdk-platform'),
-            'x-sdk-language' => $request->getHeader('x-sdk-language'),
-            'x-sdk-version' => $request->getHeader('x-sdk-version'),
-        ];
-        $res = array_map(function ($key, $value) {
-            return $key . ': ' . $value;
-        }, array_keys($res), $res);
-        $res = implode("; ", $res);
-
-        $response->dynamic(new Document(['result' => $res]), Response::MODEL_MOCK);
-    });
-
-App::get('/v1/mock/tests/general/download')
-    ->desc('Download File')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'general')
-    ->label('sdk.method', 'download')
-    ->label('sdk.methodType', 'location')
-    ->label('sdk.description', 'Mock a file download request.')
-    ->label('sdk.response.type', '*/*')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.mock', true)
-    ->inject('response')
-    ->action(function (Response $response) {
-
-        $response
-            ->setContentType('text/plain')
-            ->addHeader('Content-Disposition', 'attachment; filename="test.txt"')
-            ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache
-            ->addHeader('X-Peak', \memory_get_peak_usage())
-            ->send("GET:/v1/mock/tests/general/download:passed")
-        ;
-    });
-
-App::post('/v1/mock/tests/general/upload')
-    ->desc('Upload File')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'general')
-    ->label('sdk.method', 'upload')
-    ->label('sdk.description', 'Mock a file upload request.')
-    ->label('sdk.request.type', 'multipart/form-data')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->param('x', '', new Text(100), 'Sample string param')
-    ->param('y', '', new Integer(true), 'Sample numeric param')
-    ->param('z', null, new ArrayList(new Text(256), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Sample array param')
-    ->param('file', [], new File(), 'Sample file param', skipValidation: true)
-    ->inject('request')
-    ->inject('response')
-    ->action(function (string $x, int $y, array $z, mixed $file, Request $request, Response $response) {
-
-        $file = $request->getFiles('file');
-
-        $contentRange = $request->getHeader('content-range');
-
-        $chunkSize = 5 * 1024 * 1024; // 5MB
-
-        if (!empty($contentRange)) {
-            $start = $request->getContentRangeStart();
-            $end = $request->getContentRangeEnd();
-            $size = $request->getContentRangeSize();
-            $id = $request->getHeader('x-appwrite-id', '');
-            $file['size'] = (\is_array($file['size'])) ? $file['size'][0] : $file['size'];
-
-            if (is_null($start) || is_null($end) || is_null($size) || $end >= $size) {
-                throw new Exception(Exception::GENERAL_MOCK, 'Invalid content-range header');
-            }
-
-            if ($start > $end || $end > $size) {
-                throw new Exception(Exception::GENERAL_MOCK, 'Invalid content-range header');
-            }
-
-            if ($start === 0 && !empty($id)) {
-                throw new Exception(Exception::GENERAL_MOCK, 'First chunked request cannot have id header');
-            }
-
-            if ($start !== 0 && $id !== 'newfileid') {
-                throw new Exception(Exception::GENERAL_MOCK, 'All chunked request must have id header (except first)');
-            }
-
-            if ($end !== $size - 1 && $end - $start + 1 !== $chunkSize) {
-                throw new Exception(Exception::GENERAL_MOCK, 'Chunk size must be 5MB (except last chunk)');
-            }
-
-            if ($end !== $size - 1 && $file['size'] !== $chunkSize) {
-                throw new Exception(Exception::GENERAL_MOCK, 'Wrong chunk size');
-            }
-
-            if ($file['size'] > $chunkSize) {
-                throw new Exception(Exception::GENERAL_MOCK, 'Chunk size must be 5MB or less');
-            }
-
-            if ($end !== $size - 1) {
-                $response->json([
-                    '$id' => ID::custom('newfileid'),
-                    'chunksTotal' => (int) ceil($size / ($end + 1 - $start)),
-                    'chunksUploaded' => ceil($start / $chunkSize) + 1
-                ]);
-            }
-        } else {
-            $file['tmp_name'] = (\is_array($file['tmp_name'])) ? $file['tmp_name'][0] : $file['tmp_name'];
-            $file['name'] = (\is_array($file['name'])) ? $file['name'][0] : $file['name'];
-            $file['size'] = (\is_array($file['size'])) ? $file['size'][0] : $file['size'];
-
-            if ($file['name'] !== 'file.png') {
-                throw new Exception(Exception::GENERAL_MOCK, 'Wrong file name');
-            }
-
-            if ($file['size'] !== 38756) {
-                    throw new Exception(Exception::GENERAL_MOCK, 'Wrong file size');
-            }
-
-            if (\md5(\file_get_contents($file['tmp_name'])) !== 'd80e7e6999a3eb2ae0d631a96fe135a4') {
-                throw new Exception(Exception::GENERAL_MOCK, 'Wrong file uploaded');
-            }
-        }
-    });
-
-App::get('/v1/mock/tests/general/redirect')
-    ->desc('Redirect')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'general')
-    ->label('sdk.method', 'redirect')
-    ->label('sdk.description', 'Mock a redirect request.')
-    ->label('sdk.response.code', Response::STATUS_CODE_MOVED_PERMANENTLY)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_HTML)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->inject('response')
-    ->action(function (Response $response) {
-
-        $response->redirect('/v1/mock/tests/general/redirect/done');
-    });
-
-App::get('/v1/mock/tests/general/redirect/done')
-    ->desc('Redirected')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'general')
-    ->label('sdk.method', 'redirected')
-    ->label('sdk.description', 'Mock a redirected request.')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->action(function () {
-    });
-
-App::get('/v1/mock/tests/general/set-cookie')
-    ->desc('Set Cookie')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'general')
-    ->label('sdk.method', 'setCookie')
-    ->label('sdk.description', 'Mock a set cookie request.')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->inject('response')
-    ->inject('request')
-    ->action(function (Response $response, Request $request) {
-
-        $response->addCookie('cookieName', 'cookieValue', \time() + 31536000, '/', $request->getHostname(), true, true);
-    });
-
-App::get('/v1/mock/tests/general/get-cookie')
-    ->desc('Get Cookie')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'general')
-    ->label('sdk.method', 'getCookie')
-    ->label('sdk.description', 'Mock a cookie response.')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->inject('request')
-    ->action(function (Request $request) {
-
-        if ($request->getCookie('cookieName', '') !== 'cookieValue') {
-            throw new Exception(Exception::GENERAL_MOCK, 'Missing cookie value');
-        }
-    });
-
-App::get('/v1/mock/tests/general/empty')
-    ->desc('Empty Response')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'general')
-    ->label('sdk.method', 'empty')
-    ->label('sdk.description', 'Mock an empty response.')
-    ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT)
-    ->label('sdk.response.model', Response::MODEL_NONE)
-    ->label('sdk.mock', true)
-    ->inject('response')
-    ->action(function (Response $response) {
-
-        $response->noContent();
-    });
-
-App::post('/v1/mock/tests/general/nullable')
-    ->desc('Nullable Test')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'general')
-    ->label('sdk.method', 'nullable')
-    ->label('sdk.description', 'Mock a nullable parameter.')
-    ->label('sdk.mock', true)
-    ->param('required', '', new Text(100), 'Sample string param')
-    ->param('nullable', '', new Nullable(new Text(100)), 'Sample string param')
-    ->param('optional', '', new Text(100), 'Sample string param', true)
-    ->action(function (string $required, string $nullable, ?string $optional) {
-    });
-
-App::post('/v1/mock/tests/general/enum')
-    ->desc('Enum Test')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'general')
-    ->label('sdk.method', 'enum')
-    ->label('sdk.description', 'Mock an enum parameter.')
-    ->label('sdk.mock', true)
-    ->param('mockType', '', new WhiteList(['first', 'second', 'third']), 'Sample enum param')
-    ->action(function (string $mockType) {
-    });
-
-App::get('/v1/mock/tests/general/400-error')
-    ->desc('400 Error')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'general')
-    ->label('sdk.method', 'error400')
-    ->label('sdk.description', 'Mock a 400 failed request.')
-    ->label('sdk.response.code', Response::STATUS_CODE_BAD_REQUEST)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_ERROR)
-    ->label('sdk.mock', true)
-    ->action(function () {
-        throw new Exception(Exception::GENERAL_MOCK, 'Mock 400 error');
-    });
-
-App::get('/v1/mock/tests/general/500-error')
-    ->desc('500 Error')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'general')
-    ->label('sdk.method', 'error500')
-    ->label('sdk.description', 'Mock a 500 failed request.')
-    ->label('sdk.response.code', Response::STATUS_CODE_INTERNAL_SERVER_ERROR)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_ERROR)
-    ->label('sdk.mock', true)
-    ->action(function () {
-        throw new Exception(Exception::GENERAL_MOCK, 'Mock 500 error', 500);
-    });
-
-App::get('/v1/mock/tests/general/502-error')
-    ->desc('502 Error')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.platform', [APP_PLATFORM_CLIENT, APP_PLATFORM_SERVER])
-    ->label('sdk.namespace', 'general')
-    ->label('sdk.method', 'error502')
-    ->label('sdk.description', 'Mock a 502 bad gateway.')
-    ->label('sdk.response.code', Response::STATUS_CODE_BAD_GATEWAY)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_TEXT)
-    ->label('sdk.response.model', Response::MODEL_ANY)
-    ->label('sdk.mock', true)
-    ->inject('response')
-    ->action(function (Response $response) {
-
-        $response
-            ->setStatusCode(502)
-            ->text('This is a text error')
-        ;
-    });
+use Utopia\VCS\Adapter\Git\GitHub;
+use Utopia\Database\Helpers\Permission;
+use Utopia\Database\Helpers\Role;
 
 App::get('/v1/mock/tests/general/oauth2')
     ->desc('OAuth Login')
@@ -646,6 +153,66 @@ App::patch('/v1/mock/functions-v2')
         $response->noContent();
     });
 
+App::get('/v1/mock/github/callback')
+    ->desc('Create installation document using GitHub installation id')
+    ->groups(['mock', 'api', 'vcs'])
+    ->label('scope', 'public')
+    ->label('docs', false)
+    ->param('providerInstallationId', '', new UID(), 'GitHub installation ID')
+    ->param('projectId', '', new UID(), 'Project ID of the project where app is to be installed')
+    ->inject('gitHub')
+    ->inject('project')
+    ->inject('response')
+    ->inject('dbForConsole')
+    ->action(function (string $providerInstallationId, string $projectId, GitHub $github, Document $project, Response $response, Database $dbForConsole) {
+        $isDevelopment = App::getEnv('_APP_ENV', 'development') === 'development';
+
+        if (!$isDevelopment) {
+            throw new Exception(Exception::GENERAL_NOT_IMPLEMENTED);
+        }
+
+        $project = $dbForConsole->getDocument('projects', $projectId);
+
+        if ($project->isEmpty()) {
+            $error = 'Project with the ID from state could not be found.';
+            throw new Exception(Exception::PROJECT_NOT_FOUND, $error);
+        }
+
+        if (!empty($providerInstallationId)) {
+            $privateKey = App::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY');
+            $githubAppId = App::getEnv('_APP_VCS_GITHUB_APP_ID');
+            $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId);
+            $owner = $github->getOwnerName($providerInstallationId) ?? '';
+
+            $projectInternalId = $project->getInternalId();
+
+            $teamId = $project->getAttribute('teamId', '');
+
+            $installation = new Document([
+                '$id' => ID::unique(),
+                '$permissions' => [
+                    Permission::read(Role::team(ID::custom($teamId))),
+                    Permission::update(Role::team(ID::custom($teamId), 'owner')),
+                    Permission::update(Role::team(ID::custom($teamId), 'developer')),
+                    Permission::delete(Role::team(ID::custom($teamId), 'owner')),
+                    Permission::delete(Role::team(ID::custom($teamId), 'developer')),
+                ],
+                'providerInstallationId' => $providerInstallationId,
+                'projectId' => $projectId,
+                'projectInternalId' => $projectInternalId,
+                'provider' => 'github',
+                'organization' => $owner,
+                'personal' => false
+            ]);
+
+            $installation = $dbForConsole->createDocument('installations', $installation);
+        }
+
+        $response->json([
+            'installationId' => $installation->getId(),
+        ]);
+    });
+
 App::shutdown()
     ->groups(['mock'])
     ->inject('utopia')

app/controllers/shared/api.php

@@ -8,8 +8,8 @@ use Appwrite\Event\Event;
 use Appwrite\Event\Func;
 use Appwrite\Event\Mail;
 use Appwrite\Extend\Exception;
+use Appwrite\Event\Usage;
 use Appwrite\Messaging\Adapter\Realtime;
-use Appwrite\Usage\Stats;
 use Appwrite\Utopia\Response;
 use Appwrite\Utopia\Request;
 use Utopia\App;
@@ -48,43 +48,95 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar
     return $label;
 };
 
-$databaseListener = function (string $event, Document $document, Stats $usage) {
-    $multiplier = 1;
+$databaseListener = function (string $event, Document $document, Document $project, Usage $queueForUsage, Database $dbForProject) {
+
+    $value = 1;
     if ($event === Database::EVENT_DOCUMENT_DELETE) {
-        $multiplier = -1;
+        $value = -1;
     }
 
-    $collection = $document->getCollection();
-    switch ($collection) {
-        case 'users':
-            $usage->setParam('users.{scope}.count.total', 1 * $multiplier);
+    switch (true) {
+        case $document->getCollection() === 'teams':
+            $queueForUsage
+                ->addMetric(METRIC_TEAMS, $value); // per project
             break;
-        case 'databases':
-            $usage->setParam('databases.{scope}.count.total', 1 * $multiplier);
+        case $document->getCollection() === 'users':
+            $queueForUsage
+                ->addMetric(METRIC_USERS, $value); // per project
+            if ($event === Database::EVENT_DOCUMENT_DELETE) {
+                $queueForUsage
+                    ->addReduce($document);
+            }
             break;
-        case 'buckets':
-            $usage->setParam('buckets.{scope}.count.total', 1 * $multiplier);
+        case $document->getCollection() === 'sessions': // sessions
+            $queueForUsage
+                ->addMetric(METRIC_SESSIONS, $value); //per project
             break;
-        case 'deployments':
-            $usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size') * $multiplier);
+        case $document->getCollection() === 'databases': // databases
+            $queueForUsage
+                ->addMetric(METRIC_DATABASES, $value); // per project
+
+            if ($event === Database::EVENT_DOCUMENT_DELETE) {
+                $queueForUsage
+                    ->addReduce($document);
+            }
+            break;
+        case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
+            $parts = explode('_', $document->getCollection());
+            $databaseInternalId = $parts[1] ?? 0;
+            $queueForUsage
+                ->addMetric(METRIC_COLLECTIONS, $value) // per project
+                ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database
+            ;
+
+            if ($event === Database::EVENT_DOCUMENT_DELETE) {
+                $queueForUsage
+                    ->addReduce($document);
+            }
+            break;
+        case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents
+            $parts = explode('_', $document->getCollection());
+            $databaseInternalId   = $parts[1] ?? 0;
+            $collectionInternalId = $parts[3] ?? 0;
+            $queueForUsage
+                ->addMetric(METRIC_DOCUMENTS, $value)  // per project
+                ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database
+                ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value);  // per collection
+            break;
+        case $document->getCollection() === 'buckets': //buckets
+            $queueForUsage
+                ->addMetric(METRIC_BUCKETS, $value); // per project
+            if ($event === Database::EVENT_DOCUMENT_DELETE) {
+                $queueForUsage
+                    ->addReduce($document);
+            }
+            break;
+        case str_starts_with($document->getCollection(), 'bucket_'): // files
+            $parts = explode('_', $document->getCollection());
+            $bucketInternalId  = $parts[1];
+            $queueForUsage
+                ->addMetric(METRIC_FILES, $value) // per project
+                ->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project
+                ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES), $value) // per bucket
+                ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket
+            break;
+        case $document->getCollection() === 'functions':
+            $queueForUsage
+                ->addMetric(METRIC_FUNCTIONS, $value); // per project
+
+            if ($event === Database::EVENT_DOCUMENT_DELETE) {
+                $queueForUsage
+                    ->addReduce($document);
+            }
+            break;
+        case $document->getCollection() === 'deployments':
+            $queueForUsage
+                ->addMetric(METRIC_DEPLOYMENTS, $value) // per project
+                ->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project
+                ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value)// per function
+                ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), $document->getAttribute('size') * $value);
             break;
         default:
-            if (strpos($collection, 'bucket_') === 0) {
-                $usage
-                    ->setParam('bucketId', $document->getAttribute('bucketId'))
-                    ->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal') * $multiplier)
-                    ->setParam('files.{scope}.count.total', 1 * $multiplier);
-            } elseif (strpos($collection, 'database_') === 0) {
-                $usage
-                    ->setParam('databaseId', $document->getAttribute('databaseId'));
-                if (strpos($collection, '_collection_') !== false) {
-                    $usage
-                        ->setParam('collectionId', $document->getAttribute('$collectionId'))
-                        ->setParam('documents.{scope}.count.total', 1 * $multiplier);
-                } else {
-                    $usage->setParam('collections.{scope}.count.total', 1 * $multiplier);
-                }
-            }
             break;
     }
 };
@@ -103,8 +155,8 @@ App::init()
     ->inject('dbForProject')
     ->inject('mode')
     ->inject('queueForMails')
-    ->inject('usage')
-    ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, string $mode, Mail $queueForMails, Stats $usage) use ($databaseListener) {
+    ->inject('queueForUsage')
+    ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, string $mode, Mail $queueForMails, Usage $queueForUsage) use ($databaseListener) {
 
         $route = $utopia->getRoute();
 
@@ -121,13 +173,16 @@ App::init()
         $abuseKeyLabel = (!is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel;
 
         foreach ($abuseKeyLabel as $abuseKey) {
+            $start = $request->getContentRangeStart();
+            $end = $request->getContentRangeEnd();
             $timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $dbForProject);
             $timeLimit
                 ->setParam('{userId}', $user->getId())
                 ->setParam('{userAgent}', $request->getUserAgent(''))
                 ->setParam('{ip}', $request->getIP())
                 ->setParam('{url}', $request->getHostname() . $route->getPath())
-                ->setParam('{method}', $request->getMethod());
+                ->setParam('{method}', $request->getMethod())
+                ->setParam('{chunkId}', (int) ($start / ($end + 1 - $start)));
             $timeLimitArray[] = $timeLimit;
         }
 
@@ -186,19 +241,14 @@ App::init()
             ->setProject($project)
             ->setUser($user);
 
-        $usage
-            ->setParam('projectInternalId', $project->getInternalId())
-            ->setParam('projectId', $project->getId())
-            ->setParam('project.{scope}.network.requests', 1)
-            ->setParam('httpMethod', $request->getMethod())
-            ->setParam('project.{scope}.network.inbound', 0)
-            ->setParam('project.{scope}.network.outbound', 0);
 
         $queueForDeletes->setProject($project);
         $queueForDatabase->setProject($project);
 
-        $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage));
-        $dbForProject->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage));
+        $dbForProject
+            ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject))
+            ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject))
+        ;
 
         $useCache = $route->getLabel('cache', false);
 
@@ -362,14 +412,14 @@ App::shutdown()
     ->inject('user')
     ->inject('queueForEvents')
     ->inject('queueForAudits')
-    ->inject('usage')
+    ->inject('queueForUsage')
     ->inject('queueForDeletes')
     ->inject('queueForDatabase')
     ->inject('dbForProject')
     ->inject('queueForFunctions')
     ->inject('mode')
     ->inject('dbForConsole')
-    ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Stats $usage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) {
+    ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) {
 
         $responsePayload = $response->getPayload();
 
@@ -522,35 +572,32 @@ App::shutdown()
             }
         }
 
-        if (
-            App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled'
-            && $project->getId()
-            && !empty($route->getLabel('sdk.namespace', null))
-        ) { // Don't calculate console usage on admin mode
-            $metric = $route->getLabel('usage.metric', '');
-            $usageParams = $route->getLabel('usage.params', []);
 
-            if (!empty($metric)) {
-                $usage->setParam($metric, 1);
-                foreach ($usageParams as $param) {
-                    $param = $parseLabel($param, $responsePayload, $requestParams, $user);
-                    $parts = explode(':', $param);
-                    if (count($parts) != 2) {
-                        throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Usage params not properly set');
-                    }
-                    $usage->setParam($parts[0], $parts[1]);
+
+        if ($project->getId() !== 'console') {
+            if ($mode !== APP_MODE_ADMIN) {
+                $fileSize = 0;
+                $file = $request->getFiles('file');
+                if (!empty($file)) {
+                    $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
                 }
+
+                $queueForUsage
+                    ->addMetric(METRIC_NETWORK_REQUESTS, 1)
+                    ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize)
+                    ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize());
             }
 
-            $fileSize = 0;
-            $file = $request->getFiles('file');
-            if (!empty($file)) {
-                $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
-            }
+            $queueForUsage
+                ->setProject($project)
+                ->trigger();
+        }
+    });
 
-            $usage
-                ->setParam('project.{scope}.network.inbound', $request->getSize() + $fileSize)
-                ->setParam('project.{scope}.network.outbound', $response->getSize())
-                ->submit();
+App::init()
+    ->groups(['usage'])
+    ->action(function () {
+        if (App::getEnv('_APP_USAGE_STATS', 'enabled') !== 'enabled') {
+            throw new Exception(Exception::GENERAL_USAGE_DISABLED);
         }
     });

app/init.php

@@ -19,6 +19,7 @@ ini_set('default_socket_timeout', -1);
 error_reporting(E_ALL);
 
 use Appwrite\Event\Migration;
+use Appwrite\Event\Usage;
 use Appwrite\Extend\Exception;
 use Appwrite\Auth\Auth;
 use Appwrite\Event\Audit;
@@ -32,7 +33,6 @@ use Appwrite\Network\Validator\Email;
 use Appwrite\Network\Validator\Origin;
 use Appwrite\OpenSSL\OpenSSL;
 use Appwrite\URL\URL as AppwriteURL;
-use Appwrite\Usage\Stats;
 use Utopia\App;
 use Utopia\Logger\Logger;
 use Utopia\Cache\Adapter\Redis as RedisCache;
@@ -72,6 +72,7 @@ use Ahc\Jwt\JWTException;
 use Appwrite\Event\Build;
 use Appwrite\Event\Certificate;
 use Appwrite\Event\Func;
+use Appwrite\Hooks\Hooks;
 use MaxMind\Db\Reader;
 use PHPMailer\PHPMailer\PHPMailer;
 use Swoole\Database\PDOProxy;
@@ -109,8 +110,8 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return
 const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
-const APP_CACHE_BUSTER = 514;
-const APP_VERSION_STABLE = '1.4.6';
+const APP_CACHE_BUSTER = 328;
+const APP_VERSION_STABLE = '1.4.13';
 const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
 const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
 const APP_DATABASE_ATTRIBUTE_IP = 'ip';
@@ -242,6 +243,7 @@ Config::load('platforms', __DIR__ . '/config/platforms.php');
 Config::load('collections', __DIR__ . '/config/collections.php');
 Config::load('runtimes', __DIR__ . '/config/runtimes.php');
 Config::load('runtimes-v2', __DIR__ . '/config/runtimes-v2.php');
+Config::load('usage', __DIR__ . '/config/usage.php');
 Config::load('roles', __DIR__ . '/config/roles.php');  // User roles and scopes
 Config::load('scopes', __DIR__ . '/config/scopes.php');  // User roles and scopes
 Config::load('services', __DIR__ . '/config/services.php');  // List of services
@@ -767,31 +769,26 @@ $register->set('pools', function () {
     return $group;
 });
 
-$register->set('influxdb', function () {
+$register->set('db', function () {
+    // This is usually for our workers or CLI commands scope
+    $dbHost = App::getEnv('_APP_DB_HOST', '');
+    $dbPort = App::getEnv('_APP_DB_PORT', '');
+    $dbUser = App::getEnv('_APP_DB_USER', '');
+    $dbPass = App::getEnv('_APP_DB_PASS', '');
+    $dbScheme = App::getEnv('_APP_DB_SCHEMA', '');
 
- // Register DB connection
-    $host = App::getEnv('_APP_INFLUXDB_HOST', '');
-    $port = App::getEnv('_APP_INFLUXDB_PORT', '');
+    $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array(
+        PDO::ATTR_TIMEOUT => 3, // Seconds
+        PDO::ATTR_PERSISTENT => true,
+        PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+        PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
+        PDO::ATTR_EMULATE_PREPARES => true,
+        PDO::ATTR_STRINGIFY_FETCHES => true,
+    ));
 
-    if (empty($host) || empty($port)) {
-        return;
-    }
-    $driver = new InfluxDB\Driver\Curl("http://{$host}:{$port}");
-    $client = new InfluxDB\Client($host, $port, '', '', false, false, 5);
-    $client->setDriver($driver);
-
-    return $client;
+    return $pdo;
 });
-$register->set('statsd', function () {
-    // Register DB connection
-    $host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
-    $port = App::getEnv('_APP_STATSD_PORT', 8125);
 
-    $connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port);
-    $statsd = new \Domnikl\Statsd\Client($connection);
-
-    return $statsd;
-});
 $register->set('smtp', function () {
     $mail = new PHPMailer(true);
 
@@ -832,6 +829,9 @@ $register->set('passwordsDictionary', function () {
 $register->set('promiseAdapter', function () {
     return new Swoole();
 });
+$register->set('hooks', function () {
+    return new Hooks();
+});
 /*
  * Localization
  */
@@ -871,6 +871,10 @@ App::setResource('logger', function ($register) {
     return $register->get('logger');
 }, ['register']);
 
+App::setResource('hooks', function ($register) {
+    return $register->get('hooks');
+}, ['register']);
+
 App::setResource('loggerBreadcrumbs', function () {
     return [];
 });
@@ -910,15 +914,15 @@ App::setResource('queueForAudits', function (Connection $queue) {
 App::setResource('queueForFunctions', function (Connection $queue) {
     return new Func($queue);
 }, ['queue']);
+App::setResource('queueForUsage', function (Connection $queue) {
+    return new Usage($queue);
+}, ['queue']);
 App::setResource('queueForCertificates', function (Connection $queue) {
     return new Certificate($queue);
 }, ['queue']);
 App::setResource('queueForMigrations', function (Connection $queue) {
     return new Migration($queue);
 }, ['queue']);
-App::setResource('usage', function ($register) {
-    return new Stats($register->get('statsd'));
-}, ['register']);
 App::setResource('clients', function ($request, $console, $project) {
     $console->setAttribute('platforms', [ // Always allow current host
         '$collection' => ID::custom('platforms'),
@@ -1049,6 +1053,9 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons
         }
     }
 
+    $dbForProject->setMetadata('user', $user->getId());
+    $dbForConsole->setMetadata('user', $user->getId());
+
     return $user;
 }, ['mode', 'project', 'console', 'request', 'response', 'dbForProject', 'dbForConsole']);
 
@@ -1122,6 +1129,8 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole,
 
     $database
         ->setNamespace('_' . $project->getInternalId())
+        ->setMetadata('host', \gethostname())
+        ->setMetadata('project', $project->getId())
         ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);
 
     return $database;
@@ -1138,6 +1147,8 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) {
 
     $database
         ->setNamespace('_console')
+        ->setMetadata('host', \gethostname())
+        ->setMetadata('project', 'console')
         ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);
 
     return $database;
@@ -1158,6 +1169,8 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
 
             $database
                 ->setNamespace('_' . $project->getInternalId())
+                ->setMetadata('host', \gethostname())
+                ->setMetadata('project', $project->getId())
                 ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);
 
             return $database;
@@ -1174,6 +1187,8 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
 
         $database
             ->setNamespace('_' . $project->getInternalId())
+            ->setMetadata('host', \gethostname())
+            ->setMetadata('project', $project->getId())
             ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS);
 
         return $database;

app/realtime.php

@@ -1,8 +1,10 @@
 <?php
 
 use Appwrite\Auth\Auth;
+use Appwrite\Extend\Exception;
 use Appwrite\Messaging\Adapter\Realtime;
 use Appwrite\Network\Validator\Origin;
+use Appwrite\Utopia\Request;
 use Appwrite\Utopia\Response;
 use Swoole\Http\Request as SwooleRequest;
 use Swoole\Http\Response as SwooleResponse;
@@ -20,7 +22,6 @@ use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
-use Appwrite\Utopia\Request;
 use Utopia\Cache\Adapter\Sharding;
 use Utopia\Cache\Cache;
 use Utopia\Config\Config;
@@ -28,6 +29,9 @@ use Utopia\Database\Database;
 use Utopia\WebSocket\Server;
 use Utopia\WebSocket\Adapter;
 
+/**
+ * @var \Utopia\Registry\Registry $register
+ */
 require_once __DIR__ . '/init.php';
 
 Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
@@ -47,7 +51,10 @@ function getConsoleDB(): Database
 
     $database = new Database($dbAdapter, getCache());
 
-    $database->setNamespace('_console');
+    $database
+        ->setNamespace('_console')
+        ->setMetadata('host', \gethostname())
+        ->setMetadata('project', '_console');
 
     return $database;
 }
@@ -70,7 +77,11 @@ function getProjectDB(Document $project): Database
     ;
 
     $database = new Database($dbAdapter, getCache());
-    $database->setNamespace('_' . $project->getInternalId());
+
+    $database
+        ->setNamespace('_' . $project->getInternalId())
+        ->setMetadata('host', \gethostname())
+        ->setMetadata('project', $project->getId());
 
     return $database;
 }
@@ -122,12 +133,12 @@ $server = new Server($adapter);
 $logError = function (Throwable $error, string $action) use ($register) {
     $logger = $register->get('logger');
 
-    if ($logger) {
+    if ($logger && !$error instanceof Exception) {
         $version = App::getEnv('_APP_VERSION', 'UNKNOWN');
 
         $log = new Log();
         $log->setNamespace("realtime");
-        $log->setServer(\gethostname());
+        $log->setServer(gethostname());
         $log->setVersion($version);
         $log->setType(Log::TYPE_ERROR);
         $log->setMessage($error->getMessage());
@@ -182,7 +193,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
 
                 $statsDocument = Authorization::skip(fn () => $database->createDocument('realtime', $document));
                 break;
-            } catch (\Throwable $th) {
+            } catch (Throwable) {
                 Console::warning("Collection not ready. Retrying connection ({$attempts})...");
                 sleep(DATABASE_RECONNECT_SLEEP);
             }
@@ -210,7 +221,7 @@ $server->onStart(function () use ($stats, $register, $containerId, &$statsDocume
                 ->setAttribute('value', json_encode($payload));
 
             Authorization::skip(fn () => $database->updateDocument('realtime', $statsDocument->getId(), $statsDocument));
-        } catch (\Throwable $th) {
+        } catch (Throwable $th) {
             call_user_func($logError, $th, "updateWorkerDocument");
         } finally {
             $register->get('pools')->reclaim();
@@ -362,7 +373,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats,
                     $stats->incr($event['project'], 'messages', $num);
                 }
             });
-        } catch (\Throwable $th) {
+        } catch (Throwable $th) {
             call_user_func($logError, $th, "pubSubConnection");
 
             Console::error('Pub/sub error: ' . $th->getMessage());
@@ -389,19 +400,19 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
     App::setResource('response', fn() => $response);
 
     try {
-        /** @var \Utopia\Database\Document $project */
+        /** @var Document $project */
         $project = $app->getResource('project');
 
         /*
          *  Project Check
          */
         if (empty($project->getId())) {
-            throw new Exception('Missing or unknown project ID', 1008);
+            throw new Exception(Exception::REALTIME_POLICY_VIOLATION, 'Missing or unknown project ID');
         }
 
         $dbForProject = getProjectDB($project);
-        $console = $app->getResource('console'); /** @var \Utopia\Database\Document $console */
-        $user = $app->getResource('user'); /** @var \Utopia\Database\Document $user */
+        $console = $app->getResource('console'); /** @var Document $console */
+        $user = $app->getResource('user'); /** @var Document $user */
 
         /*
          * Abuse Check
@@ -416,7 +427,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
         $abuse = new Abuse($timeLimit);
 
         if (App::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled' && $abuse->check()) {
-            throw new Exception('Too many requests', 1013);
+            throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many requests');
         }
 
         /*
@@ -425,10 +436,10 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
          * Skip this check for non-web platforms which are not required to send an origin header.
          */
         $origin = $request->getOrigin();
-        $originValidator = new Origin(\array_merge($project->getAttribute('platforms', []), $console->getAttribute('platforms', [])));
+        $originValidator = new Origin(array_merge($project->getAttribute('platforms', []), $console->getAttribute('platforms', [])));
 
         if (!$originValidator->isValid($origin) && $project->getId() !== 'console') {
-            throw new Exception($originValidator->getDescription(), 1008);
+            throw new Exception(Exception::REALTIME_POLICY_VIOLATION, $originValidator->getDescription());
         }
 
         $roles = Auth::getRoles($user);
@@ -439,7 +450,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
          * Channels Check
          */
         if (empty($channels)) {
-            throw new Exception('Missing channels', 1008);
+            throw new Exception(Exception::REALTIME_POLICY_VIOLATION, 'Missing channels');
         }
 
         $realtime->subscribe($project->getId(), $connection, $roles, $channels);
@@ -460,7 +471,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
         ]);
         $stats->incr($project->getId(), 'connections');
         $stats->incr($project->getId(), 'connectionsTotal');
-    } catch (\Throwable $th) {
+    } catch (Throwable $th) {
         call_user_func($logError, $th, "initServer");
 
         $response = [
@@ -486,7 +497,6 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server,
 
 $server->onMessage(function (int $connection, string $message) use ($server, $register, $realtime, $containerId) {
     try {
-        $app = new App('UTC');
         $response = new Response(new SwooleResponse());
         $projectId = $realtime->connections[$connection]['projectId'];
         $database = getConsoleDB();
@@ -494,6 +504,8 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
         if ($projectId !== 'console') {
             $project = Authorization::skip(fn() => $database->getDocument('projects', $projectId));
             $database = getProjectDB($project);
+        } else {
+            $project = null;
         }
 
         /*
@@ -510,22 +522,22 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
         $abuse = new Abuse($timeLimit);
 
         if ($abuse->check() && App::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') {
-            throw new Exception('Too many messages', 1013);
+            throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many messages.');
         }
 
         $message = json_decode($message, true);
 
         if (is_null($message) || (!array_key_exists('type', $message) && !array_key_exists('data', $message))) {
-            throw new Exception('Message format is not valid.', 1003);
+            throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Message format is not valid.');
         }
 
         switch ($message['type']) {
-                /**
+            /**
              * This type is used to authenticate.
              */
             case 'authentication':
                 if (!array_key_exists('session', $message['data'])) {
-                    throw new Exception('Payload is not valid.', 1003);
+                    throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Payload is not valid.');
                 }
 
                 $session = Auth::decodeSession($message['data']['session']);
@@ -540,7 +552,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
                     || !Auth::sessionVerify($user->getAttribute('sessions', []), Auth::$secret, $authDuration) // Validate user has valid login token
                 ) {
                     // cookie not valid
-                    throw new Exception('Session is not valid.', 1003);
+                    throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Session is not valid.');
                 }
 
                 $roles = Auth::getRoles($user);
@@ -560,9 +572,9 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re
                 break;
 
             default:
-                throw new Exception('Message type is not valid.', 1003);
+                throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Message type is not valid.');
         }
-    } catch (\Throwable $th) {
+    } catch (Throwable $th) {
         $response = [
             'type' => 'error',
             'data' => [

app/views/install/compose.phtml

@@ -71,7 +71,6 @@ services:
       - mariadb
       - redis
 #      - clamav
-      - influxdb
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
@@ -106,8 +105,6 @@ services:
       - _APP_SMTP_USERNAME
       - _APP_SMTP_PASSWORD
       - _APP_USAGE_STATS
-      - _APP_INFLUXDB_HOST
-      - _APP_INFLUXDB_PORT
       - _APP_STORAGE_LIMIT
       - _APP_STORAGE_PREVIEW_LIMIT
       - _APP_STORAGE_ANTIVIRUS
@@ -144,8 +141,6 @@ services:
       - _APP_EXECUTOR_HOST
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
-      - _APP_STATSD_HOST
-      - _APP_STATSD_PORT
       - _APP_MAINTENANCE_INTERVAL
       - _APP_MAINTENANCE_RETENTION_EXECUTION
       - _APP_MAINTENANCE_RETENTION_CACHE
@@ -595,16 +590,15 @@ services:
       - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
       - _APP_MAINTENANCE_RETENTION_SCHEDULES
 
-  appwrite-usage:
+  appwrite-worker-usage:
     image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
     entrypoint: usage
-    container_name: appwrite-usage
+    container_name: appwrite-worker-usage
     <<: *x-logging
     restart: unless-stopped
     networks:
       - appwrite
     depends_on:
-      - influxdb
       - mariadb
     environment:
       - _APP_ENV
@@ -615,9 +609,6 @@ services:
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_INFLUXDB_HOST
-      - _APP_INFLUXDB_PORT
-      - _APP_USAGE_AGGREGATION_INTERVAL
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
@@ -751,27 +742,6 @@ services:
   #   volumes:
   #     - appwrite-uploads:/storage/uploads
 
-  influxdb:
-    image: appwrite/influxdb:1.5.0
-    container_name: appwrite-influxdb
-    <<: *x-logging
-    restart: unless-stopped
-    networks:
-      - appwrite
-    volumes:
-      - appwrite-influxdb:/var/lib/influxdb:rw
-
-  telegraf:
-    image: appwrite/telegraf:1.4.0
-    container_name: appwrite-telegraf
-    <<: *x-logging
-    restart: unless-stopped
-    networks:
-      - appwrite
-    environment:
-      - _APP_INFLUXDB_HOST
-      - _APP_INFLUXDB_PORT
-
 networks:
   gateway:
     name: gateway
@@ -788,5 +758,4 @@ volumes:
   appwrite-certificates:
   appwrite-functions:
   appwrite-builds:
-  appwrite-influxdb:
   appwrite-config:

app/worker.php

@@ -10,11 +10,10 @@ use Appwrite\Event\Database as EventDatabase;
 use Appwrite\Event\Delete;
 use Appwrite\Event\Func;
 use Appwrite\Event\Mail;
-use Appwrite\Event\Messaging;
 use Appwrite\Event\Migration;
 use Appwrite\Event\Phone;
+use Appwrite\Event\Usage;
 use Appwrite\Platform\Appwrite;
-use Appwrite\Usage\Stats;
 use Swoole\Runtime;
 use Utopia\App;
 use Utopia\Cache\Adapter\Sharding;
@@ -22,6 +21,7 @@ use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
 use Utopia\Database\Database;
+use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Platform\Service;
@@ -36,7 +36,6 @@ use Utopia\Queue\Connection;
 Authorization::disable();
 Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
 
-
 Server::setResource('register', fn () => $register);
 
 Server::setResource('dbForConsole', function (Cache $cache, Registry $register) {
@@ -73,6 +72,17 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register,
     return $adapter;
 }, ['cache', 'register', 'message', 'dbForConsole']);
 
+Server::setResource('project', function (Message $message, Database $dbForConsole) {
+    $payload = $message->getPayload() ?? [];
+    $project = new Document($payload['project'] ?? []);
+
+    if ($project->getId() === 'console') {
+        return $project;
+    }
+    return $dbForConsole->getDocument('projects', $project->getId());
+    ;
+}, ['message', 'dbForConsole']);
+
 Server::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) {
     $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools
 
@@ -104,6 +114,18 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForConso
     };
 }, ['pools', 'dbForConsole', 'cache']);
 
+Server::setResource('abuseRetention', function () {
+    return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400));
+});
+
+Server::setResource('auditRetention', function () {
+    return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600));
+});
+
+Server::setResource('executionRetention', function () {
+    return DateTime::addSeconds(new \DateTime(), -1 * App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', 1209600));
+});
+
 Server::setResource('cache', function (Registry $register) {
     $pools = $register->get('pools');
     $list = Config::getParam('pools-cache', []);
@@ -120,9 +142,9 @@ Server::setResource('cache', function (Registry $register) {
     return new Cache(new Sharding($adapters));
 }, ['register']);
 Server::setResource('log', fn() => new Log());
-Server::setResource('usage', function ($register) {
-    return new Stats($register->get('statsd'));
-}, ['register']);
+Server::setResource('queueForUsage', function (Connection $queue) {
+    return new Usage($queue);
+}, ['queue']);
 Server::setResource('queue', function (Group $pools) {
     return $pools->get('queue')->pop()->getResource();
 }, ['pools']);
@@ -273,9 +295,12 @@ $worker
         Console::error('[Error] Line: ' . $error->getLine());
     });
 
-     $worker->workerStart()
-         ->action(function () use ($workerName) {
-             Console::info("Worker $workerName  started");
-         });
+try {
+    $workerStart = $worker->getWorkerStart();
+} catch (\Throwable $error) {
+    $worker->workerStart();
+} finally {
+    Console::info("Worker $workerName  started");
+}
 
-     $worker->start();
+$worker->start();

bin/create-inf-metric

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php create-inf-metric $@

bin/delete-orphaned-projects

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php delete-orphaned-projects $@

bin/get-migration-stats

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php get-migration-stats $@

bin/patch-recreate-repositories-documents

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php patch-recreate-repositories-documents $@

bin/worker-usage

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/worker.php usage $@

composer.json

@@ -43,39 +43,37 @@
         "ext-sockets": "*",
         "appwrite/php-runtimes": "0.13.*",
         "appwrite/php-clamav": "2.0.*",
-        "utopia-php/abuse": "0.32.*",
+        "utopia-php/abuse": "0.33.*",
         "utopia-php/analytics": "0.10.*",
-        "utopia-php/audit": "0.34.*",
+        "utopia-php/audit": "0.35.*",
         "utopia-php/cache": "0.8.*",
         "utopia-php/cli": "0.15.*",
         "utopia-php/config": "0.2.*",
-        "utopia-php/database": "0.44.*",
+        "utopia-php/database": "0.45.*",
         "utopia-php/domains": "0.3.*",
         "utopia-php/dsn": "0.1.*",
         "utopia-php/framework": "0.31.0",
         "utopia-php/image": "0.5.*",
         "utopia-php/locale": "0.4.*",
         "utopia-php/logger": "0.3.*",
-        "utopia-php/messaging": "0.1.*",
+        "utopia-php/messaging": "0.2.*",
         "utopia-php/migration": "0.3.*",
         "utopia-php/orchestration": "0.9.*",
         "utopia-php/platform": "0.5.*",
         "utopia-php/pools": "0.4.*",
         "utopia-php/preloader": "0.2.*",
-        "utopia-php/queue": "0.5.*",
+        "utopia-php/queue": "0.6.*",
         "utopia-php/registry": "0.5.*",
-        "utopia-php/storage": "0.17.*",
+        "utopia-php/storage": "0.18.*",
         "utopia-php/swoole": "0.5.*",
-        "utopia-php/vcs": "0.5.*",
+        "utopia-php/vcs": "0.6.*",
         "utopia-php/websocket": "0.1.*",
         "matomo/device-detector": "6.1.*",
         "dragonmantank/cron-expression": "3.3.2",
-        "influxdb/influxdb-php": "1.15.2",
         "phpmailer/phpmailer": "6.8.0",
         "chillerlan/php-qrcode": "4.3.4",
         "adhocore/jwt": "1.1.2",
         "webonyx/graphql-php": "14.11.*",
-        "slickdeals/statsd": "3.1.0",
         "league/csv": "9.7.1"
     },
     "repositories": [
@@ -86,7 +84,7 @@
     ],
     "require-dev": {
         "ext-fileinfo": "*",
-        "appwrite/sdk-generator": "0.35.*",
+        "appwrite/sdk-generator": "0.36.*",
         "phpunit/phpunit": "9.5.20",
         "squizlabs/php_codesniffer": "^3.7",
         "swoole/ide-helper": "5.0.2",

docker-compose.yml

@@ -127,8 +127,6 @@ services:
       - _APP_SMTP_USERNAME
       - _APP_SMTP_PASSWORD
       - _APP_USAGE_STATS
-      - _APP_INFLUXDB_HOST
-      - _APP_INFLUXDB_PORT
       - _APP_STORAGE_LIMIT
       - _APP_STORAGE_PREVIEW_LIMIT
       - _APP_STORAGE_ANTIVIRUS
@@ -165,8 +163,6 @@ services:
       - _APP_EXECUTOR_HOST
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
-      - _APP_STATSD_HOST
-      - _APP_STATSD_PORT
       - _APP_MAINTENANCE_INTERVAL
       - _APP_MAINTENANCE_RETENTION_EXECUTION
       - _APP_MAINTENANCE_RETENTION_CACHE
@@ -648,19 +644,18 @@ services:
       - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
       - _APP_MAINTENANCE_RETENTION_SCHEDULES
 
-  appwrite-usage:
-    entrypoint: usage
+  appwrite-worker-usage:
+    entrypoint: worker-usage
     <<: *x-logging
-    container_name: appwrite-usage
+    container_name: appwrite-worker-usage
     image: appwrite-dev
     networks:
       - appwrite
     volumes:
       - ./app:/usr/src/code/app
       - ./src:/usr/src/code/src
-      - ./dev:/usr/local/dev
     depends_on:
-      - influxdb
+      - redis
       - mariadb
     environment:
       - _APP_ENV
@@ -671,9 +666,6 @@ services:
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_INFLUXDB_HOST
-      - _APP_INFLUXDB_PORT
-      - _APP_USAGE_AGGREGATION_INTERVAL
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
@@ -843,26 +835,6 @@ services:
   #     - appwrite
   #   volumes:
   #     - appwrite-uploads:/storage/uploads
-
-  influxdb:
-    image: appwrite/influxdb:1.5.0
-    container_name: appwrite-influxdb
-    <<: *x-logging
-    networks:
-      - appwrite
-    volumes:
-      - appwrite-influxdb:/var/lib/influxdb:rw
-
-  telegraf:
-    image: appwrite/telegraf:1.4.0
-    container_name: appwrite-telegraf
-    <<: *x-logging
-    networks:
-      - appwrite
-    environment:
-      - _APP_INFLUXDB_HOST
-      - _APP_INFLUXDB_PORT
-
   # Dev Tools Start ------------------------------------------------------------------------------------------
   #
   # The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack
@@ -872,7 +844,6 @@ services:
   # MailCatcher - An SMTP server. Catches all system emails and displays them in a nice UI.
   # RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks
   # RedisCommander - A nice UI for exploring Redis data
-  # Chronograf - A nice UI for exploring InfluxDB data
   # Webgrind - A nice UI for exploring and debugging code-level stuff
 
   maildev: # used mainly for dev tests
@@ -912,26 +883,6 @@ services:
   #   - REDIS_HOSTS=redis
   #   ports:
   #   - "8081:8081"
-
-  # chronograf:
-  #   image: chronograf:1.6
-  #   container_name: appwrite-chronograf
-  #   restart: unless-stopped
-  #   networks:
-  #   - appwrite
-  #   volumes:
-  #   - appwrite-chronograf:/var/lib/chronograf
-  #   ports:
-  #   - "8888:8888"
-  #   environment:
-  #   - INFLUXDB_URL=http://influxdb:8086
-  #   - KAPACITOR_URL=http://kapacitor:9092
-  #   - AUTH_DURATION=48h
-  #   - TOKEN_SECRET=duperduper5674829!jwt
-  #   - GH_CLIENT_ID=d86f7145a41eacfc52cc
-  #   - GH_CLIENT_SECRET=9e0081062367a2134e7f2ea95ba1a32d08b6c8ab
-  #   - GH_ORGS=appwrite
-
   # webgrind:
   #   image: 'jokkedk/webgrind:latest'
   #   volumes:
@@ -969,6 +920,4 @@ volumes:
   appwrite-certificates:
   appwrite-functions:
   appwrite-builds:
-  appwrite-influxdb:
   appwrite-config:
-  # appwrite-chronograf:

docs/examples/1.4.x/console-cli/examples/health/get-queue-builds.md

@@ -1 +1,2 @@
-appwrite health getQueueBuilds
+appwrite health getQueueBuilds \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-certificates.md

@@ -1 +1,2 @@
-appwrite health getQueueCertificates
+appwrite health getQueueCertificates \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-databases.md

@@ -1,2 +1,3 @@
 appwrite health getQueueDatabases \
 
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-deletes.md

@@ -1 +1,2 @@
-appwrite health getQueueDeletes
+appwrite health getQueueDeletes \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-functions.md

@@ -1 +1,2 @@
-appwrite health getQueueFunctions
+appwrite health getQueueFunctions \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-logs.md

@@ -1 +1,2 @@
-appwrite health getQueueLogs
+appwrite health getQueueLogs \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-mails.md

@@ -1 +1,2 @@
-appwrite health getQueueMails
+appwrite health getQueueMails \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-messaging.md

@@ -1 +1,2 @@
-appwrite health getQueueMessaging
+appwrite health getQueueMessaging \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-migrations.md

@@ -1 +1,2 @@
-appwrite health getQueueMigrations
+appwrite health getQueueMigrations \
+

docs/examples/1.4.x/console-cli/examples/health/get-queue-webhooks.md

@@ -1 +1,2 @@
-appwrite health getQueueWebhooks
+appwrite health getQueueWebhooks \
+

docs/sdks/dart/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 10.1.0
+
+* Add new queue health endpoints
+* Fix between queries
+
 ## 10.0.0
 
 * Parameter `url` is now optional in the `createMembership` endpoint

docs/sdks/flutter/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 11.0.1
+
+* Fix between queries
+
 ## 11.0.0
 
 * Parameter `url` is now optional in the `createMembership` endpoint

src/Appwrite/Auth/Validator/PasswordHistory.php

@@ -44,7 +44,7 @@ class PasswordHistory extends Password
     public function isValid($value): bool
     {
         foreach ($this->history as $hash) {
-            if (Auth::passwordVerify($value, $hash, $this->algo, $this->algoOptions)) {
+            if (!empty($hash) && Auth::passwordVerify($value, $hash, $this->algo, $this->algoOptions)) {
                 return false;
             }
         }

src/Appwrite/Event/Mail.php

@@ -13,6 +13,8 @@ class Mail extends Event
     protected string $body = '';
     protected array $smtp = [];
     protected array $variables = [];
+    protected string $bodyTemplate = '';
+    protected array $attachment = [];
 
     public function __construct(protected Connection $connection)
     {
@@ -115,6 +117,29 @@ class Mail extends Event
         return $this->name;
     }
 
+    /**
+     * Sets bodyTemplate for the mail event.
+     *
+     * @param string $bodyTemplate
+     * @return self
+     */
+    public function setbodyTemplate(string $bodyTemplate): self
+    {
+        $this->bodyTemplate = $bodyTemplate;
+
+        return $this;
+    }
+
+    /**
+     * Returns subject for the mail event.
+     *
+     * @return string
+     */
+    public function getbodyTemplate(): string
+    {
+        return $this->bodyTemplate;
+    }
+
     /**
      * Set SMTP Host
      *
@@ -313,6 +338,22 @@ class Mail extends Event
         return $this;
     }
 
+    public function setAttachment(string $content, string $filename, string $encoding = 'base64', string $type = 'plain/text')
+    {
+        $this->attachment = [
+            'content' => base64_encode($content),
+            'filename' => $filename,
+            'encoding' => $encoding,
+            'type' => $type,
+        ];
+        return $this;
+    }
+
+    public function getAttachment(): array
+    {
+        return $this->attachment;
+    }
+
     /**
      * Executes the event and sends it to the mails worker.
      *
@@ -327,9 +368,11 @@ class Mail extends Event
             'recipient' => $this->recipient,
             'name' => $this->name,
             'subject' => $this->subject,
+            'bodyTemplate' => $this->bodyTemplate,
             'body' => $this->body,
             'smtp' => $this->smtp,
             'variables' => $this->variables,
+            'attachment' => $this->attachment,
             'events' => Event::generateEvents($this->getEvent(), $this->getParams())
         ]);
     }

src/Appwrite/Extend/Exception.php

@@ -84,6 +84,8 @@ class Exception extends \Exception
     public const USER_OAUTH2_BAD_REQUEST           = 'user_oauth2_bad_request';
     public const USER_OAUTH2_UNAUTHORIZED          = 'user_oauth2_unauthorized';
     public const USER_OAUTH2_PROVIDER_ERROR        = 'user_oauth2_provider_error';
+    public const USER_EMAIL_ALREADY_VERIFIED        = 'user_email_alread_verified';
+    public const USER_PHONE_ALREADY_VERIFIED        = 'user_phone_already_verified';
 
     /** Teams */
     public const TEAM_NOT_FOUND                    = 'team_not_found';
@@ -224,9 +226,19 @@ class Exception extends \Exception
     public const MIGRATION_NOT_FOUND                 = 'migration_not_found';
     public const MIGRATION_ALREADY_EXISTS            = 'migration_already_exists';
     public const MIGRATION_IN_PROGRESS               = 'migration_in_progress';
+    public const MIGRATION_PROVIDER_ERROR            = 'migration_provider_error';
 
-    protected $type = '';
-    protected $errors = [];
+    /** Realtime */
+    public const REALTIME_MESSAGE_FORMAT_INVALID    = 'realtime_message_format_invalid';
+    public const REALTIME_TOO_MANY_MESSAGES         = 'realtime_too_many_messages';
+    public const REALTIME_POLICY_VIOLATION          = 'realtime_policy_violation';
+
+    /** Health */
+    public const QUEUE_SIZE_EXCEEDED                 = 'queue_size_exceeded';
+
+    protected string $type = '';
+    protected array $errors = [];
+    protected bool $publish = true;
 
     public function __construct(string $type = Exception::GENERAL_UNKNOWN, string $message = null, int $code = null, \Throwable $previous = null)
     {
@@ -236,6 +248,7 @@ class Exception extends \Exception
         if (isset($this->errors[$type])) {
             $this->code = $this->errors[$type]['code'];
             $this->message = $this->errors[$type]['description'];
+            $this->publish = $this->errors[$type]['publish'] ?? true;
         }
 
         $this->message = $message ?? $this->message;
@@ -265,4 +278,14 @@ class Exception extends \Exception
     {
         $this->type = $type;
     }
+
+    /**
+     * Check whether the log is publishable for the exception.
+     *
+     * @return bool
+     */
+    public function isPublishable(): bool
+    {
+        return $this->publish;
+    }
 }

src/Appwrite/Hooks/Hooks.php

@@ -0,0 +1,26 @@
+<?php
+
+namespace Appwrite\Hooks;
+
+class Hooks
+{
+    /**
+     * @var callable[] $hooks
+     */
+    private static array $hooks = [];
+
+    public static function add(string $name, callable $action)
+    {
+        self::$hooks[$name] = $action;
+    }
+
+    /**
+     * @param mixed[] $params
+     */
+    public function trigger(string $name, array $params = [])
+    {
+        if (isset(self::$hooks[$name])) {
+            call_user_func_array(self::$hooks[$name], $params);
+        }
+    }
+}

src/Appwrite/Migration/Migration.php

@@ -71,6 +71,14 @@ abstract class Migration
         '1.4.4' => 'V19',
         '1.4.5' => 'V19',
         '1.4.6' => 'V19',
+        '1.4.7' => 'V19',
+        '1.4.8' => 'V19',
+        '1.4.9' => 'V19',
+        '1.4.10' => 'V19',
+        '1.4.11' => 'V19',
+        '1.4.12' => 'V19',
+        '1.4.13' => 'V19',
+        '1.4.14' => 'V20'
     ];
 
     /**
@@ -188,17 +196,24 @@ abstract class Migration
      * @return iterable<Document>
      * @throws \Exception
      */
-    public function documentsIterator(string $collectionId): iterable
+    public function documentsIterator(string $collectionId, $queries = []): iterable
     {
         $sum = 0;
         $nextDocument = null;
         $collectionCount = $this->projectDB->count($collectionId);
+        $queries[] = Query::limit($this->limit);
 
         do {
-            $queries = [Query::limit($this->limit)];
             if ($nextDocument !== null) {
-                $queries[] = Query::cursorAfter($nextDocument);
+                $cursorQueryIndex = \array_search('cursorAfter', \array_map(fn (Query $query) => $query->getMethod(), $queries));
+
+                if ($cursorQueryIndex !== false) {
+                    $queries[$cursorQueryIndex] = Query::cursorAfter($nextDocument);
+                } else {
+                    $queries[] = Query::cursorAfter($nextDocument);
+                }
             }
+
             $documents = $this->projectDB->find($collectionId, $queries);
             $count = count($documents);
             $sum += $count;

src/Appwrite/Migration/Version/V19.php

@@ -10,6 +10,7 @@ use Utopia\Database\Database;
 use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Database\Exception;
+use Utopia\Database\Query;
 
 class V19 extends Migration
 {
@@ -41,6 +42,11 @@ class V19 extends Migration
         Console::info('Migrating Buckets');
         $this->migrateBuckets();
 
+        if ($this->project->getId() !== 'console') {
+            Console::info('Migrating Enum Attribute Size');
+            $this->migrateEnumAttributeSize();
+        }
+
         Console::info('Migrating Documents');
         $this->forEachDocument([$this, 'fixDocument']);
 
@@ -640,6 +646,22 @@ class V19 extends Migration
         return $commands;
     }
 
+    private function migrateEnumAttributeSize(): void
+    {
+        foreach (
+            $this->documentsIterator('attributes', [
+            Query::equal('format', ['enum']),
+            Query::lessThan('size', Database::LENGTH_KEY)
+            ]) as $attribute
+        ) {
+            $attribute->setAttribute('size', Database::LENGTH_KEY);
+            $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute);
+            $databaseInternalId = $attribute->getAttribute('databaseInternalId');
+            $collectionInternalId = $attribute->getAttribute('collectionInternalId');
+            $this->projectDB->updateAttribute('database_' . $databaseInternalId . '_collection_' . $collectionInternalId, $attribute->getAttribute('key'), size: 255);
+        }
+    }
+
     /**
      * Fix run on each document
      *

src/Appwrite/Migration/Version/V20.php

@@ -0,0 +1,323 @@
+<?php
+
+namespace Appwrite\Migration\Version;
+
+use Appwrite\Migration\Migration;
+use PDOException;
+use Throwable;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Database\Document;
+use Utopia\Database\Exception;
+use Utopia\Database\Exception\Authorization;
+use Utopia\Database\Exception\Duplicate;
+use Utopia\Database\Exception\Structure;
+use Utopia\Database\Query;
+
+class V20 extends Migration
+{
+    /**
+     * @throws Throwable
+     */
+    public function execute(): void
+    {
+        if ($this->project->getInternalId() == 'console') {
+            return;
+        }
+
+        /**
+         * Disable SubQueries for Performance.
+         */
+        foreach (['subQueryIndexes', 'subQueryPlatforms', 'subQueryDomains', 'subQueryKeys', 'subQueryWebhooks', 'subQuerySessions', 'subQueryTokens', 'subQueryMemberships', 'subQueryVariables'] as $name) {
+            Database::addFilter(
+                $name,
+                fn () => null,
+                fn () => []
+            );
+        }
+
+        $this->migrateUsageMetrics('project.$all.network.requests', 'network.requests');
+        $this->migrateUsageMetrics('project.$all.network.outbound', 'network.outbound');
+        $this->migrateUsageMetrics('project.$all.network.inbound', 'network.inbound');
+        $this->migrateUsageMetrics('users.$all.count.total', 'users');
+        $this->migrateSessionsMetric();
+
+        Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')');
+        $this->projectDB->setNamespace("_{$this->project->getInternalId()}");
+
+        Console::info('Migrating Functions');
+        $this->migrateFunctions();
+
+        Console::info('Migrating Databases');
+        $this->migrateDatabases();
+
+        Console::info('Migrating Collections');
+        $this->migrateCollections();
+
+        Console::info('Migrating Buckets');
+        $this->migrateBuckets();
+    }
+
+    /**
+     * @return void
+     * @throws Authorization
+     * @throws Exception
+     * @throws Structure
+     */
+    protected function migrateSessionsMetric(): void
+    {
+        /**
+         * Creating inf metric
+         */
+
+        Console::info('Migrating Sessions metric');
+
+        $sessionsCreated = $this->projectDB->sum('stats', 'value', [
+            Query::equal('metric', [
+                'sessions.email-password.requests.create',
+                'sessions.magic-url.requests.create',
+                'sessions.anonymous.requests.create',
+                'sessions.invites.requests.create',
+                'sessions.jwt.requests.create',
+                'sessions.phone.requests.create'
+            ]),
+            Query::equal('period', ['1d']),
+        ]);
+
+        $query = $this->projectDB->findOne('stats', [
+            Query::equal('metric', ['sessions.$all.requests.delete']),
+            Query::equal('period', ['1d']),
+        ]);
+
+        $sessionsDeleted =  $query['value'] ?? 0;
+        $value = $sessionsCreated - $sessionsDeleted;
+        $this->createInfMetric('sessions', $value);
+    }
+
+    /**
+     * @param string $metric
+     * @param int $value
+     * @return void
+     * @throws Exception
+     * @throws Authorization
+     * @throws Structure
+     */
+    protected function createInfMetric(string $metric, int $value): void
+    {
+
+        try {
+            /**
+             * Creating inf metric
+             */
+            console::log("Creating inf metric to {$metric}");
+            $id = \md5("_inf_{$metric}");
+            $this->projectDB->createDocument('stats', new Document([
+                '$id' => $id,
+                'metric' => $metric,
+                'period' => 'inf',
+                'value'  => $value,
+                'time'   => null,
+                'region' => 'default',
+            ]));
+        } catch (Duplicate $th) {
+            console::log("Error while creating inf metric: duplicate id {$metric}  {$id}");
+        }
+    }
+
+    /**
+     * @param string $from
+     * @param string $to
+     * @return void
+     * @throws Exception
+     */
+    protected function migrateUsageMetrics(string $from, string $to): void
+    {
+        /**
+         * inf metric
+         */
+        if (
+            str_contains($from, '$all') ||
+            str_contains($from, '.total')
+        ) {
+            $query = $this->projectDB->sum('stats', 'value', [
+                Query::equal('metric', [$from]),
+                Query::equal('period', ['1d']),
+            ]);
+
+            $value = $query ?? 0;
+            $this->createInfMetric($to, $value);
+        }
+
+        try {
+            /**
+             * Update old metric format to new
+             */
+            $limit = 1000;
+            $sum = $limit;
+            $total = 0;
+            $latestDocument = null;
+            while ($sum === $limit) {
+                $paginationQueries = [Query::limit($limit)];
+                if ($latestDocument !== null) {
+                    $paginationQueries[] =  Query::cursorAfter($latestDocument);
+                }
+                $stats = $this->projectDB->find('stats', \array_merge($paginationQueries, [
+                    Query::equal('metric', [$from]),
+                ]));
+
+                $sum = count($stats);
+                $total = $total + $sum;
+                foreach ($stats as $stat) {
+                    $format = $stat['period'] === '1d' ? 'Y-m-d 00:00' : 'Y-m-d H:00';
+                    $time = date($format, strtotime($stat['time']));
+                    $this->projectDB->deleteDocument('stats', $stat->getId());
+                    $stat->setAttribute('$id', \md5("{$time}_{$stat['period']}_{$to}"));
+                    $stat->setAttribute('metric', $to);
+                    $this->projectDB->createDocument('stats', $stat);
+                    console::log("deleting metric {$from} and creating {$to}");
+                }
+                $latestDocument = !empty(array_key_last($stats)) ? $stats[array_key_last($stats)] : null;
+            }
+        } catch (Throwable $th) {
+            Console::warning("Error while updating metric  {$from}  " . $th->getMessage());
+        }
+    }
+    /**
+     * Migrate functions.
+     *
+     * @return void
+     * @throws \Exception
+     */
+    private function migrateFunctions(): void
+    {
+
+        $this->migrateUsageMetrics('deployment.$all.storage.size', 'deployments.storage');
+        $this->migrateUsageMetrics('builds.$all.compute.total', 'builds');
+        $this->migrateUsageMetrics('builds.$all.compute.time', 'builds.compute');
+        $this->migrateUsageMetrics('executions.$all.compute.total', 'executions');
+        $this->migrateUsageMetrics('executions.$all.compute.time', 'executions.compute');
+
+        foreach ($this->documentsIterator('functions') as $function) {
+            Console::log("Migrating Functions usage stats of {$function->getId()} ({$function->getAttribute('name')})");
+
+            $functionId = $function->getId();
+            $functionInternalId = $function->getInternalId();
+
+            $this->migrateUsageMetrics("deployment.$functionId.storage.size", "function.$functionInternalId.deployments.storage");
+            $this->migrateUsageMetrics("builds.$functionId.compute.total", "$functionInternalId.builds");
+            $this->migrateUsageMetrics("builds.$functionId.compute.time", "$functionInternalId.builds.compute");
+            $this->migrateUsageMetrics("executions.$functionId.compute.total", "$functionInternalId.executions");
+            $this->migrateUsageMetrics("executions.$functionId.compute.time", "$functionInternalId.executions.compute");
+        }
+    }
+
+    /**
+     * Migrate  Databases.
+     *
+     * @return void
+     * @throws \Exception
+     */
+    private function migrateDatabases(): void
+    {
+        // Project level
+        $this->migrateUsageMetrics('databases.$all.count.total', 'databases');
+        $this->migrateUsageMetrics('collections.$all.count.total', 'collections');
+        $this->migrateUsageMetrics('documents.$all.count.total', 'documents');
+
+        foreach ($this->documentsIterator('databases') as $database) {
+            Console::log("Migrating Collections of {$database->getId()} ({$database->getAttribute('name')})");
+
+            $databaseTable = "database_{$database->getInternalId()}";
+
+            // Database level
+            $databaseId = $database->getId();
+            $databaseInternalId = $database->getInternalId();
+
+            $this->migrateUsageMetrics("collections.$databaseId.count.total", "$databaseInternalId.collections");
+            $this->migrateUsageMetrics("documents.$databaseId.count.total", "$databaseInternalId.documents");
+
+            foreach ($this->documentsIterator($databaseTable) as $collection) {
+                $collectionTable = "{$databaseTable}_collection_{$collection->getInternalId()}";
+                Console::log("Migrating Collections of {$collectionTable} {$collection->getId()} ({$collection->getAttribute('name')})");
+
+                // Collection level
+                $collectionId =  $collection->getId() ;
+                $collectionInternalId =  $collection->getInternalId();
+
+                $this->migrateUsageMetrics("documents.$databaseId/$collectionId.count.total", "$databaseInternalId.$collectionInternalId.documents");
+            }
+        }
+    }
+
+    /**
+     * Migrate Collections.
+     *
+     * @return void
+     * @throws \Exception
+     */
+    private function migrateCollections(): void
+    {
+        $internalProjectId = $this->project->getInternalId();
+        $collectionType = match ($internalProjectId) {
+            'console' => 'console',
+            default => 'projects',
+        };
+
+        $collections = $this->collections[$collectionType];
+
+        foreach ($collections as $collection) {
+            $id = $collection['$id'];
+
+            Console::log("Migrating Collection \"{$id}\"");
+
+            $this->projectDB->setNamespace("_$internalProjectId");
+
+            switch ($id) {
+                case 'stats':
+                    try {
+                        /**
+                         * Delete 'type' attribute
+                         */
+                        $this->projectDB->deleteAttribute($id, 'type');
+                        /**
+                         * Alter `signed`  internal type on `value` attr
+                         */
+                        $this->projectDB->updateAttribute($id, 'value', null, null, null, null, true);
+                        $this->projectDB->deleteCachedCollection($id);
+                    } catch (Throwable $th) {
+                        Console::warning("'type' from {$id}: {$th->getMessage()}");
+                    }
+                    break;
+            }
+        }
+    }
+
+    /**
+     * Migrating all Bucket tables.
+     *
+     * @return void
+     * @throws \Exception
+     * @throws PDOException
+     */
+    protected function migrateBuckets(): void
+    {
+        // Project level
+        $this->migrateUsageMetrics('buckets.$all.count.total', 'buckets');
+        $this->migrateUsageMetrics('files.$all.count.total', 'files');
+        $this->migrateUsageMetrics('files.$all.storage.size', 'files.storage');
+        // There is also project.$all.storage.size which is the same as  files.$all.storage.size
+
+        foreach ($this->documentsIterator('buckets') as $bucket) {
+            $id = "bucket_{$bucket->getInternalId()}";
+            Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})");
+
+            // Bucket level
+            $bucketId = $bucket->getId();
+            $bucketInternalId = $bucket->getInternalId();
+
+             $this->migrateUsageMetrics("files.$bucketId.count.total", "$bucketInternalId.files");
+            $this->migrateUsageMetrics("files.$bucketId.storage.size", "$bucketInternalId.files.storage");
+            // some stats come with $ prefix in front of the id -> files.$650c3fda307b7fec4934.storage.size;
+        }
+    }
+}

src/Appwrite/Platform/Services/Tasks.php

@@ -12,12 +12,15 @@ use Appwrite\Platform\Tasks\SDKs;
 use Appwrite\Platform\Tasks\Specs;
 use Appwrite\Platform\Tasks\SSL;
 use Appwrite\Platform\Tasks\Hamster;
-use Appwrite\Platform\Tasks\Usage;
 use Appwrite\Platform\Tasks\Vars;
 use Appwrite\Platform\Tasks\Version;
 use Appwrite\Platform\Tasks\VolumeSync;
 use Appwrite\Platform\Tasks\CalcTierStats;
 use Appwrite\Platform\Tasks\Upgrade;
+use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
+use Appwrite\Platform\Tasks\GetMigrationStats;
+use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments;
+use Appwrite\Platform\Tasks\CreateInfMetric;
 
 class Tasks extends Service
 {
@@ -26,7 +29,6 @@ class Tasks extends Service
         $this->type = self::TYPE_CLI;
         $this
             ->addAction(Version::getName(), new Version())
-            ->addAction(Usage::getName(), new Usage())
             ->addAction(Vars::getName(), new Vars())
             ->addAction(SSL::getName(), new SSL())
             ->addAction(Hamster::getName(), new Hamster())
@@ -40,6 +42,11 @@ class Tasks extends Service
             ->addAction(VolumeSync::getName(), new VolumeSync())
             ->addAction(Specs::getName(), new Specs())
             ->addAction(CalcTierStats::getName(), new CalcTierStats())
+            ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects())
+            ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments())
+            ->addAction(GetMigrationStats::getName(), new GetMigrationStats())
+            ->addAction(CreateInfMetric::getName(), new CreateInfMetric())
+
         ;
     }
 }

src/Appwrite/Platform/Services/Workers.php

@@ -12,6 +12,8 @@ use Appwrite\Platform\Workers\Databases;
 use Appwrite\Platform\Workers\Functions;
 use Appwrite\Platform\Workers\Builds;
 use Appwrite\Platform\Workers\Deletes;
+use Appwrite\Platform\Workers\Usage;
+use Appwrite\Platform\Workers\UsageHook;
 use Appwrite\Platform\Workers\Migrations;
 
 class Workers extends Service
@@ -29,6 +31,8 @@ class Workers extends Service
             ->addAction(Functions::getName(), new Functions())
             ->addAction(Builds::getName(), new Builds())
             ->addAction(Deletes::getName(), new Deletes())
+            ->addAction(UsageHook::getName(), new UsageHook())
+            ->addAction(Usage::getName(), new Usage())
             ->addAction(Migrations::getName(), new Migrations())
 
         ;

src/Appwrite/Platform/Tasks/CalcTierStats.php

@@ -5,6 +5,7 @@ namespace Appwrite\Platform\Tasks;
 use Exception;
 use League\Csv\CannotInsertRecord;
 use Utopia\App;
+use Utopia\Database\Document;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Platform\Action;
 use Utopia\Cache\Cache;
@@ -15,6 +16,7 @@ use League\Csv\Writer;
 use PHPMailer\PHPMailer\PHPMailer;
 use Utopia\Pools\Group;
 use Utopia\Registry\Registry;
+use Utopia\Validator\Text;
 
 class CalcTierStats extends Action
 {
@@ -24,6 +26,7 @@ class CalcTierStats extends Action
     private array $columns = [
         'Project ID',
         'Organization ID',
+        'Organization Email',
         'Organization Members',
         'Teams',
         'Users',
@@ -41,6 +44,7 @@ class CalcTierStats extends Action
         'Functions',
         'Deployments',
         'Executions',
+        'Migrations',
     ];
 
     protected string $directory = '/usr/local';
@@ -48,8 +52,9 @@ class CalcTierStats extends Action
     protected string $date;
 
     private array $usageStats = [
-        'project.$all.network.requests'  => 'Requests',
-        'project.$all.network.bandwidth' => 'Bandwidth',
+        'network.requests'  => 'Requests',
+        'network.inbound' => 'Inbound',
+        'network.outbound' => 'Outbound',
 
     ];
 
@@ -63,263 +68,112 @@ class CalcTierStats extends Action
 
         $this
             ->desc('Get stats for projects')
+            ->param('after', '', new Text(36), 'After cursor', true)
+            ->param('projectId', '', new Text(36), 'Select project to validate', true)
             ->inject('pools')
             ->inject('cache')
             ->inject('dbForConsole')
+            ->inject('getProjectDB')
             ->inject('register')
-            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
-                $this->action($pools, $cache, $dbForConsole, $register);
+            ->callback(function ($after, $projectId, Group $pools, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register) {
+                $this->action($after, $projectId, $pools, $cache, $dbForConsole, $getProjectDB, $register);
             });
     }
 
-    /**
-     * @throws \Utopia\Exception
-     * @throws CannotInsertRecord
-     */
-    public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
+
+    public function action(string $after, string $projectId, Group $pools, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register): void
     {
         //docker compose exec -t appwrite calc-tier-stats
 
         Console::title('Cloud free tier  stats calculation V1');
         Console::success(APP_NAME . ' cloud free tier  stats calculation has started');
 
-        /* Initialise new Utopia app */
-        $app = new App('UTC');
-        $console = $app->getResource('console');
-
         /** CSV stuff */
         $this->date = date('Y-m-d');
         $this->path = "{$this->directory}/tier_stats_{$this->date}.csv";
         $csv = Writer::createFromPath($this->path, 'w');
         $csv->insertOne($this->columns);
 
-        /** Database connections */
-        $totalProjects = $dbForConsole->count('projects');
-        Console::success("Found a total of: {$totalProjects} projects");
-
-        $projects = [$console];
-        $count = 0;
-        $limit = 30;
-        $sum = 30;
-        $offset = 0;
-        while (!empty($projects)) {
-            foreach ($projects as $project) {
-
-                /**
-                 * Skip user projects with id 'console'
-                 */
-                if ($project->getId() === 'console') {
-                    continue;
-                }
-
-                Console::info("Getting stats for {$project->getId()}");
+        if (!empty($projectId)) {
+            try {
+                console::log("Project " . $projectId);
+                $project = $dbForConsole->getDocument('projects', $projectId);
+                $dbForProject = call_user_func($getProjectDB, $project);
+                $data = $this->getData($project, $dbForConsole, $dbForProject);
+                $csv->insertOne($data);
+                $this->sendMail($register);
 
+                return;
+            } catch (\Throwable $th) {
+                Console::error("Unexpected error occured with Project ID {$projectId}");
+                Console::error('[Error] Type: ' . get_class($th));
+                Console::error('[Error] Message: ' . $th->getMessage());
+                Console::error('[Error] File: ' . $th->getFile());
+                Console::error('[Error] Line: ' . $th->getLine());
+            }
+        }
+            $queries = [];
+        if (!empty($after)) {
+            Console::info("Iterating remaining projects after project with ID {$after}");
+            $project = $dbForConsole->getDocument('projects', $after);
+            $queries = [Query::cursorAfter($project)];
+        } else {
+            Console::info("Iterating all projects");
+        }
+            $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole, $csv) {
+                $projectId = $project->getId();
+                console::log("Project " . $projectId);
                 try {
-                    $db = $project->getAttribute('database');
-                    $adapter = $pools
-                        ->get($db)
-                        ->pop()
-                        ->getResource();
-
-                    $dbForProject = new Database($adapter, $cache);
-                    $dbForProject->setDefaultDatabase('appwrite');
-                    $dbForProject->setNamespace('_' . $project->getInternalId());
-
-                    /** Get Project ID */
-                    $stats['Project ID'] = $project->getId();
-
-                    $stats['Organization ID']   = $project->getAttribute('teamId', null);
-
-                    /** Get Total Members */
-                    $teamInternalId = $project->getAttribute('teamInternalId', null);
-                    if ($teamInternalId) {
-                        $stats['Organization Members'] = $dbForConsole->count('memberships', [
-                            Query::equal('teamInternalId', [$teamInternalId])
-                        ]);
-                    } else {
-                        $stats['Organization Members'] = 0;
-                    }
-
-                    /** Get Total internal Teams */
-                    try {
-                        $stats['Teams'] = $dbForProject->count('teams', []);
-                    } catch (\Throwable) {
-                        $stats['Teams'] = 0;
-                    }
-
-                    /** Get Total users */
-                    try {
-                        $stats['Users'] = $dbForProject->count('users', []);
-                    } catch (\Throwable) {
-                        $stats['Users'] = 0;
-                    }
-
-                    /** Get Usage stats */
-                    $range = '30d';
-                    $periods = [
-                        '30d' => [
-                            'period' => '1d',
-                            'limit' => 30,
-                        ]
-                    ];
-
-                    $tmp = [];
-                    $metrics = $this->usageStats;
-                    Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) {
-                        foreach ($metrics as $metric => $name) {
-                            $limit = $periods[$range]['limit'];
-                            $period = $periods[$range]['period'];
-
-                            $requestDocs = $dbForProject->find('stats', [
-                                Query::equal('period', [$period]),
-                                Query::equal('metric', [$metric]),
-                                Query::limit($limit),
-                                Query::orderDesc('time'),
-                            ]);
-
-                            $tmp[$metric] = [];
-                            foreach ($requestDocs as $requestDoc) {
-                                if (empty($requestDoc)) {
-                                    continue;
-                                }
-
-                                $tmp[$metric][] = [
-                                    'value' => $requestDoc->getAttribute('value'),
-                                    'date' => $requestDoc->getAttribute('time'),
-                                ];
-                            }
-
-                            $tmp[$metric] = array_reverse($tmp[$metric]);
-                            $tmp[$metric] = array_sum(array_column($tmp[$metric], 'value'));
-                        }
-                    });
-
-                    foreach ($tmp as $key => $value) {
-                        $stats[$metrics[$key]]  = $value;
-                    }
-
-                    try {
-                        /** Get Domains */
-                        $stats['Domains'] = $dbForConsole->count('domains', [
-                            Query::equal('projectInternalId', [$project->getInternalId()]),
-                        ]);
-                    } catch (\Throwable) {
-                        $stats['Domains'] = 0;
-                    }
-
-                    try {
-                    /** Get Api keys */
-                        $stats['Api keys'] = $dbForConsole->count('keys', [
-                        Query::equal('projectInternalId', [$project->getInternalId()]),
-                        ]);
-                    } catch (\Throwable) {
-                        $stats['Api keys'] = 0;
-                    }
-
-                    try {
-                    /** Get Webhooks */
-                        $stats['Webhooks'] = $dbForConsole->count('webhooks', [
-                        Query::equal('projectInternalId', [$project->getInternalId()]),
-                        ]);
-                    } catch (\Throwable) {
-                        $stats['Webhooks'] = 0;
-                    }
-
-                    try {
-                    /** Get Platforms */
-                        $stats['Platforms'] = $dbForConsole->count('platforms', [
-                        Query::equal('projectInternalId', [$project->getInternalId()]),
-                        ]);
-                    } catch (\Throwable) {
-                        $stats['Platforms'] = 0;
-                    }
-
-                    /** Get Files & Buckets */
-                    $filesCount = 0;
-                    $filesSum = 0;
-                    $maxFileSize = 0;
-                    $counter = 0;
-                    try {
-                        $buckets = $dbForProject->find('buckets', []);
-                        foreach ($buckets as $bucket) {
-                            $file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]);
-                            if (empty($file)) {
-                                continue;
-                            }
-                            $filesSum   += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', []);
-                            $filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []);
-                            if ($file->getAttribute('sizeOriginal') > $maxFileSize) {
-                                $maxFileSize = $file->getAttribute('sizeOriginal');
-                            }
-                            $counter++;
-                        }
-                    } catch (\Throwable) {
-                        ;
-                    }
-                    $stats['Buckets'] = $counter;
-                    $stats['Files'] = $filesCount;
-                    $stats['Storage (bytes)'] = $filesSum;
-                    $stats['Max File Size (bytes)'] = $maxFileSize;
-
-
-                    try {
-                    /** Get Total Functions */
-                        $stats['Databases'] = $dbForProject->count('databases', []);
-                    } catch (\Throwable) {
-                        $stats['Databases'] = 0;
-                    }
-
-                    /** Get Total Functions */
-                    try {
-                        $stats['Functions'] = $dbForProject->count('functions', []);
-                    } catch (\Throwable) {
-                        $stats['Functions'] = 0;
-                    }
-
-                    /** Get Total Deployments */
-                    try {
-                        $stats['Deployments'] = $dbForProject->count('deployments', []);
-                    } catch (\Throwable) {
-                        $stats['Deployments'] = 0;
-                    }
-
-                    /** Get Total Executions */
-                    try {
-                        $stats['Executions'] = $dbForProject->count('executions', []);
-                    } catch (\Throwable) {
-                        $stats['Executions'] = 0;
-                    }
-
-                    $csv->insertOne(array_values($stats));
+                    $dbForProject = call_user_func($getProjectDB, $project);
+                    $data = $this->getData($project, $dbForConsole, $dbForProject);
+                    $csv->insertOne($data);
                 } catch (\Throwable $th) {
-                    Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . '  line no: ' . $th->getline() . ' with message: ' . $th->getMessage());
-                } finally {
-                    $pools
-                        ->get($db)
-                        ->reclaim();
+                    Console::error("Unexpected error occured with Project ID {$projectId}");
+                    Console::error('[Error] Type: ' . get_class($th));
+                    Console::error('[Error] Message: ' . $th->getMessage());
+                    Console::error('[Error] File: ' . $th->getFile());
+                    Console::error('[Error] Line: ' . $th->getLine());
                 }
+            });
+
+        $this->sendMail($register);
+    }
+
+    private function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void
+    {
+        $limit = 1000;
+        $results = [];
+        $sum = $limit;
+        $latestDocument = null;
+
+        while ($sum === $limit) {
+            $newQueries = $queries;
+
+            if ($latestDocument != null) {
+                array_unshift($newQueries, Query::cursorAfter($latestDocument));
+            }
+            $newQueries[] = Query::limit($limit);
+            $results = $database->find('projects', $newQueries);
+
+            if (empty($results)) {
+                return;
             }
 
-            $sum = \count($projects);
+            $sum = count($results);
 
-            $projects = $dbForConsole->find('projects', [
-                Query::limit($limit),
-                Query::offset($offset),
-            ]);
-
-            $offset = $offset + $limit;
-            $count = $count + $sum;
+            foreach ($results as $document) {
+                if (is_callable($callback)) {
+                    $callback($document);
+                }
+            }
+            $latestDocument = $results[array_key_last($results)];
         }
+    }
 
-        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
-
-        $pools
-            ->get('console')
-            ->reclaim();
-
+    private function sendMail(Registry $register): void
+    {
         /** @var PHPMailer $mail */
         $mail = $register->get('smtp');
-
         $mail->clearAddresses();
         $mail->clearAllRecipients();
         $mail->clearReplyTos();
@@ -331,7 +185,6 @@ class CalcTierStats extends Action
             /** Addresses */
             $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
             $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
-
             foreach ($recipients as $recipient) {
                 $mail->addAddress($recipient);
             }
@@ -348,4 +201,208 @@ class CalcTierStats extends Action
             Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
         }
     }
+
+
+    private function getData(Document $project, Database $dbForConsole, Database $dbForProject): array
+    {
+
+            $stats['Project ID'] = $project->getId();
+            $stats['Organization ID']   = $project->getAttribute('teamId', null);
+
+            $teamInternalId = $project->getAttribute('teamInternalId', 0);
+
+        if ($teamInternalId) {
+            $membership = $dbForConsole->findOne('memberships', [
+                Query::equal('teamInternalId', [$teamInternalId]),
+            ]);
+
+            if (!$membership || $membership->isEmpty()) {
+                Console::error('Membership not found. Skipping project : ' . $project->getId());
+            }
+
+            $userId = $membership->getAttribute('userId', null);
+            if ($userId) {
+                $user = $dbForConsole->getDocument('users', $userId);
+                $stats['Organization Email'] = $user->getAttribute('email', null);
+            }
+        } else {
+            Console::error("Email was not found for this Organization ID :{$teamInternalId}");
+        }
+
+            /** Get Total Members */
+            $teamInternalId = $project->getAttribute('teamInternalId', null);
+        if ($teamInternalId) {
+            $stats['Organization Members'] = $dbForConsole->count('memberships', [
+                Query::equal('$internalId', [(string)$teamInternalId])
+            ]);
+        } else {
+            $stats['Organization Members'] = 0;
+        }
+
+            /** Get Total internal Teams */
+        try {
+            $stats['Teams'] = $dbForProject->count('teams', []);
+        } catch (\Throwable) {
+            $stats['Teams'] = 0;
+        }
+
+            /** Get Total users */
+        try {
+            $stats['Users'] = $dbForProject->count('users', []);
+        } catch (\Throwable) {
+            $stats['Users'] = 0;
+        }
+
+            /** Get Usage stats */
+            $range = '30d';
+            $periods = [
+                '30d' => [
+                    'period' => '1d',
+                    'limit' => 30,
+                ]
+            ];
+
+            $tmp = [];
+            $metrics = $this->usageStats;
+            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$tmp) {
+                foreach ($metrics as $metric => $name) {
+                    $limit = $periods[$range]['limit'];
+                    $period = $periods[$range]['period'];
+
+                    $requestDocs = $dbForProject->find('stats_v2', [
+                        Query::equal('metric', [$metric]),
+                        Query::equal('period', [$period]),
+                        Query::limit($limit),
+                        Query::orderDesc('time'),
+                    ]);
+
+                    $tmp[$metric] = [];
+                    foreach ($requestDocs as $requestDoc) {
+                        if (empty($requestDoc)) {
+                            continue;
+                        }
+
+                        $tmp[$metric][] = [
+                            'value' => $requestDoc->getAttribute('value'),
+                            'date' => $requestDoc->getAttribute('time'),
+                        ];
+                    }
+
+                    $tmp[$metric] = array_reverse($tmp[$metric]);
+                    $tmp[$metric] = array_sum(array_column($tmp[$metric], 'value'));
+                }
+            });
+
+        foreach ($tmp as $key => $value) {
+            $stats[$metrics[$key]]  = $value;
+        }
+
+            /**
+             * Workaround to combine network.inbound+network.outbound as network.
+             */
+            $stats['Network'] = ($stats['Inbound'] ?? 0) + ($stats['Outbound'] ?? 0);
+            unset($stats['Inbound']);
+            unset($stats['Outbound']);
+
+        try {
+            /** Get Domains */
+            $stats['Domains'] = $dbForConsole->count('rules', [
+                Query::equal('projectInternalId', [$project->getInternalId()]),
+            ]);
+        } catch (\Throwable) {
+            $stats['Domains'] = 0;
+        }
+
+        try {
+            /** Get Api keys */
+            $stats['Api keys'] = $dbForConsole->count('keys', [
+                Query::equal('projectInternalId', [$project->getInternalId()]),
+            ]);
+        } catch (\Throwable) {
+            $stats['Api keys'] = 0;
+        }
+
+        try {
+            /** Get Webhooks */
+            $stats['Webhooks'] = $dbForConsole->count('webhooks', [
+                Query::equal('projectInternalId', [$project->getInternalId()]),
+            ]);
+        } catch (\Throwable) {
+            $stats['Webhooks'] = 0;
+        }
+
+        try {
+            /** Get Platforms */
+            $stats['Platforms'] = $dbForConsole->count('platforms', [
+                Query::equal('projectInternalId', [$project->getInternalId()]),
+            ]);
+        } catch (\Throwable) {
+            $stats['Platforms'] = 0;
+        }
+
+            /** Get Files & Buckets */
+            $filesCount = 0;
+            $filesSum = 0;
+            $maxFileSize = 0;
+            $counter = 0;
+        try {
+            $buckets = $dbForProject->find('buckets', []);
+            foreach ($buckets as $bucket) {
+                $file = $dbForProject->findOne('bucket_' . $bucket->getInternalId(), [Query::orderDesc('sizeOriginal'),]);
+                if (empty($file)) {
+                    continue;
+                }
+                $filesSum   += $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal', []);
+                $filesCount += $dbForProject->count('bucket_' . $bucket->getInternalId(), []);
+                if ($file->getAttribute('sizeOriginal') > $maxFileSize) {
+                    $maxFileSize = $file->getAttribute('sizeOriginal');
+                }
+                $counter++;
+            }
+        } catch (\Throwable $t) {
+            Console::error("Error while counting buckets: {$project->getId()}");
+        }
+            $stats['Buckets'] = $counter;
+            $stats['Files'] = $filesCount;
+            $stats['Storage (bytes)'] = $filesSum;
+            $stats['Max File Size (bytes)'] = $maxFileSize;
+
+
+        try {
+            /** Get Total Functions */
+            $stats['Databases'] = $dbForProject->count('databases', []);
+        } catch (\Throwable) {
+            $stats['Databases'] = 0;
+        }
+
+            /** Get Total Functions */
+        try {
+            $stats['Functions'] = $dbForProject->count('functions', []);
+        } catch (\Throwable) {
+            $stats['Functions'] = 0;
+        }
+
+            /** Get Total Deployments */
+        try {
+            $stats['Deployments'] = $dbForProject->count('deployments', []);
+        } catch (\Throwable) {
+            $stats['Deployments'] = 0;
+        }
+
+            /** Get Total Executions */
+        try {
+            $stats['Executions'] = $dbForProject->count('executions', []);
+        } catch (\Throwable) {
+            $stats['Executions'] = 0;
+        }
+
+            /** Get Total Migrations */
+        try {
+            $stats['Migrations'] = $dbForProject->count('migrations', []);
+        } catch (\Throwable) {
+            $stats['Migrations'] = 0;
+        }
+
+        return array_values($stats);
+    }
 }

src/Appwrite/Platform/Tasks/CreateInfMetric.php

@@ -0,0 +1,314 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Utopia\App;
+use Utopia\Database\Document;
+use Utopia\Database\Exception;
+use Utopia\Database\Exception\Duplicate;
+use Utopia\Database\Query;
+use Utopia\Platform\Action;
+use Utopia\Cache\Cache;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Pools\Group;
+use Utopia\Registry\Registry;
+
+class CreateInfMetric extends Action
+{
+    public static function getName(): string
+    {
+        return 'create-inf-metric';
+    }
+
+    public function __construct()
+    {
+
+        $this
+            ->desc('Create infinity stats metric')
+            ->inject('pools')
+            ->inject('cache')
+            ->inject('dbForConsole')
+            ->inject('register')
+            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
+                $this->action($pools, $cache, $dbForConsole, $register);
+            });
+    }
+
+
+    public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
+    {
+
+        Console::title('Create infinity metric V1');
+        Console::success(APP_NAME . ' Create infinity metric started');
+
+
+        /* Initialise new Utopia app */
+        $app = new App('UTC');
+        $console = $app->getResource('console');
+        $projects = [$console];
+
+        /** Database connections */
+        $totalProjects = $dbForConsole->count('projects');
+        Console::success("Found a total of: {$totalProjects} projects");
+
+        $count = 0;
+        $limit = 30;
+        $sum = 30;
+        $offset = 0;
+        while (!empty($projects)) {
+            foreach ($projects as $project) {
+
+                /**
+                 * Skip user projects with id 'console'
+                 */
+                if ($project->getId() === 'console') {
+                    continue;
+                }
+
+                try {
+                    $db = $project->getAttribute('database');
+                    $adapter = $pools
+                        ->get($db)
+                        ->pop()
+                        ->getResource();
+
+                    $dbForProject = new Database($adapter, $cache);
+                    $dbForProject->setDefaultDatabase('appwrite');
+                    $dbForProject->setNamespace('_' . $project->getInternalId());
+
+                    $this->network($dbForProject);
+                    $this->sessions($dbForProject);
+                    $this->users($dbForProject);
+                    $this->teams($dbForProject);
+                    $this->databases($dbForProject);
+                    $this->functions($dbForProject);
+                    $this->storage($dbForProject);
+                } catch (\Throwable $th) {
+                    var_dump($th->getMessage());
+                } finally {
+                    $pools
+                        ->get($db)
+                        ->reclaim();
+                }
+
+                Console::log('Finished project ' . $project->getId() . ' ' . $project->getInternalId());
+            }
+
+            $sum = \count($projects);
+
+            $projects = $dbForConsole->find('projects', [
+                Query::limit($limit),
+                Query::offset($offset),
+            ]);
+
+            $offset = $offset + $limit;
+            $count = $count + $sum;
+        }
+
+        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects');
+    }
+
+    /**
+     * @param string $metric
+     * @param int|float $value
+     * @return void
+     */
+    private function createInfMetric(database $dbForProject, string $metric, int|float $value): void
+    {
+
+        try {
+            $id = \md5("_inf_{$metric}");
+
+            $dbForProject->deleteDocument('stats_v2', $id);
+
+            $dbForProject->createDocument('stats_v2', new Document([
+                '$id' => $id,
+                'metric' => $metric,
+                'period' => 'inf',
+                'value'  => (int)$value,
+                'time'   => null,
+                'region' => 'default',
+            ]));
+        } catch (Duplicate $th) {
+            console::log("Error while creating inf metric: duplicate id {$metric}  {$id}");
+        }
+    }
+
+    /**
+     * @param Database $dbForProject
+     * @param string $metric
+     * @return int|float
+     * @throws Exception
+     */
+    protected function getFromMetric(database $dbForProject, string $metric): int|float
+    {
+
+        return  $dbForProject->sum('stats_v2', 'value', [
+            Query::equal('metric', [
+                $metric,
+            ]),
+            Query::equal('period', ['1d']),
+        ]);
+    }
+
+    /**
+     * @throws Exception
+     */
+    private function network(database $dbForProject)
+    {
+        $this->createInfMetric($dbForProject, 'network.inbound', $this->getFromMetric($dbForProject, 'network.inbound'));
+        $this->createInfMetric($dbForProject, 'network.outbound', $this->getFromMetric($dbForProject, 'network.outbound'));
+        $this->createInfMetric($dbForProject, 'network.requests', $this->getFromMetric($dbForProject, 'network.requests'));
+    }
+
+
+    private function storage(database $dbForProject)
+    {
+        $bucketsCount = 0;
+        $filesCount = 0;
+        $filesStorageSum = 0;
+
+        $buckets = $dbForProject->find('buckets');
+        foreach ($buckets as $bucket) {
+            $files = $dbForProject->count('bucket_' . $bucket->getInternalId());
+            $this->createInfMetric($dbForProject, $bucket->getInternalId() . '.files', $files);
+
+            $filesStorage = $dbForProject->sum('bucket_' . $bucket->getInternalId(), 'sizeOriginal');
+            $this->createInfMetric($dbForProject, $bucket->getInternalId() . '.files.storage', $filesStorage);
+
+            $bucketsCount++;
+            $filesCount += $files;
+            $filesStorageSum += $filesStorage;
+        }
+
+        $this->createInfMetric($dbForProject, 'buckets', $bucketsCount);
+        $this->createInfMetric($dbForProject, 'files', $filesCount);
+        $this->createInfMetric($dbForProject, 'files.storage', $filesStorageSum);
+    }
+
+
+    private function functions(Database $dbForProject)
+    {
+        $functionsCount = 0;
+        $deploymentsCount = 0;
+        $buildsCount = 0;
+        $buildsStorageSum = 0;
+        $buildsComputeSum = 0;
+        $executionsCount = 0;
+        $executionsComputeSum = 0;
+        $deploymentsStorageSum = 0;
+
+        //functions
+        $functions = $dbForProject->find('functions');
+        foreach ($functions as $function) {
+            //deployments
+            $deployments = $dbForProject->find('deployments', [
+                Query::equal('resourceType', ['functions']),
+                Query::equal('resourceInternalId', [$function->getInternalId()]),
+            ]);
+
+            $deploymentCount = 0;
+            $deploymentStorageSum = 0;
+            foreach ($deployments as $deployment) {
+                //builds
+                $builds = $dbForProject->count('builds', [
+                    Query::equal('deploymentInternalId', [$deployment->getInternalId()]),
+                ]);
+
+                $buildsCompute = $dbForProject->sum('builds', 'duration', [
+                    Query::equal('deploymentInternalId', [$deployment->getInternalId()]),
+                ]);
+
+                $buildsStorage = $dbForProject->sum('builds', 'size', [
+                    Query::equal('deploymentInternalId', [$deployment->getInternalId()]),
+                ]);
+
+                $this->createInfMetric($dbForProject, $function->getInternalId() . '.builds', $builds);
+                $this->createInfMetric($dbForProject, $function->getInternalId() . '.builds.storage', $buildsCompute * 1000);
+                $this->createInfMetric($dbForProject, $function->getInternalId() . '.builds.compute', $buildsStorage);
+
+                $buildsCount += $builds;
+                $buildsComputeSum += $buildsCompute;
+                $buildsStorageSum += $buildsStorage;
+
+
+                $deploymentCount++;
+                $deploymentsCount++;
+                $deploymentsStorageSum += $deployment['size'];
+                $deploymentStorageSum += $deployment['size'];
+            }
+            $this->createInfMetric($dbForProject, 'functions.' . $function->getInternalId() . '.deployments', $deploymentCount);
+            $this->createInfMetric($dbForProject, 'functions.' . $function->getInternalId() . '.deployments.storage', $deploymentStorageSum);
+
+            //executions
+            $executions = $dbForProject->count('executions', [
+                Query::equal('functionInternalId', [$function->getInternalId()]),
+            ]);
+
+            $executionsCompute = $dbForProject->sum('executions', 'duration', [
+                Query::equal('functionInternalId', [$function->getInternalId()]),
+            ]);
+
+            $this->createInfMetric($dbForProject, $function->getInternalId() . '.executions', $executions);
+            $this->createInfMetric($dbForProject, $function->getInternalId() . '.executions.compute', $executionsCompute * 1000);
+            $executionsCount += $executions;
+            $executionsComputeSum += $executionsCompute;
+
+            $functionsCount++;
+        }
+
+        $this->createInfMetric($dbForProject, 'functions', $functionsCount);
+        $this->createInfMetric($dbForProject, 'deployments', $deploymentsCount);
+        $this->createInfMetric($dbForProject, 'deployments.storage', $deploymentsStorageSum);
+        $this->createInfMetric($dbForProject, 'builds', $buildsCount);
+        $this->createInfMetric($dbForProject, 'builds.compute', $buildsComputeSum * 1000);
+        $this->createInfMetric($dbForProject, 'builds.storage', $buildsStorageSum);
+        $this->createInfMetric($dbForProject, 'executions', $executionsCount);
+        $this->createInfMetric($dbForProject, 'executions.compute', $executionsComputeSum * 1000);
+    }
+
+    private function databases(Database $dbForProject)
+    {
+        $databasesCount = 0;
+        $collectionsCount = 0;
+        $documentsCount = 0;
+        $databases = $dbForProject->find('databases');
+        foreach ($databases as $database) {
+            $collectionCount = 0;
+            $collections = $dbForProject->find('database_' . $database->getInternalId());
+            foreach ($collections as $collection) {
+                $documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
+                $this->createInfMetric($dbForProject, $database->getInternalId() . '.' . $collection->getInternalId() . '.documents', $documents);
+                $documentsCount += $documents;
+                $collectionCount++;
+                $collectionsCount++;
+            }
+            $this->createInfMetric($dbForProject, $database->getInternalId() . '.collections', $collectionCount);
+            $this->createInfMetric($dbForProject, $database->getInternalId() . '.documents', $documentsCount);
+            $databasesCount++;
+        }
+        $this->createInfMetric($dbForProject, 'collections', $collectionsCount);
+        $this->createInfMetric($dbForProject, 'databases', $databasesCount);
+        $this->createInfMetric($dbForProject, 'documents', $documentsCount);
+    }
+
+
+    private function users(Database $dbForProject)
+    {
+        $users = $dbForProject->count('users');
+        $this->createInfMetric($dbForProject, 'users', $users);
+    }
+
+    private function sessions(Database $dbForProject)
+    {
+        $users = $dbForProject->count('sessions');
+        $this->createInfMetric($dbForProject, 'sessions', $users);
+    }
+
+    private function teams(Database $dbForProject)
+    {
+        $teams = $dbForProject->count('teams');
+        $this->createInfMetric($dbForProject, 'teams', $teams);
+    }
+}

src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php

@@ -0,0 +1,161 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Utopia\App;
+use Utopia\Config\Config;
+use Utopia\Database\Helpers\ID;
+use Utopia\Database\Query;
+use Utopia\Platform\Action;
+use Utopia\Cache\Cache;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Pools\Group;
+use Utopia\Registry\Registry;
+use Utopia\Validator\Boolean;
+
+class DeleteOrphanedProjects extends Action
+{
+    public static function getName(): string
+    {
+        return 'delete-orphaned-projects';
+    }
+
+    public function __construct()
+    {
+
+        $this
+            ->desc('Delete orphaned projects')
+            ->param('commit', false, new Boolean(true), 'Commit  project deletion', true)
+            ->inject('pools')
+            ->inject('cache')
+            ->inject('dbForConsole')
+            ->inject('register')
+            ->callback(function (bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
+                $this->action($commit, $pools, $cache, $dbForConsole, $register);
+            });
+    }
+
+
+    public function action(bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
+    {
+
+        Console::title('Delete orphaned projects V1');
+        Console::success(APP_NAME . ' Delete orphaned projects started');
+
+        /** @var array $collections */
+        $collectionsConfig = Config::getParam('collections', [])['projects'] ?? [];
+
+        $collectionsConfig = array_merge([
+            'audit' => [
+                '$id' => ID::custom('audit'),
+                '$collection' => Database::METADATA
+            ],
+            'abuse' => [
+                '$id' => ID::custom('abuse'),
+                '$collection' => Database::METADATA
+            ]
+        ], $collectionsConfig);
+
+        /* Initialise new Utopia app */
+        $app = new App('UTC');
+        $console = $app->getResource('console');
+        $projects = [$console];
+
+        /** Database connections */
+        $totalProjects = $dbForConsole->count('projects');
+        Console::success("Found a total of: {$totalProjects} projects");
+
+        $orphans = 1;
+        $cnt = 0;
+        $count = 0;
+        $limit = 30;
+        $sum = 30;
+        $offset = 0;
+        while (!empty($projects)) {
+            foreach ($projects as $project) {
+
+                /**
+                 * Skip user projects with id 'console'
+                 */
+                if ($project->getId() === 'console') {
+                    continue;
+                }
+
+                try {
+                    $db = $project->getAttribute('database');
+                    $adapter = $pools
+                        ->get($db)
+                        ->pop()
+                        ->getResource();
+
+                    $dbForProject = new Database($adapter, $cache);
+                    $dbForProject->setDefaultDatabase('appwrite');
+                    $dbForProject->setNamespace('_' . $project->getInternalId());
+
+                    $collectionsCreated = 0;
+                    $cnt++;
+                    if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) {
+                        $collectionsCreated = $dbForProject->count(Database::METADATA);
+                    }
+
+                    $msg = '(' . $cnt . ') found (' . $collectionsCreated . ') collections on project  (' . $project->getInternalId() . ') , database (' . $project['database'] . ')';
+
+                    if ($collectionsCreated >= count($collectionsConfig)) {
+                        Console::log($msg . ' ignoring....');
+                        continue;
+                    }
+
+                    Console::log($msg);
+
+                    if ($collectionsCreated > 0) {
+                        $collections = $dbForProject->find(Database::METADATA, []);
+                        foreach ($collections as $collection) {
+                            if ($commit) {
+                                $dbForProject->deleteCollection($collection->getId());
+                                $dbForConsole->deleteCachedCollection($collection->getId());
+                            }
+                            Console::info('--Deleting collection  (' . $collection->getId() . ') project no (' . $project->getInternalId() . ')');
+                        }
+                    }
+
+                    if ($commit) {
+                        $dbForConsole->deleteDocument('projects', $project->getId());
+                        $dbForConsole->deleteCachedDocument('projects', $project->getId());
+
+                        if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) {
+                            try {
+                                $dbForProject->deleteCollection(Database::METADATA);
+                                $dbForProject->deleteCachedCollection(Database::METADATA);
+                            } catch (\Throwable $th) {
+                                Console::warning('Metadata collection does not exist');
+                            }
+                        }
+                    }
+
+                    Console::info('--Deleting project no (' . $project->getInternalId() . ')');
+
+                    $orphans++;
+                } catch (\Throwable $th) {
+                    Console::error('Error: ' . $th->getMessage() . ' ' . $th->getTraceAsString());
+                } finally {
+                    $pools
+                        ->get($db)
+                        ->reclaim();
+                }
+            }
+
+            $sum = \count($projects);
+
+            $projects = $dbForConsole->find('projects', [
+                Query::limit($limit),
+                Query::offset($offset),
+            ]);
+
+            $offset = $offset + $limit;
+            $count = $count + $sum;
+        }
+
+        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans');
+    }
+}

src/Appwrite/Platform/Tasks/GetMigrationStats.php

@@ -0,0 +1,187 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Exception;
+use League\Csv\CannotInsertRecord;
+use Utopia\App;
+use Utopia\Platform\Action;
+use Utopia\Cache\Cache;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Database\Query;
+use League\Csv\Writer;
+use PHPMailer\PHPMailer\PHPMailer;
+use Utopia\Pools\Group;
+use Utopia\Registry\Registry;
+
+class GetMigrationStats extends Action
+{
+    /*
+     * Csv cols headers
+     */
+    private array $columns = [
+        'Project ID',
+        '$id',
+        '$createdAt',
+        'status',
+        'stage',
+        'source'
+    ];
+
+    protected string $directory = '/usr/local';
+    protected string $path;
+    protected string $date;
+
+    public static function getName(): string
+    {
+        return 'get-migration-stats';
+    }
+
+    public function __construct()
+    {
+
+        $this
+            ->desc('Get stats for projects')
+            ->inject('pools')
+            ->inject('cache')
+            ->inject('dbForConsole')
+            ->inject('register')
+            ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
+                $this->action($pools, $cache, $dbForConsole, $register);
+            });
+    }
+
+    /**
+     * @throws \Utopia\Exception
+     * @throws CannotInsertRecord
+     */
+    public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
+    {
+        //docker compose exec -t appwrite get-migration-stats
+
+        Console::title('Migration stats calculation V1');
+        Console::success(APP_NAME . ' Migration stats calculation has started');
+
+        /* Initialise new Utopia app */
+        $app = new App('UTC');
+        $console = $app->getResource('console');
+
+        /** CSV stuff */
+        $this->date = date('Y-m-d');
+        $this->path = "{$this->directory}/migration_stats_{$this->date}.csv";
+        $csv = Writer::createFromPath($this->path, 'w');
+        $csv->insertOne($this->columns);
+
+        /** Database connections */
+        $totalProjects = $dbForConsole->count('projects');
+        Console::success("Found a total of: {$totalProjects} projects");
+
+        $projects = [$console];
+        $count = 0;
+        $limit = 100;
+        $sum = 100;
+        $offset = 0;
+        while (!empty($projects)) {
+            foreach ($projects as $project) {
+
+                /**
+                 * Skip user projects with id 'console'
+                 */
+                if ($project->getId() === 'console') {
+                    continue;
+                }
+
+                Console::info("Getting stats for {$project->getId()}");
+
+                try {
+                    $db = $project->getAttribute('database');
+                    $adapter = $pools
+                        ->get($db)
+                        ->pop()
+                        ->getResource();
+
+                    $dbForProject = new Database($adapter, $cache);
+                    $dbForProject->setDefaultDatabase('appwrite');
+                    $dbForProject->setNamespace('_' . $project->getInternalId());
+
+                    /** Get Project ID */
+                    $stats['Project ID'] = $project->getId();
+
+                    /** Get Migration details */
+                    $migrations = $dbForProject->find('migrations', [
+                        Query::limit(500)
+                    ]);
+
+                    $migrations = array_map(function ($migration) use ($project) {
+                        return [
+                            $project->getId(),
+                            $migration->getAttribute('$id'),
+                            $migration->getAttribute('$createdAt'),
+                            $migration->getAttribute('status'),
+                            $migration->getAttribute('stage'),
+                            $migration->getAttribute('source'),
+                        ];
+                    }, $migrations);
+
+                    if (!empty($migrations)) {
+                        $csv->insertAll($migrations);
+                    }
+                } catch (\Throwable $th) {
+                    Console::error('Failed on project ("' . $project->getId() . '") with error on File: ' . $th->getFile() . '  line no: ' . $th->getline() . ' with message: ' . $th->getMessage());
+                } finally {
+                    $pools
+                        ->get($db)
+                        ->reclaim();
+                }
+            }
+
+            $sum = \count($projects);
+
+            $projects = $dbForConsole->find('projects', [
+                Query::limit($limit),
+                Query::offset($offset),
+            ]);
+
+            $offset = $offset + $limit;
+            $count = $count + $sum;
+        }
+
+        Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
+
+        $pools
+            ->get('console')
+            ->reclaim();
+
+        /** @var PHPMailer $mail */
+        $mail = $register->get('smtp');
+
+        $mail->clearAddresses();
+        $mail->clearAllRecipients();
+        $mail->clearReplyTos();
+        $mail->clearAttachments();
+        $mail->clearBCCs();
+        $mail->clearCCs();
+
+        try {
+            /** Addresses */
+            $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
+            $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
+
+            foreach ($recipients as $recipient) {
+                $mail->addAddress($recipient);
+            }
+
+            /** Attachments */
+            $mail->addAttachment($this->path);
+
+            /** Content */
+            $mail->Subject = "Migration Report for {$this->date}";
+            $mail->Body = "Please find the migration report atttached";
+            $mail->send();
+            Console::success('Email has been sent!');
+        } catch (Exception $e) {
+            Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
+        }
+    }
+}

src/Appwrite/Platform/Tasks/Hamster.php

@@ -20,17 +20,18 @@ use Utopia\Pools\Group;
 class Hamster extends Action
 {
     private array $metrics = [
-        'usage_files' => 'files.$all.count.total',
-        'usage_buckets' => 'buckets.$all.count.total',
-        'usage_databases' => 'databases.$all.count.total',
-        'usage_documents' => 'documents.$all.count.total',
-        'usage_collections' => 'collections.$all.count.total',
-        'usage_storage' => 'project.$all.storage.size',
-        'usage_requests' => 'project.$all.network.requests',
-        'usage_bandwidth' => 'project.$all.network.bandwidth',
-        'usage_users' => 'users.$all.count.total',
-        'usage_sessions' => 'sessions.email.requests.create',
-        'usage_executions' => 'executions.$all.compute.total',
+        'usage_files' => 'files',
+        'usage_buckets' => 'buckets',
+        'usage_databases' => 'databases',
+        'usage_documents' => 'documents',
+        'usage_collections' => 'collections',
+        'usage_storage' => 'files.storage',
+        'usage_requests' => 'network.requests',
+        'usage_inbound' => 'network.inbound',
+        'usage_outbound' => 'network.outbound',
+        'usage_users' => 'users',
+        'usage_sessions' => 'sessions',
+        'usage_executions' => 'executions',
     ];
 
     protected string $directory = '/usr/local';
@@ -149,6 +150,9 @@ class Hamster extends Action
                 /** Get Total Teams */
                 $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT);
 
+                /** Get Total Migrations */
+                $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT);
+
                 /** Get Total Members */
                 $teamInternalId = $project->getAttribute('teamInternalId', null);
                 if ($teamInternalId) {
@@ -236,9 +240,9 @@ class Hamster extends Action
                             $limit = $periodValue['limit'];
                             $period = $periodValue['period'];
 
-                            $requestDocs = $dbForProject->find('stats', [
-                                Query::equal('period', [$period]),
+                            $requestDocs = $dbForProject->find('stats_v2', [
                                 Query::equal('metric', [$metric]),
+                                Query::equal('period', [$period]),
                                 Query::limit($limit),
                                 Query::orderDesc('time'),
                             ]);
@@ -258,6 +262,16 @@ class Hamster extends Action
                     }
                 });
 
+                /**
+                 * Workaround to combine network.inbound+network.outbound as network.
+                 */
+                $statsPerProject["usage_network_infinity"]  = $statsPerProject["usage_inbound_infinity"] + $statsPerProject["usage_outbound_infinity"];
+                $statsPerProject["usage_network_24h"]  = $statsPerProject["usage_inbound_24h"] + $statsPerProject["usage_outbound_24h"];
+                unset($statsPerProject["usage_outbound_24h"]);
+                unset($statsPerProject["usage_inbound_24h"]);
+                unset($statsPerProject["usage_outbound_infinity"]);
+                unset($statsPerProject["usage_inbound_infinity"]);
+
                 if (isset($statsPerProject['email'])) {
                     /** Send data to mixpanel */
                     $res = $this->mixpanel->createProfile($statsPerProject['email'], '', [
@@ -297,7 +311,6 @@ class Hamster extends Action
         Console::success(APP_NAME . ' cloud hamster process has started');
 
         $sleep = (int) App::getEnv('_APP_HAMSTER_INTERVAL', '30'); // 30 seconds (by default)
-
         $jobInitTime = App::getEnv('_APP_HAMSTER_TIME', '22:00'); // (hour:minutes)
         $now = new \DateTime();
         $now->setTimezone(new \DateTimeZone(date_default_timezone_get()));
@@ -312,7 +325,6 @@ class Hamster extends Action
             $next->add(\DateInterval::createFromDateString('1 days'));
             $delay = $next->getTimestamp() - $now->getTimestamp();
         }
-
         Console::log('[' . $now->format("Y-m-d H:i:s.v") . '] Delaying for ' . $delay . ' setting loop to [' . $next->format("Y-m-d H:i:s.v") . ']');
 
         Console::loop(function () use ($pools, $cache, $dbForConsole, $sleep) {

src/Appwrite/Platform/Tasks/Install.php

@@ -6,9 +6,6 @@ use Appwrite\Auth\Auth;
 use Appwrite\Docker\Compose;
 use Appwrite\Docker\Env;
 use Appwrite\Utopia\View;
-use Utopia\Analytics\Adapter;
-use Utopia\Analytics\Adapter\GoogleAnalytics;
-use Utopia\Analytics\Event;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
 use Utopia\Validator\Text;
@@ -43,12 +40,6 @@ class Install extends Action
         /** @var array<string, array<string, string>> $vars array whre key is variable name and value is variable */
         $vars = [];
 
-        /**
-         * We are using a random value every execution for identification.
-         * This allows us to collect information without invading the privacy of our users.
-         */
-        $analytics = new GoogleAnalytics('UA-26264668-9', uniqid('server.', true));
-
         foreach ($config as $category) {
             foreach ($category['variables'] ?? [] as $var) {
                 $vars[$var['name']] = $var;
@@ -82,7 +73,7 @@ class Install extends Action
             file_put_contents($this->path . '/docker-compose.yml.' . $time . '.backup', $data);
             $compose = new Compose($data);
             $appwrite = $compose->getService('appwrite');
-            $oldVersion = ($appwrite) ? $appwrite->getImageVersion() : null;
+            $oldVersion = $appwrite?->getImageVersion();
             try {
                 $ports = $compose->getService('traefik')->getPorts();
             } catch (\Throwable $th) {
@@ -209,14 +200,12 @@ class Install extends Action
 
         if (!file_put_contents($this->path . '/docker-compose.yml', $templateForCompose->render(false))) {
             $message = 'Failed to save Docker Compose file';
-            $this->sendEvent($analytics, $message);
             Console::error($message);
             Console::exit(1);
         }
 
         if (!file_put_contents($this->path . '/.env', $templateForEnv->render(false))) {
             $message = 'Failed to save environment variables file';
-            $this->sendEvent($analytics, $message);
             Console::error($message);
             Console::exit(1);
         }
@@ -237,29 +226,12 @@ class Install extends Action
 
         if ($exit !== 0) {
             $message = 'Failed to install Appwrite dockers';
-            $this->sendEvent($analytics, $message);
             Console::error($message);
             Console::error($stderr);
             Console::exit($exit);
         } else {
             $message = 'Appwrite installed successfully';
-            $this->sendEvent($analytics, $message);
             Console::success($message);
         }
     }
-
-    private function sendEvent(Adapter $analytics, string $message): void
-    {
-        $event = new Event();
-        $event->setName(APP_VERSION_STABLE);
-        $event->setValue($message);
-        $event->setUrl('http://localhost/');
-        $event->setProps([
-            'category' => 'install/server',
-            'action' => 'install',
-        ]);
-        $event->setType('install/server');
-
-        $analytics->createEvent($event);
-    }
 }

src/Appwrite/Platform/Tasks/Maintenance.php

@@ -36,50 +36,77 @@ class Maintenance extends Action
 
         // # of days in seconds (1 day = 86400s)
         $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400');
-        $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600');
-        $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600');
-        $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400');
         $usageStatsRetentionHourly = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_HOURLY', '8640000'); //100 days
         $cacheRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_CACHE', '2592000'); // 30 days
         $schedulesDeletionRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_SCHEDULES', '86400'); // 1 Day
 
-        Console::loop(function () use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) {
+        Console::loop(function () use ($interval, $cacheRetention, $schedulesDeletionRetention, $usageStatsRetentionHourly, $dbForConsole, $queueForDeletes, $queueForCertificates) {
             $time = DateTime::now();
 
             Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
-            $this->notifyDeleteExecutionLogs($executionLogsRetention, $queueForDeletes);
-            $this->notifyDeleteAbuseLogs($abuseLogsRetention, $queueForDeletes);
-            $this->notifyDeleteAuditLogs($auditLogRetention, $queueForDeletes);
-            $this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
+
+            $this->foreachProject($dbForConsole, function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) {
+                $queueForDeletes->setProject($project);
+
+                $this->notifyDeleteExecutionLogs($queueForDeletes);
+                $this->notifyDeleteAbuseLogs($queueForDeletes);
+                $this->notifyDeleteAuditLogs($queueForDeletes);
+                $this->notifyDeleteUsageStats($usageStatsRetentionHourly, $queueForDeletes);
+                $this->notifyDeleteExpiredSessions($queueForDeletes);
+            });
+
             $this->notifyDeleteConnections($queueForDeletes);
-            $this->notifyDeleteExpiredSessions($queueForDeletes);
             $this->renewCertificates($dbForConsole, $queueForCertificates);
             $this->notifyDeleteCache($cacheRetention, $queueForDeletes);
             $this->notifyDeleteSchedules($schedulesDeletionRetention, $queueForDeletes);
         }, $interval);
     }
 
-    private function notifyDeleteExecutionLogs(int $interval, Delete $queueForDeletes): void
+    protected function foreachProject(Database $dbForConsole, callable $callback): void
+    {
+        // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
+        $count = 0;
+        $chunk = 0;
+        $limit = 50;
+        $sum = $limit;
+        $executionStart = \microtime(true);
+
+        while ($sum === $limit) {
+            $projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
+
+            $chunk++;
+
+            /** @var string[] $projectIds */
+            $sum = count($projects);
+
+            foreach ($projects as $project) {
+                $callback($project);
+                $count++;
+            }
+        }
+
+        $executionEnd = \microtime(true);
+        Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
+    }
+
+    private function notifyDeleteExecutionLogs(Delete $queueForDeletes): void
     {
         ($queueForDeletes)
             ->setType(DELETE_TYPE_EXECUTIONS)
-            ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
             ->trigger();
     }
 
-    private function notifyDeleteAbuseLogs(int $interval, Delete $queueForDeletes): void
+    private function notifyDeleteAbuseLogs(Delete $queueForDeletes): void
     {
         ($queueForDeletes)
             ->setType(DELETE_TYPE_ABUSE)
-            ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
             ->trigger();
     }
 
-    private function notifyDeleteAuditLogs(int $interval, Delete $queueForDeletes): void
+    private function notifyDeleteAuditLogs(Delete $queueForDeletes): void
     {
         ($queueForDeletes)
             ->setType(DELETE_TYPE_AUDIT)
-            ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
             ->trigger();
     }
 

src/Appwrite/Platform/Tasks/Migrate.php

@@ -11,6 +11,7 @@ use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
+use Utopia\Registry\Registry;
 use Utopia\Validator\Text;
 
 class Migrate extends Action
@@ -29,7 +30,8 @@ class Migrate extends Action
             ->inject('cache')
             ->inject('dbForConsole')
             ->inject('getProjectDB')
-            ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB) => $this->action($version, $cache, $dbForConsole, $getProjectDB));
+            ->inject('register')
+            ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB, Registry $register) => $this->action($version, $cache, $dbForConsole, $getProjectDB, $register));
     }
 
     private function clearProjectsCache(Cache $cache, Document $project)
@@ -41,7 +43,7 @@ class Migrate extends Action
         }
     }
 
-    public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB)
+    public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register)
     {
         Authorization::disable();
         if (!array_key_exists($version, Migration::$versions)) {
@@ -89,9 +91,11 @@ class Migrate extends Action
 
                 try {
                     // TODO: Iterate through all project DBs
+                    /** @var Database $projectDB */
                     $projectDB = $getProjectDB($project);
                     $migration
                         ->setProject($project, $projectDB, $dbForConsole)
+                        ->setPDO($register->get('db', true))
                         ->execute();
                 } catch (\Throwable $th) {
                     Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());

src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php

@@ -0,0 +1,169 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Utopia\Platform\Action;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Database\Document;
+use Utopia\Database\Helpers\ID;
+use Utopia\Database\Helpers\Permission;
+use Utopia\Database\Helpers\Role;
+use Utopia\Database\Query;
+use Utopia\Validator\Text;
+
+class PatchRecreateRepositoriesDocuments extends Action
+{
+    public static function getName(): string
+    {
+        return 'patch-recreate-repositories-documents';
+    }
+
+    public function __construct()
+    {
+        $this
+            ->desc('Recreate missing repositories in consoleDB from projectDBs. They can be missing if you used Appwrite 1.4.10 or 1.4.11, and deleted a function.')
+            ->param('after', '', new Text(36), 'After cursor', true)
+            ->param('projectId', '', new Text(36), 'Select project to validate', true)
+            ->inject('dbForConsole')
+            ->inject('getProjectDB')
+            ->callback(fn ($after, $projectId, $dbForConsole, $getProjectDB) => $this->action($after, $projectId, $dbForConsole, $getProjectDB));
+    }
+
+    public function action($after, $projectId, Database $dbForConsole, callable $getProjectDB): void
+    {
+        Console::info("Starting the patch");
+
+        $startTime = microtime(true);
+
+        if (!empty($projectId)) {
+            try {
+                $project = $dbForConsole->getDocument('projects', $projectId);
+                $dbForProject = call_user_func($getProjectDB, $project);
+                $this->recreateRepositories($dbForConsole, $dbForProject, $project);
+            } catch (\Throwable $th) {
+                Console::error("Unexpected error occured with Project ID {$projectId}");
+                Console::error('[Error] Type: ' . get_class($th));
+                Console::error('[Error] Message: ' . $th->getMessage());
+                Console::error('[Error] File: ' . $th->getFile());
+                Console::error('[Error] Line: ' . $th->getLine());
+            }
+        } else {
+            $queries = [];
+            if (!empty($after)) {
+                Console::info("Iterating remaining projects after project with ID {$after}");
+                $project = $dbForConsole->getDocument('projects', $after);
+                $queries = [Query::cursorAfter($project)];
+            } else {
+                Console::info("Iterating all projects");
+            }
+            $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole) {
+                $projectId = $project->getId();
+
+                try {
+                    $dbForProject = call_user_func($getProjectDB, $project);
+                    $this->recreateRepositories($dbForConsole, $dbForProject, $project);
+                } catch (\Throwable $th) {
+                    Console::error("Unexpected error occured with Project ID {$projectId}");
+                    Console::error('[Error] Type: ' . get_class($th));
+                    Console::error('[Error] Message: ' . $th->getMessage());
+                    Console::error('[Error] File: ' . $th->getFile());
+                    Console::error('[Error] Line: ' . $th->getLine());
+                }
+            });
+        }
+
+        $endTime = microtime(true);
+        $timeTaken = $endTime - $startTime;
+
+        $hours = (int)($timeTaken / 3600);
+        $timeTaken -= $hours * 3600;
+        $minutes = (int)($timeTaken / 60);
+        $timeTaken -= $minutes * 60;
+        $seconds = (int)$timeTaken;
+        $milliseconds = ($timeTaken - $seconds) * 1000;
+        Console::info("Recreate patch completed in $hours h, $minutes m, $seconds s, $milliseconds mis ( total $timeTaken milliseconds)");
+    }
+
+    protected function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void
+    {
+        $limit = 1000;
+        $results = [];
+        $sum = $limit;
+        $latestDocument = null;
+
+        while ($sum === $limit) {
+            $newQueries = $queries;
+
+            if ($latestDocument != null) {
+                array_unshift($newQueries, Query::cursorAfter($latestDocument));
+            }
+            $newQueries[] = Query::limit($limit);
+            $results = $database->find($collection, $newQueries);
+
+            if (empty($results)) {
+                return;
+            }
+
+            $sum = count($results);
+
+            foreach ($results as $document) {
+                if (is_callable($callback)) {
+                    $callback($document);
+                }
+            }
+            $latestDocument = $results[array_key_last($results)];
+        }
+    }
+
+    public function recreateRepositories(Database $dbForConsole, Database $dbForProject, Document $project): void
+    {
+        $projectId = $project->getId();
+        Console::log("Running patch for project {$projectId}");
+
+        $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForConsole, $project) {
+            $isConnected = !empty($function->getAttribute('providerRepositoryId', ''));
+
+            if ($isConnected) {
+                $repository = $dbForConsole->getDocument('repositories', $function->getAttribute('repositoryId', ''));
+
+                if ($repository->isEmpty()) {
+                    $projectId = $project->getId();
+                    $functionId = $function->getId();
+                    Console::success("Recreating repositories document for project ID {$projectId}, function ID {$functionId}");
+
+                    $repository = $dbForConsole->createDocument('repositories', new Document([
+                        '$id' => ID::unique(),
+                        '$permissions' => [
+                            Permission::read(Role::any()),
+                            Permission::update(Role::any()),
+                            Permission::delete(Role::any()),
+                        ],
+                        'installationId' => $function->getAttribute('installationId', ''),
+                        'installationInternalId' => $function->getAttribute('installationInternalId', ''),
+                        'projectId' => $project->getId(),
+                        'projectInternalId' => $project->getInternalId(),
+                        'providerRepositoryId' => $function->getAttribute('providerRepositoryId', ''),
+                        'resourceId' => $function->getId(),
+                        'resourceInternalId' => $function->getInternalId(),
+                        'resourceType' => 'function',
+                        'providerPullRequestIds' => []
+                    ]));
+
+                    $function = $dbForProject->updateDocument('functions', $function->getId(), $function
+                        ->setAttribute('repositoryId', $repository->getId())
+                        ->setAttribute('repositoryInternalId', $repository->getInternalId()));
+
+                    $this->foreachDocument($dbForProject, 'deployments', [
+                        Query::equal('resourceInternalId', [$function->getInternalId()]),
+                        Query::equal('resourceType', ['functions'])
+                    ], function (Document $deployment) use ($dbForProject, $repository) {
+                        $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment
+                            ->setAttribute('repositoryId', $repository->getId())
+                            ->setAttribute('repositoryInternalId', $repository->getInternalId()));
+                    });
+                }
+            }
+        });
+    }
+}

src/Appwrite/Platform/Tasks/SDKs.php

@@ -235,6 +235,11 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
                         'X-Appwrite-Response-Format' => '1.4.0',
                     ]);
 
+                // Make sure we have a clean slate.
+                // Otherwise, all files in this dir will be pushed,
+                // regardless of whether they were just generated or not.
+                \exec('rm -rf ' . $result);
+
                 try {
                     $sdk->generate($result);
                 } catch (Exception $exception) {

src/Appwrite/Platform/Tasks/Usage.php

@@ -1,60 +0,0 @@
-<?php
-
-namespace Appwrite\Platform\Tasks;
-
-use Appwrite\Usage\Calculators\TimeSeries;
-use InfluxDB\Database as InfluxDatabase;
-use Utopia\App;
-use Utopia\CLI\Console;
-use Utopia\Database\Database as UtopiaDatabase;
-use Throwable;
-use Utopia\Platform\Action;
-use Utopia\Registry\Registry;
-
-class Usage extends Action
-{
-    public static function getName(): string
-    {
-        return 'usage';
-    }
-
-    public function __construct()
-    {
-        $this
-            ->desc('Schedules syncing data from influxdb to Appwrite console db')
-            ->inject('dbForConsole')
-            ->inject('influxdb')
-            ->inject('register')
-            ->inject('getProjectDB')
-            ->inject('logError')
-            ->callback(fn ($dbForConsole, $influxDB, $register, $getProjectDB, $logError) => $this->action($dbForConsole, $influxDB, $register, $getProjectDB, $logError));
-    }
-
-    protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void
-    {
-    }
-
-    public function action(UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register, callable $getProjectDB, callable $logError)
-    {
-        Console::title('Usage Aggregation V1');
-        Console::success(APP_NAME . ' usage aggregation process v1 has started');
-
-        $errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action);
-
-        $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
-        $region = App::getEnv('region', 'default');
-        $usage = new TimeSeries($region, $dbForConsole, $influxDB, $getProjectDB, $register, $errorLogger);
-
-        Console::loop(function () use ($interval, $usage) {
-            $now = date('d-m-Y H:i:s', time());
-            Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds");
-            $loopStart = microtime(true);
-
-            $usage->collect();
-
-            $loopTook = microtime(true) - $loopStart;
-            $now = date('d-m-Y H:i:s', time());
-            Console::info("[{$now}] Aggregation took {$loopTook} seconds");
-        }, $interval);
-    }
-}

src/Appwrite/Platform/Workers/Builds.php

@@ -6,7 +6,6 @@ use Appwrite\Event\Event;
 use Appwrite\Event\Func;
 use Appwrite\Event\Usage;
 use Appwrite\Messaging\Adapter\Realtime;
-use Appwrite\Usage\Stats;
 use Appwrite\Utopia\Response\Model\Deployment;
 use Appwrite\Vcs\Comment;
 use Exception;
@@ -48,11 +47,11 @@ class Builds extends Action
             ->inject('dbForConsole')
             ->inject('queueForEvents')
             ->inject('queueForFunctions')
-            ->inject('usage')
+            ->inject('queueForUsage')
             ->inject('cache')
             ->inject('dbForProject')
             ->inject('getFunctionsDevice')
-            ->callback(fn($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Stats $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $getFunctionsDevice));
+            ->callback(fn($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $queueForUsage, $cache, $dbForProject, $getFunctionsDevice));
     }
 
     /**
@@ -60,14 +59,14 @@ class Builds extends Action
      * @param Database $dbForConsole
      * @param Event $queueForEvents
      * @param Func $queueForFunctions
-     * @param Stats $usage
+     * @param Usage $queueForUsage
      * @param Cache $cache
      * @param Database $dbForProject
      * @param callable $getFunctionsDevice
      * @return void
      * @throws \Utopia\Database\Exception
      */
-    public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Stats $usage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice): void
+    public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, callable $getFunctionsDevice): void
     {
         $payload = $message->getPayload() ?? [];
 
@@ -86,7 +85,7 @@ class Builds extends Action
             case BUILD_TYPE_RETRY:
                 Console::info('Creating build for deployment: ' . $deployment->getId());
                 $github = new GitHub($cache);
-                $this->buildDeployment($getFunctionsDevice, $queueForFunctions, $queueForEvents, $usage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template);
+                $this->buildDeployment($getFunctionsDevice, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template);
                 break;
 
             default:
@@ -98,7 +97,7 @@ class Builds extends Action
      * @param callable $getFunctionsDevice
      * @param Func $queueForFunctions
      * @param Event $queueForEvents
-     * @param Stats $usage
+     * @param Usage $queueForUsage
      * @param Database $dbForConsole
      * @param Database $dbForProject
      * @param GitHub $github
@@ -110,7 +109,7 @@ class Builds extends Action
      * @throws \Utopia\Database\Exception
      * @throws Exception
      */
-    protected function buildDeployment(callable $getFunctionsDevice, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template): void
+    protected function buildDeployment(callable $getFunctionsDevice, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $function, Document $deployment, Document $template): void
     {
         $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
 
@@ -529,19 +528,16 @@ class Builds extends Action
                 roles: $target['roles']
             );
 
-            /** Update usage stats */
-            if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
-                $usage
-                    ->setParam('projectInternalId', $project->getInternalId())
-                    ->setParam('projectId', $project->getId())
-                    ->setParam('functionId', $function->getId())
-                    ->setParam('builds.{scope}.compute', 1)
-                    ->setParam('buildStatus', $build->getAttribute('status', ''))
-                    ->setParam('buildTime', $build->getAttribute('duration'))
-                    ->setParam('networkRequestSize', 0)
-                    ->setParam('networkResponseSize', 0)
-                    ->submit();
-            }
+            /** Trigger usage queue */
+            $queueForUsage
+                ->setProject($project)
+                ->addMetric(METRIC_BUILDS, 1) // per project
+                ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
+                ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
+                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function
+                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0))
+                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000)
+                ->trigger();
         }
     }
 

src/Appwrite/Platform/Workers/Certificates.php

@@ -426,29 +426,31 @@ class Certificates extends Action
             $locale->setDefault('en');
         }
 
-        $body = Template::fromFile(__DIR__ . '/../config/locale/templates/email-base.tpl');
-
         $subject = \sprintf($locale->getText("emails.certificate.subject"), $domain);
-        $body
-            ->setParam('{{domain}}', $domain)
-            ->setParam('{{error}}', $errorMessage)
-            ->setParam('{{attempt}}', $attempt)
-            ->setParam('{{subject}}', $subject)
-            ->setParam('{{hello}}', $locale->getText("emails.certificate.hello"))
+
+        $message = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-inner-base.tpl');
+        $message
             ->setParam('{{body}}', $locale->getText("emails.certificate.body"))
-            ->setParam('{{redirect}}', 'https://' . $domain)
+            ->setParam('{{hello}}', $locale->getText("emails.certificate.hello"))
             ->setParam('{{footer}}', $locale->getText("emails.certificate.footer"))
             ->setParam('{{thanks}}', $locale->getText("emails.certificate.thanks"))
-            ->setParam('{{signature}}', $locale->getText("emails.certificate.signature"))
-            ->setParam('{{project}}', 'Console')
-            ->setParam('{{direction}}', $locale->getText('settings.direction'))
-            ->setParam('{{bg-body}}', '#f7f7f7')
-            ->setParam('{{bg-content}}', '#ffffff')
-            ->setParam('{{text-content}}', '#000000');
+            ->setParam('{{signature}}', $locale->getText("emails.certificate.signature"));
+        $body = $message->render();
+
+        $emailVariables = [
+            'direction' => $locale->getText('settings.direction'),
+            'domain' => $domain,
+            'error' => '<br><pre>' . $errorMessage . '</pre>',
+            'attempt' => $attempt,
+            'project' => 'Console',
+            'redirect' => 'https://' . $domain,
+        ];
 
         $queueForMails
             ->setRecipient(App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS'))
-            ->setBody($body->render())
+            ->setSubject($subject)
+            ->setBody($body)
+            ->setVariables($emailVariables)
             ->setName('Appwrite Administrator')
             ->trigger();
     }

src/Appwrite/Platform/Workers/Deletes.php

@@ -2,6 +2,7 @@
 
 namespace Appwrite\Platform\Workers;
 
+use Appwrite\Auth\Auth;
 use Executor\Executor;
 use Throwable;
 use Utopia\Abuse\Abuse;
@@ -45,14 +46,17 @@ class Deletes extends Action
             ->inject('getFunctionsDevice')
             ->inject('getBuildsDevice')
             ->inject('getCacheDevice')
-            ->callback(fn($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice));
+            ->inject('abuseRetention')
+            ->inject('executionRetention')
+            ->inject('auditRetention')
+            ->callback(fn ($message, $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention) => $this->action($message, $dbForConsole, $getProjectDB, $getFilesDevice, $getFunctionsDevice, $getBuildsDevice, $getCacheDevice, $abuseRetention, $executionRetention, $auditRetention));
     }
 
     /**
      * @throws Exception
      * @throws Throwable
      */
-    public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice): void
+    public function action(Message $message, Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, string $abuseRetention, string $executionRetention, string $auditRetention): void
     {
         $payload = $message->getPayload() ?? [];
 
@@ -114,12 +118,12 @@ class Deletes extends Action
                 break;
 
             case DELETE_TYPE_EXECUTIONS:
-                $this->deleteExecutionLogs($dbForConsole, $getProjectDB, $datetime);
+                $this->deleteExecutionLogs($project, $getProjectDB, $executionRetention);
                 break;
 
             case DELETE_TYPE_AUDIT:
-                if (!empty($datetime)) {
-                    $this->deleteAuditLogs($dbForConsole, $getProjectDB, $datetime);
+                if (!$project->isEmpty()) {
+                    $this->deleteAuditLogs($project, $getProjectDB, $auditRetention);
                 }
 
                 if (!$document->isEmpty()) {
@@ -127,7 +131,7 @@ class Deletes extends Action
                 }
                 break;
             case DELETE_TYPE_ABUSE:
-                $this->deleteAbuseLogs($dbForConsole, $getProjectDB, $datetime);
+                $this->deleteAbuseLogs($project, $getProjectDB, $abuseRetention);
                 break;
 
             case DELETE_TYPE_REALTIME:
@@ -135,10 +139,10 @@ class Deletes extends Action
                 break;
 
             case DELETE_TYPE_SESSIONS:
-                $this->deleteExpiredSessions($dbForConsole, $getProjectDB);
+                $this->deleteExpiredSessions($project, $getProjectDB);
                 break;
             case DELETE_TYPE_USAGE:
-                $this->deleteUsageStats($dbForConsole, $getProjectDB, $hourlyUsageRetentionDatetime);
+                $this->deleteUsageStats($project, $getProjectDB, $hourlyUsageRetentionDatetime);
                 break;
             case DELETE_TYPE_CACHE_BY_RESOURCE:
                 $this->deleteCacheByResource($project, $getProjectDB, $resource);
@@ -178,7 +182,8 @@ class Deletes extends Action
                 $project = $dbForConsole->getDocument('projects', $document->getAttribute('projectId'));
 
                 if ($project->isEmpty()) {
-                    Console::warning('Unable to delete schedule for function ' . $document->getAttribute('resourceId'));
+                    $dbForConsole->deleteDocument('schedules', $document->getId());
+                    Console::success('Deleted schedule for deleted project ' . $document->getAttribute('projectId'));
                     return;
                 }
 
@@ -336,16 +341,14 @@ class Deletes extends Action
      * @return void
      * @throws Exception
      */
-    private function deleteUsageStats(Database $dbForConsole, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void
+    private function deleteUsageStats(Document $project, callable $getProjectDB, string $hourlyUsageRetentionDatetime): void
     {
-        $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $hourlyUsageRetentionDatetime) {
-            $dbForProject = $getProjectDB($project);
-            // Delete Usage stats
-            $this->deleteByGroup('stats', [
-                Query::lessThan('time', $hourlyUsageRetentionDatetime),
-                Query::equal('period', ['1h']),
-            ], $dbForProject);
-        });
+        $dbForProject = $getProjectDB($project);
+        // Delete Usage stats
+        $this->deleteByGroup('stats_v2', [
+            Query::lessThan('time', $hourlyUsageRetentionDatetime),
+            Query::equal('period', ['1h']),
+        ], $dbForProject);
     }
 
     /**
@@ -386,6 +389,7 @@ class Deletes extends Action
      */
     private function deleteProjectsByTeam(Database $dbForConsole, callable $getProjectDB, callable $getFilesDevice, callable $getFunctionsDevice, callable $getBuildsDevice, callable $getCacheDevice, Document $document): void
     {
+
         $projects = $dbForConsole->find('projects', [
             Query::equal('teamInternalId', [$document->getInternalId()])
         ]);
@@ -450,6 +454,21 @@ class Deletes extends Action
             Query::equal('projectInternalId', [$projectInternalId])
         ], $dbForConsole);
 
+        // Delete VCS Installations
+        $this->deleteByGroup('installations', [
+            Query::equal('projectInternalId', [$projectInternalId])
+        ], $dbForConsole);
+
+        // Delete VCS Repositories
+        $this->deleteByGroup('repositories', [
+            Query::equal('projectInternalId', [$projectInternalId]),
+        ], $dbForConsole);
+
+        // Delete VCS commments
+        $this->deleteByGroup('vcsComments', [
+            Query::equal('projectInternalId', [$projectInternalId]),
+        ], $dbForConsole);
+
         // Delete metadata tables
         try {
             $dbForProject->deleteCollection('_metadata');
@@ -526,15 +545,13 @@ class Deletes extends Action
      * @return void
      * @throws Exception
      */
-    private function deleteExecutionLogs(database $dbForConsole, callable $getProjectDB, string $datetime): void
+    private function deleteExecutionLogs(Document $project, callable $getProjectDB, string $datetime): void
     {
-        $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) {
-            $dbForProject = $getProjectDB($project);
-            // Delete Executions
-            $this->deleteByGroup('executions', [
-                Query::lessThan('$createdAt', $datetime)
-            ], $dbForProject);
-        });
+        $dbForProject = $getProjectDB($project);
+        // Delete Executions
+        $this->deleteByGroup('executions', [
+            Query::lessThan('$createdAt', $datetime)
+        ], $dbForProject);
     }
 
     /**
@@ -543,20 +560,16 @@ class Deletes extends Action
      * @return void
      * @throws Exception|Throwable
      */
-    private function deleteExpiredSessions(Database $dbForConsole, callable $getProjectDB): void
+    private function deleteExpiredSessions(Document $project, callable $getProjectDB): void
     {
+        $dbForProject = $getProjectDB($project);
+        $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
+        $expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
 
-        $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($dbForConsole, $getProjectDB) {
-            $dbForProject = $getProjectDB($project);
-            $project = $dbForConsole->getDocument('projects', $project->getId());
-            $duration = $project->getAttribute('auths', [])['duration'] ?? Auth::TOKEN_EXPIRATION_LOGIN_LONG;
-            $expired = DateTime::addSeconds(new \DateTime(), -1 * $duration);
-
-            // Delete Sessions
-            $this->deleteByGroup('sessions', [
-                Query::lessThan('$createdAt', $expired)
-            ], $dbForProject);
-        });
+        // Delete Sessions
+        $this->deleteByGroup('sessions', [
+            Query::lessThan('$createdAt', $expired)
+        ], $dbForProject);
     }
 
     /**
@@ -580,22 +593,16 @@ class Deletes extends Action
      * @return void
      * @throws Exception
      */
-    private function deleteAbuseLogs(Database $dbForConsole, callable $getProjectDB, string $datetime): void
+    private function deleteAbuseLogs(Document $project, callable $getProjectDB, string $abuseRetention): void
     {
-        if (empty($datetime)) {
-            throw new Exception('Failed to delete audit logs. No datetime provided');
+        $projectId = $project->getId();
+        $dbForProject = $getProjectDB($project);
+        $timeLimit = new TimeLimit("", 0, 1, $dbForProject);
+        $abuse = new Abuse($timeLimit);
+        $status = $abuse->cleanup($abuseRetention);
+        if (!$status) {
+            throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
         }
-
-        $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) {
-            $projectId = $project->getId();
-            $dbForProject = $getProjectDB($project);
-            $timeLimit = new TimeLimit("", 0, 1, $dbForProject);
-            $abuse = new Abuse($timeLimit);
-            $status = $abuse->cleanup($datetime);
-            if (!$status) {
-                throw new Exception('Failed to delete Abuse logs for project ' . $projectId);
-            }
-        });
     }
 
     /**
@@ -605,21 +612,15 @@ class Deletes extends Action
      * @return void
      * @throws Exception
      */
-    private function deleteAuditLogs(Database $dbForConsole, callable $getProjectDB, string $datetime): void
+    private function deleteAuditLogs(Document $project, callable $getProjectDB, string $auditRetention): void
     {
-        if (empty($datetime)) {
-            throw new Exception('Failed to delete audit logs. No datetime provided');
+        $projectId = $project->getId();
+        $dbForProject = $getProjectDB($project);
+        $audit = new Audit($dbForProject);
+        $status = $audit->cleanup($auditRetention);
+        if (!$status) {
+            throw new Exception('Failed to delete Audit logs for project' . $projectId);
         }
-
-        $this->deleteForProjectIds($dbForConsole, function (Document $project) use ($getProjectDB, $datetime) {
-            $projectId = $project->getId();
-            $dbForProject = $getProjectDB($project);
-            $audit = new Audit($dbForProject);
-            $status = $audit->cleanup($datetime);
-            if (!$status) {
-                throw new Exception('Failed to delete Audit logs for project' . $projectId);
-            }
-        });
     }
 
     /**
@@ -688,7 +689,7 @@ class Deletes extends Action
             $this->deleteDeploymentFiles($functionsStorage, $document);
         });
 
-         /**
+        /**
          * Delete builds
          */
         Console::info("Deleting builds for function " . $functionId);
@@ -709,6 +710,23 @@ class Deletes extends Action
             Query::equal('functionInternalId', [$functionInternalId])
         ], $dbForProject);
 
+        /**
+         * Delete VCS Repositories and VCS Comments
+         */
+        Console::info("Deleting VCS repositories and comments linked to function " . $functionId);
+        $this->deleteByGroup('repositories', [
+            Query::equal('projectInternalId', [$project->getInternalId()]),
+            Query::equal('resourceInternalId', [$functionInternalId]),
+            Query::equal('resourceType', ['function']),
+        ], $dbForConsole, function (Document $document) use ($dbForConsole) {
+            $providerRepositoryId = $document->getAttribute('providerRepositoryId', '');
+            $projectInternalId = $document->getAttribute('projectInternalId', '');
+            $this->deleteByGroup('vcsComments', [
+                Query::equal('providerRepositoryId', [$providerRepositoryId]),
+                Query::equal('projectInternalId', [$projectInternalId]),
+            ], $dbForConsole);
+        });
+
         /**
          * Request executor to delete all deployment containers
          */
@@ -837,39 +855,6 @@ class Deletes extends Action
         }
     }
 
-    /**
-     * @param Database $dbForConsole
-     * @param callable $callback
-     * @throws Exception
-     */
-    private function deleteForProjectIds(database $dbForConsole, callable $callback): void
-    {
-        // TODO: @Meldiron name of this method no longer matches. It does not delete, and it gives whole document
-        $count = 0;
-        $chunk = 0;
-        $limit = 50;
-        $sum = $limit;
-        $executionStart = \microtime(true);
-
-        while ($sum === $limit) {
-            $projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($chunk * $limit)]);
-
-            $chunk++;
-
-            /** @var string[] $projectIds */
-            $sum = count($projects);
-
-            Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects');
-            foreach ($projects as $project) {
-                $callback($project);
-                $count++;
-            }
-        }
-
-        $executionEnd = \microtime(true);
-        Console::info("Found {$count} projects " . ($executionEnd - $executionStart) . " seconds");
-    }
-
     /**
      * @param string $collection collectionID
      * @param array $queries
@@ -920,17 +905,28 @@ class Deletes extends Action
         $count = 0;
         $chunk = 0;
         $limit = 50;
+        $results = [];
         $sum = $limit;
+        $cursor = null;
 
         $executionStart = \microtime(true);
 
         while ($sum === $limit) {
             $chunk++;
 
-            $results = $database->find($collection, \array_merge([Query::limit($limit)], $queries));
+            $mergedQueries = \array_merge([Query::limit($limit)], $queries);
+            if ($cursor instanceof Document) {
+                $mergedQueries[] = Query::cursorAfter($cursor);
+            }
+
+            $results = $database->find($collection, $mergedQueries);
 
             $sum = count($results);
 
+            if ($sum > 0) {
+                $cursor = $results[$sum - 1];
+            }
+
             foreach ($results as $document) {
                 if (is_callable($callback)) {
                     $callback($document);

src/Appwrite/Platform/Workers/Functions.php

@@ -2,9 +2,9 @@
 
 namespace Appwrite\Platform\Workers;
 
-use Appwrite\Usage\Stats;
 use Appwrite\Event\Event;
 use Appwrite\Event\Func;
+use Appwrite\Event\Usage;
 use Appwrite\Messaging\Adapter\Realtime;
 use Appwrite\Utopia\Response\Model\Execution;
 use Exception;
@@ -39,13 +39,14 @@ class Functions extends Action
     {
         $this
             ->desc('Functions worker')
+            ->groups(['functions'])
             ->inject('message')
             ->inject('dbForProject')
             ->inject('queueForFunctions')
             ->inject('queueForEvents')
-            ->inject('usage')
+            ->inject('queueForUsage')
             ->inject('log')
-            ->callback(fn(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $usage, $log));
+            ->callback(fn(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForUsage, $log));
     }
 
     /**
@@ -53,7 +54,7 @@ class Functions extends Action
      * @param Database $dbForProject
      * @param Func $queueForFunctions
      * @param Event $queueForEvents
-     * @param Stats $usage
+     * @param Usage $queueForUsage
      * @param Log $log
      * @return void
      * @throws Authorization
@@ -61,7 +62,7 @@ class Functions extends Action
      * @throws \Utopia\Database\Exception
      * @throws Conflict
      */
-    public function action(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Stats $usage, Log $log): void
+    public function action(Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Log $log): void
     {
         $payload = $message->getPayload() ?? [];
 
@@ -77,7 +78,7 @@ class Functions extends Action
 
         $type = $payload['type'] ?? '';
         $events = $payload['events'] ?? [];
-        $data = $payload['data'] ?? '';
+        $data = $payload['body'] ?? '';
         $eventData = $payload['payload'] ?? '';
         $project = new Document($payload['project'] ?? []);
         $function = new Document($payload['function'] ?? []);
@@ -117,7 +118,7 @@ class Functions extends Action
                         log: $log,
                         dbForProject: $dbForProject,
                         queueForFunctions: $queueForFunctions,
-                        usage: $usage,
+                        queueForUsage: $queueForUsage,
                         queueForEvents: $queueForEvents,
                         project: $project,
                         function: $function,
@@ -153,7 +154,7 @@ class Functions extends Action
                     log: $log,
                     dbForProject: $dbForProject,
                     queueForFunctions: $queueForFunctions,
-                    usage: $usage,
+                    queueForUsage: $queueForUsage,
                     queueForEvents: $queueForEvents,
                     project: $project,
                     function: $function,
@@ -174,7 +175,7 @@ class Functions extends Action
                     log: $log,
                     dbForProject: $dbForProject,
                     queueForFunctions: $queueForFunctions,
-                    usage: $usage,
+                    queueForUsage: $queueForUsage,
                     queueForEvents: $queueForEvents,
                     project: $project,
                     function: $function,
@@ -197,7 +198,7 @@ class Functions extends Action
      * @param Log $log
      * @param Database $dbForProject
      * @param Func $queueForFunctions
-     * @param Stats $usage
+     * @param Usage $queueForUsage
      * @param Event $queueForEvents
      * @param Document $project
      * @param Document $function
@@ -221,7 +222,7 @@ class Functions extends Action
         Log $log,
         Database $dbForProject,
         Func $queueForFunctions,
-        stats $usage,
+        Usage $queueForUsage,
         Event $queueForEvents,
         Document $project,
         Document $function,
@@ -420,6 +421,16 @@ class Functions extends Action
 
             $error = $th->getMessage();
             $errorCode = $th->getCode();
+        } finally {
+            /** Trigger usage queue */
+            $queueForUsage
+                ->setProject($project)
+                ->addMetric(METRIC_EXECUTIONS, 1)
+                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1)
+                ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
+                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
+                ->trigger()
+            ;
         }
 
         if ($function->getAttribute('logging')) {
@@ -471,19 +482,5 @@ class Functions extends Action
         if (!empty($error)) {
             throw new Exception($error, $errorCode);
         }
-
-        /** Update usage stats */
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
-            $usage
-                ->setParam('projectId', $project->getId())
-                ->setParam('projectInternalId', $project->getInternalId())
-                ->setParam('functionId', $function->getId()) // TODO: We should use functionInternalId in usage stats
-                ->setParam('executions.{scope}.compute', 1)
-                ->setParam('executionStatus', $execution->getAttribute('status', ''))
-                ->setParam('executionTime', $execution->getAttribute('duration'))
-                ->setParam('networkRequestSize', 0)
-                ->setParam('networkResponseSize', 0)
-                ->submit();
-        }
     }
 }

src/Appwrite/Platform/Workers/Mails.php

@@ -5,6 +5,7 @@ namespace Appwrite\Platform\Workers;
 use Appwrite\Template\Template;
 use Exception;
 use PHPMailer\PHPMailer\PHPMailer;
+use Swoole\Runtime;
 use Utopia\App;
 use Utopia\CLI\Console;
 use Utopia\Platform\Action;
@@ -39,7 +40,7 @@ class Mails extends Action
      */
     public function action(Message $message, Registry $register): void
     {
-
+        Runtime::setHookFlags(SWOOLE_HOOK_ALL ^ SWOOLE_HOOK_TCP);
         $payload = $message->getPayload() ?? [];
 
         if (empty($payload)) {
@@ -57,13 +58,25 @@ class Mails extends Action
         $subject = $payload['subject'];
         $variables = $payload['variables'];
         $name = $payload['name'];
-        $body = Template::fromFile(__DIR__ . '/../../../../app/config/locale/templates/email-base.tpl');
-
-        foreach ($variables as $key => $value) {
-            $body->setParam('{{' . $key . '}}', $value);
+        $body = $payload['body'];
+        $attachment = $payload['attachment'] ?? [];
+        $bodyTemplate = $payload['bodyTemplate'];
+        if (empty($bodyTemplate)) {
+            $bodyTemplate = __DIR__ . '/../../../../app/config/locale/templates/email-base.tpl';
         }
+        $bodyTemplate = Template::fromFile($bodyTemplate);
+        $bodyTemplate->setParam('{{body}}', $body);
+        foreach ($variables as $key => $value) {
+            $bodyTemplate->setParam('{{' . $key . '}}', $value);
+        }
+        $body = $bodyTemplate->render();
 
-        $body = $body->render();
+        $subjectTemplate = Template::fromString($subject);
+        foreach ($variables as $key => $value) {
+            $subjectTemplate->setParam('{{' . $key . '}}', $value);
+        }
+        // render() will return the subject in <p> tags, so use strip_tags() to remove them
+        $subject = \strip_tags($subjectTemplate->render());
 
         /** @var PHPMailer $mail */
         $mail = empty($smtp)
@@ -80,6 +93,14 @@ class Mails extends Action
         $mail->Subject = $subject;
         $mail->Body = $body;
         $mail->AltBody = \strip_tags($body);
+        if (!empty($attachment['content'] ?? '')) {
+            $mail->AddStringAttachment(
+                base64_decode($attachment['content']),
+                $attachment['filename'] ?? 'unknown.file',
+                $attachment['encoding'] ?? PHPMailer::ENCODING_BASE64,
+                $attachment['type'] ?? 'plain/text'
+            );
+        }
 
         try {
             $mail->send();

src/Appwrite/Platform/Workers/Messaging.php

@@ -6,7 +6,7 @@ use Exception;
 use Utopia\App;
 use Utopia\CLI\Console;
 use Utopia\DSN\DSN;
-use Utopia\Messaging\Messages\Sms;
+use Utopia\Messaging\Messages\SMS;
 use Utopia\Messaging\Adapters\SMS\Mock;
 use Utopia\Messaging\Adapters\SMS\Msg91;
 use Utopia\Messaging\Adapters\SMS\Telesign;

src/Appwrite/Platform/Workers/Usage.php

@@ -0,0 +1,244 @@
+<?php
+
+namespace Appwrite\Platform\Workers;
+
+use Exception;
+use Utopia\CLI\Console;
+use Utopia\Database\Database;
+use Utopia\Database\Document;
+use Utopia\Platform\Action;
+use Utopia\Queue\Message;
+
+class Usage extends Action
+{
+    protected static array $stats = [];
+    protected array $periods = [
+        '1h' => 'Y-m-d H:00',
+        '1d' => 'Y-m-d 00:00',
+        'inf' => '0000-00-00 00:00'
+    ];
+
+    protected const INFINITY_PERIOD = '_inf_';
+
+    public static function getName(): string
+    {
+        return 'usage';
+    }
+
+    /**
+     * @throws Exception
+     */
+    public function __construct()
+    {
+
+        $this
+            ->desc('Usage worker')
+            ->inject('message')
+            ->inject('pools')
+            ->inject('cache')
+            ->callback(function ($message, $pools, $cache) {
+                 $this->action($message, $pools, $cache);
+            });
+    }
+
+    /**
+     * @param Message $message
+     * @param $pools
+     * @param $cache
+     * @return void
+     * @throws \Utopia\Database\Exception
+     */
+    public function action(Message $message, $pools, $cache): void
+    {
+        $payload = $message->getPayload() ?? [];
+
+        if (empty($payload)) {
+            throw new Exception('Missing payload');
+        }
+
+        $payload = $message->getPayload() ?? [];
+        $project = new Document($payload['project'] ?? []);
+        $projectId = $project->getInternalId();
+        foreach ($payload['reduce'] ?? [] as $document) {
+            if (empty($document)) {
+                continue;
+            }
+
+            $this->reduce(
+                database: $project->getAttribute('database'),
+                projectInternalId: $project->getInternalId(),
+                document: new Document($document),
+                metrics:  $payload['metrics'],
+                pools: $pools,
+                cache: $cache
+            );
+        }
+
+        self::$stats[$projectId]['database'] = $project->getAttribute('database');
+        foreach ($payload['metrics'] ?? [] as $metric) {
+            if (!isset(self::$stats[$projectId]['keys'][$metric['key']])) {
+                self::$stats[$projectId]['keys'][$metric['key']] = $metric['value'];
+                continue;
+            }
+            self::$stats[$projectId]['keys'][$metric['key']] += $metric['value'];
+        }
+    }
+
+
+     /**
+     * On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files.
+     * When we remove a parent document we need to deduct his children aggregation from the project scope.
+
+     * @param $database
+     * @param $projectInternalId
+     * @param Document $document
+     * @param array $metrics
+     * @param $pools
+     * @param $cache
+     * @return void
+     */
+    private function reduce($database, $projectInternalId, Document $document, array &$metrics, $pools, $cache)
+    {
+        try {
+            $dbForProject = new Database(
+                $pools
+                ->get($database)
+                ->pop()
+                ->getResource(),
+                $cache
+            );
+
+            $dbForProject->setNamespace('_' . $projectInternalId);
+
+            switch (true) {
+                case $document->getCollection() === 'users': // users
+                    $sessions = count($document->getAttribute(METRIC_SESSIONS, 0));
+                    if (!empty($sessions)) {
+                        $metrics[] = [
+                            'key' => METRIC_SESSIONS,
+                            'value' => ($sessions * -1),
+                        ];
+                    }
+                    break;
+                case $document->getCollection() === 'databases': // databases
+                    $collections = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
+                    $documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
+                    if (!empty($collections['value'])) {
+                        $metrics[] = [
+                        'key' => METRIC_COLLECTIONS,
+                        'value' => ($collections['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($documents['value'])) {
+                        $metrics[] = [
+                        'key' => METRIC_DOCUMENTS,
+                        'value' => ($documents['value'] * -1),
+                        ];
+                    }
+                    break;
+                case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
+                    $parts = explode('_', $document->getCollection());
+                    $databaseInternalId = $parts[1] ?? 0;
+                    $documents = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
+
+                    if (!empty($documents['value'])) {
+                        $metrics[] = [
+                            'key' => METRIC_DOCUMENTS,
+                            'value' => ($documents['value'] * -1),
+                        ];
+                        $metrics[] = [
+                            'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS),
+                            'value' => ($documents['value'] * -1),
+                        ];
+                    }
+                    break;
+
+                case $document->getCollection() === 'buckets':
+                    $files = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
+                    $storage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
+
+                    if (!empty($files['value'])) {
+                        $metrics[] = [
+                            'key' => METRIC_FILES,
+                            'value' => ($files['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($storage['value'])) {
+                        $metrics[] = [
+                            'key' => METRIC_FILES_STORAGE,
+                            'value' => ($storage['value'] * -1),
+                        ];
+                    }
+                    break;
+
+                case $document->getCollection() === 'functions':
+                    $deployments = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
+                    $deploymentsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
+                    $builds = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD .  str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
+                    $buildsStorage = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
+                    $buildsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
+                    $executions = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD .  str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
+                    $executionsCompute = $dbForProject->getDocument('stats_v2', md5(self::INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
+
+                    if (!empty($deployments['value'])) {
+                        $metrics[] = [
+                            'key' => METRIC_DEPLOYMENTS,
+                            'value' => ($deployments['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($deploymentsStorage['value'])) {
+                        $metrics[] = [
+                            'key' => METRIC_DEPLOYMENTS_STORAGE,
+                            'value' => ($deploymentsStorage['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($builds['value'])) {
+                        $metrics[] = [
+                            'key' => METRIC_BUILDS,
+                            'value' => ($builds['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($buildsStorage['value'])) {
+                        $metrics[] = [
+                            'key' => METRIC_BUILDS_STORAGE,
+                            'value' => ($buildsStorage['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($buildsCompute['value'])) {
+                        $metrics[] = [
+                            'key' => METRIC_BUILDS_COMPUTE,
+                            'value' => ($buildsCompute['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($executions['value'])) {
+                        $metrics[] = [
+                            'key' => METRIC_EXECUTIONS,
+                            'value' => ($executions['value'] * -1),
+                        ];
+                    }
+
+                    if (!empty($executionsCompute['value'])) {
+                        $metrics[] = [
+                            'key' => METRIC_EXECUTIONS_COMPUTE,
+                            'value' => ($executionsCompute['value'] * -1),
+                        ];
+                    }
+                    break;
+                default:
+                    break;
+            }
+        } catch (\Exception $e) {
+            console::error("[reducer] " . " {DateTime::now()} " .  " {$projectInternalId} " . " {$e->getMessage()}");
+        } catch (\Throwable $e) {
+        } finally {
+            $pools->reclaim();
+        }
+    }
+}

src/Appwrite/Platform/Workers/UsageHook.php

@@ -0,0 +1,105 @@
+<?php
+
+namespace Appwrite\Platform\Workers;
+
+use Utopia\App;
+use Utopia\Database\Database;
+use Utopia\Database\Document;
+use Utopia\Database\Exception\Duplicate;
+use Utopia\Platform\Action;
+use Utopia\CLI\Console;
+use Swoole\Timer;
+
+class UsageHook extends Usage
+{
+    public static function getName(): string
+    {
+        return 'usageHook';
+    }
+
+    public function __construct()
+    {
+
+        $this
+            ->setType(Action::TYPE_WORKER_START)
+            ->inject('register')
+            ->inject('cache')
+            ->inject('pools')
+            ->callback(function ($register, $cache, $pools) {
+                $this->action($register, $cache, $pools);
+            })
+        ;
+    }
+
+    /**
+     * @param $register
+     * @param $cache
+     * @param $pools
+     * @return void
+     */
+    public function action($register, $cache, $pools): void
+    {
+        Timer::tick(30000, function () use ($register, $cache, $pools) {
+
+            $offset = count(self::$stats);
+            $projects = array_slice(self::$stats, 0, $offset, true);
+            array_splice(self::$stats, 0, $offset);
+
+            foreach ($projects as $projectInternalId => $project) {
+                try {
+                    $dbForProject = new Database(
+                        $pools
+                            ->get($project['database'])
+                            ->pop()
+                            ->getResource(),
+                        $cache
+                    );
+
+                    $dbForProject->setNamespace('_' . $projectInternalId);
+
+                    foreach ($project['keys'] ?? [] as $key => $value) {
+                        if ($value == 0) {
+                            continue;
+                        }
+
+                        foreach ($this->periods as $period => $format) {
+                            $time = 'inf' === $period ? null : date($format, time());
+                            $id = \md5("{$time}_{$period}_{$key}");
+
+                            try {
+                                $dbForProject->createDocument('stats_v2', new Document([
+                                    '$id' => $id,
+                                    'period' => $period,
+                                    'time' => $time,
+                                    'metric' => $key,
+                                    'value' => $value,
+                                    'region' => App::getEnv('_APP_REGION', 'default'),
+                                ]));
+                            } catch (Duplicate $th) {
+                                if ($value < 0) {
+                                    $dbForProject->decreaseDocumentAttribute(
+                                        'stats_v2',
+                                        $id,
+                                        'value',
+                                        abs($value)
+                                    );
+                                } else {
+                                    $dbForProject->increaseDocumentAttribute(
+                                        'stats_v2',
+                                        $id,
+                                        'value',
+                                        $value
+                                    );
+                                }
+                            }
+                        }
+                    }
+                } catch (\Exception $e) {
+                    console::error("[logger] " . " {DateTime::now()} " .  " {$projectInternalId} " . " {$e->getMessage()}");
+                } finally {
+                    $pools->reclaim();
+                }
+            }
+        });
+    }
+}

src/Appwrite/Platform/Workers/Webhooks.php

@@ -25,7 +25,7 @@ class Webhooks extends Action
         $this
             ->desc('Webhooks worker')
             ->inject('message')
-            ->callback(fn($message) => $this->action($message));
+            ->callback(fn ($message) => $this->action($message));
     }
 
     /**
@@ -48,7 +48,7 @@ class Webhooks extends Action
 
         foreach ($project->getAttribute('webhooks', []) as $webhook) {
             if (array_intersect($webhook->getAttribute('events', []), $events)) {
-                    $this->execute($events, $webhookPayload, $webhook, $user, $project);
+                $this->execute($events, $webhookPayload, $webhook, $user, $project);
             }
         }
 
@@ -78,7 +78,9 @@ class Webhooks extends Action
         \curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
         \curl_setopt($ch, CURLOPT_POSTFIELDS, $payload);
         \curl_setopt($ch, CURLOPT_HEADER, 0);
-        \curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
+        \curl_setopt($ch, CURLOPT_RETURNTRANSFER, 0);
+        \curl_setopt($ch, CURLOPT_TIMEOUT, 15);
+        \curl_setopt($ch, CURLOPT_MAXFILESIZE, 5242880);
         \curl_setopt($ch, CURLOPT_USERAGENT, \sprintf(
             APP_USERAGENT,
             App::getEnv('_APP_VERSION', 'UNKNOWN'),
@@ -88,16 +90,17 @@ class Webhooks extends Action
             $ch,
             CURLOPT_HTTPHEADER,
             [
-            'Content-Type: application/json',
-            'Content-Length: ' . \strlen($payload),
-            'X-' . APP_NAME . '-Webhook-Id: ' . $webhook->getId(),
-            'X-' . APP_NAME . '-Webhook-Events: ' . implode(',', $events),
-            'X-' . APP_NAME . '-Webhook-Name: ' . $webhook->getAttribute('name', ''),
-            'X-' . APP_NAME . '-Webhook-User-Id: ' . $user->getId(),
-            'X-' . APP_NAME . '-Webhook-Project-Id: ' . $project->getId(),
-            'X-' . APP_NAME . '-Webhook-Signature: ' . $signature,
+                'Content-Type: application/json',
+                'Content-Length: ' . \strlen($payload),
+                'X-' . APP_NAME . '-Webhook-Id: ' . $webhook->getId(),
+                'X-' . APP_NAME . '-Webhook-Events: ' . implode(',', $events),
+                'X-' . APP_NAME . '-Webhook-Name: ' . $webhook->getAttribute('name', ''),
+                'X-' . APP_NAME . '-Webhook-User-Id: ' . $user->getId(),
+                'X-' . APP_NAME . '-Webhook-Project-Id: ' . $project->getId(),
+                'X-' . APP_NAME . '-Webhook-Signature: ' . $signature,
             ]
         );
+        curl_setopt($ch, CURLOPT_MAXREDIRS, 5);
 
         if (!$webhook->getAttribute('security', true)) {
             \curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);

src/Appwrite/Usage/Calculator.php

@@ -1,15 +0,0 @@
-<?php
-
-namespace Appwrite\Usage;
-
-abstract class Calculator
-{
-    protected string $region;
-
-    public function __construct(string $region)
-    {
-        $this->region = $region;
-    }
-
-    abstract public function collect(): void;
-}