usage updates

Fix: client error reporting

.env

@@ -77,7 +77,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000
 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600
 _APP_MAINTENANCE_RETENTION_ABUSE=86400
 _APP_MAINTENANCE_RETENTION_AUDIT=1209600
-_APP_USAGE_AGGREGATION_INTERVAL=60000
+_APP_USAGE_AGGREGATION_INTERVAL=30
 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400
 _APP_USAGE_STATS=enabled

Dockerfile

@@ -99,7 +99,8 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/worker-webhooks && \
     chmod +x /usr/local/bin/worker-migrations && \
     chmod +x /usr/local/bin/worker-hamster && \
-    chmod +x /usr/local/bin/worker-usage
+    chmod +x /usr/local/bin/worker-usage && \
+    chmod +x /usr/local/bin/worker-usage-dump
 
 
 # Cloud Executabless

app/controllers/api/projects.php

@@ -4,6 +4,7 @@ use Appwrite\Auth\Auth;
 use Appwrite\Event\Delete;
 use Appwrite\Event\Validator\Event;
 use Appwrite\Extend\Exception;
+use Appwrite\Hooks\Hooks;
 use Appwrite\Network\Validator\Email;
 use Appwrite\Network\Validator\Origin;
 use Appwrite\Template\Template;
@@ -75,7 +76,8 @@ App::post('/v1/projects')
     ->inject('dbForConsole')
     ->inject('cache')
     ->inject('pools')
-    ->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools) {
+    ->inject('hooks')
+    ->action(function (string $projectId, string $name, string $teamId, string $region, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools, Hooks $hooks) {
 
 
         $team = $dbForConsole->getDocument('teams', $teamId);
@@ -217,6 +219,8 @@ App::post('/v1/projects')
             $dbForProject->createCollection($key, $attributes, $indexes);
         }
 
+        $hooks->trigger('afterProjectCreation', [ $project, $pools, $cache ]);
+
         $response
             ->setStatusCode(Response::STATUS_CODE_CREATED)
             ->dynamic($project, Response::MODEL_PROJECT);

app/controllers/general.php

@@ -607,10 +607,11 @@ App::error()
     ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log) {
         $version = App::getEnv('_APP_VERSION', 'UNKNOWN');
         $route = $utopia->getRoute();
-        $publish = true;
 
         if ($error instanceof AppwriteException) {
             $publish = $error->isPublishable();
+        } else {
+            $publish = $error->getCode() === 0 || $error->getCode() >= 500;
         }
 
         if ($logger && ($publish || $error->getCode() === 0)) {

app/worker.php

@@ -14,6 +14,7 @@ use Appwrite\Event\Mail;
 use Appwrite\Event\Migration;
 use Appwrite\Event\Phone;
 use Appwrite\Event\Usage;
+use Appwrite\Event\UsageDump;
 use Appwrite\Platform\Appwrite;
 use Swoole\Runtime;
 use Utopia\App;
@@ -146,6 +147,9 @@ Server::setResource('log', fn() => new Log());
 Server::setResource('queueForUsage', function (Connection $queue) {
     return new Usage($queue);
 }, ['queue']);
+Server::setResource('queueForUsageDump', function (Connection $queue) {
+    return new UsageDump($queue);
+}, ['queue']);
 Server::setResource('queue', function (Group $pools) {
     return $pools->get('queue')->pop()->getResource();
 }, ['pools']);
@@ -299,12 +303,9 @@ $worker
         Console::error('[Error] Line: ' . $error->getLine());
     });
 
-try {
-    $workerStart = $worker->getWorkerStart();
-} catch (\Throwable $error) {
-    $worker->workerStart();
-} finally {
-    Console::info("Worker $workerName  started");
-}
+$worker->workerStart()
+    ->action(function () use ($workerName) {
+        Console::info("Worker $workerName  started");
+    });
 
 $worker->start();

bin/worker-usage-dump

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/worker.php usage-dump $@

composer.lock

@@ -815,16 +815,16 @@
         },
         {
             "name": "symfony/polyfill-php80",
-            "version": "v1.28.0",
+            "version": "v1.29.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/polyfill-php80.git",
-                "reference": "6caa57379c4aec19c0a12a38b59b26487dcfe4b5"
+                "reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/6caa57379c4aec19c0a12a38b59b26487dcfe4b5",
-                "reference": "6caa57379c4aec19c0a12a38b59b26487dcfe4b5",
+                "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/87b68208d5c1188808dd7839ee1e6c8ec3b02f1b",
+                "reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b",
                 "shasum": ""
             },
             "require": {
@@ -832,9 +832,6 @@
             },
             "type": "library",
             "extra": {
-                "branch-alias": {
-                    "dev-main": "1.28-dev"
-                },
                 "thanks": {
                     "name": "symfony/polyfill",
                     "url": "https://github.com/symfony/polyfill"
@@ -878,7 +875,7 @@
                 "shim"
             ],
             "support": {
-                "source": "https://github.com/symfony/polyfill-php80/tree/v1.28.0"
+                "source": "https://github.com/symfony/polyfill-php80/tree/v1.29.0"
             },
             "funding": [
                 {
@@ -894,7 +891,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2023-01-26T09:26:14+00:00"
+            "time": "2024-01-29T20:11:03+00:00"
         },
         {
             "name": "utopia-php/abuse",
@@ -1190,16 +1187,16 @@
         },
         {
             "name": "utopia-php/database",
-            "version": "0.45.5",
+            "version": "0.45.6",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/database.git",
-                "reference": "0b66a017f817a910acb83e6aea92bccea9571fe6"
+                "reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/database/zipball/0b66a017f817a910acb83e6aea92bccea9571fe6",
-                "reference": "0b66a017f817a910acb83e6aea92bccea9571fe6",
+                "url": "https://api.github.com/repos/utopia-php/database/zipball/c7cc6d57683a4c13d9772dbeea343adb72c443fd",
+                "reference": "c7cc6d57683a4c13d9772dbeea343adb72c443fd",
                 "shasum": ""
             },
             "require": {
@@ -1240,9 +1237,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/database/issues",
-                "source": "https://github.com/utopia-php/database/tree/0.45.5"
+                "source": "https://github.com/utopia-php/database/tree/0.45.6"
             },
-            "time": "2024-01-08T17:08:15+00:00"
+            "time": "2024-02-01T02:33:43+00:00"
         },
         {
             "name": "utopia-php/domains",
@@ -1353,16 +1350,16 @@
         },
         {
             "name": "utopia-php/framework",
-            "version": "0.33.1",
+            "version": "0.33.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/http.git",
-                "reference": "b745607aa1875554a0ad52e28f6db918da1ce11c"
+                "reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/http/zipball/b745607aa1875554a0ad52e28f6db918da1ce11c",
-                "reference": "b745607aa1875554a0ad52e28f6db918da1ce11c",
+                "url": "https://api.github.com/repos/utopia-php/http/zipball/b1423ca3e3b61c6c4c2e619d2cb80672809a19f3",
+                "reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3",
                 "shasum": ""
             },
             "require": {
@@ -1392,9 +1389,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/http/issues",
-                "source": "https://github.com/utopia-php/http/tree/0.33.1"
+                "source": "https://github.com/utopia-php/http/tree/0.33.2"
             },
-            "time": "2024-01-17T16:48:32+00:00"
+            "time": "2024-01-31T10:35:59+00:00"
         },
         {
             "name": "utopia-php/image",
@@ -2471,16 +2468,16 @@
         },
         {
             "name": "doctrine/deprecations",
-            "version": "1.1.2",
+            "version": "1.1.3",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/deprecations.git",
-                "reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931"
+                "reference": "dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/deprecations/zipball/4f2d4f2836e7ec4e7a8625e75c6aa916004db931",
-                "reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931",
+                "url": "https://api.github.com/repos/doctrine/deprecations/zipball/dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab",
+                "reference": "dfbaa3c2d2e9a9df1118213f3b8b0c597bb99fab",
                 "shasum": ""
             },
             "require": {
@@ -2512,9 +2509,9 @@
             "homepage": "https://www.doctrine-project.org/",
             "support": {
                 "issues": "https://github.com/doctrine/deprecations/issues",
-                "source": "https://github.com/doctrine/deprecations/tree/1.1.2"
+                "source": "https://github.com/doctrine/deprecations/tree/1.1.3"
             },
-            "time": "2023-09-27T20:04:15+00:00"
+            "time": "2024-01-30T19:34:25+00:00"
         },
         {
             "name": "doctrine/instantiator",
@@ -4772,16 +4769,16 @@
         },
         {
             "name": "symfony/polyfill-ctype",
-            "version": "v1.28.0",
+            "version": "v1.29.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/polyfill-ctype.git",
-                "reference": "ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb"
+                "reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb",
-                "reference": "ea208ce43cbb04af6867b4fdddb1bdbf84cc28cb",
+                "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ef4d7e442ca910c4764bce785146269b30cb5fc4",
+                "reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4",
                 "shasum": ""
             },
             "require": {
@@ -4795,9 +4792,6 @@
             },
             "type": "library",
             "extra": {
-                "branch-alias": {
-                    "dev-main": "1.28-dev"
-                },
                 "thanks": {
                     "name": "symfony/polyfill",
                     "url": "https://github.com/symfony/polyfill"
@@ -4834,7 +4828,7 @@
                 "portable"
             ],
             "support": {
-                "source": "https://github.com/symfony/polyfill-ctype/tree/v1.28.0"
+                "source": "https://github.com/symfony/polyfill-ctype/tree/v1.29.0"
             },
             "funding": [
                 {
@@ -4850,20 +4844,20 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2023-01-26T09:26:14+00:00"
+            "time": "2024-01-29T20:11:03+00:00"
         },
         {
             "name": "symfony/polyfill-mbstring",
-            "version": "v1.28.0",
+            "version": "v1.29.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/polyfill-mbstring.git",
-                "reference": "42292d99c55abe617799667f454222c54c60e229"
+                "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/42292d99c55abe617799667f454222c54c60e229",
-                "reference": "42292d99c55abe617799667f454222c54c60e229",
+                "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9773676c8a1bb1f8d4340a62efe641cf76eda7ec",
+                "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec",
                 "shasum": ""
             },
             "require": {
@@ -4877,9 +4871,6 @@
             },
             "type": "library",
             "extra": {
-                "branch-alias": {
-                    "dev-main": "1.28-dev"
-                },
                 "thanks": {
                     "name": "symfony/polyfill",
                     "url": "https://github.com/symfony/polyfill"
@@ -4917,7 +4908,7 @@
                 "shim"
             ],
             "support": {
-                "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.28.0"
+                "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.29.0"
             },
             "funding": [
                 {
@@ -4933,7 +4924,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2023-07-28T09:04:16+00:00"
+            "time": "2024-01-29T20:11:03+00:00"
         },
         {
             "name": "textalk/websocket",
@@ -5133,5 +5124,5 @@
     "platform-overrides": {
         "php": "8.0"
     },
-    "plugin-api-version": "2.6.0"
+    "plugin-api-version": "2.2.0"
 }

docker-compose.yml

@@ -677,6 +677,38 @@ services:
       - _APP_LOGGING_CONFIG
       - _APP_USAGE_AGGREGATION_INTERVAL
 
+  appwrite-worker-usage-dump:
+    entrypoint: worker-usage-dump
+    <<: *x-logging
+    container_name: appwrite-worker-usage-dump
+    image: appwrite-dev
+    networks:
+      - appwrite
+    volumes:
+      - ./app:/usr/src/code/app
+      - ./src:/usr/src/code/src
+    depends_on:
+      - redis
+      - mariadb
+    environment:
+      - _APP_ENV
+      - _APP_WORKER_PER_CORE
+      - _APP_OPENSSL_KEY_V1
+      - _APP_DB_HOST
+      - _APP_DB_PORT
+      - _APP_DB_SCHEMA
+      - _APP_DB_USER
+      - _APP_DB_PASS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_USAGE_STATS
+      - _APP_LOGGING_PROVIDER
+      - _APP_LOGGING_CONFIG
+      - _APP_USAGE_AGGREGATION_INTERVAL
+
+
   appwrite-schedule:
     entrypoint: schedule
     <<: *x-logging

src/Appwrite/Event/Event.php

@@ -27,6 +27,9 @@ class Event
     public const USAGE_QUEUE_NAME = 'v1-usage';
     public const USAGE_CLASS_NAME = 'UsageV1';
 
+    public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump';
+    public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1';
+
     public const WEBHOOK_QUEUE_NAME = 'v1-webhooks';
     public const WEBHOOK_CLASS_NAME = 'WebhooksV1';
 

src/Appwrite/Event/UsageDump.php

@@ -0,0 +1,47 @@
+<?php
+
+namespace Appwrite\Event;
+
+use Utopia\Queue\Client;
+use Utopia\Queue\Connection;
+
+class UsageDump extends Event
+{
+    protected array $stats;
+
+    public function __construct(protected Connection $connection)
+    {
+        parent::__construct($connection);
+
+        $this
+            ->setQueue(Event::USAGE_DUMP_QUEUE_NAME)
+            ->setClass(Event::USAGE_DUMP_CLASS_NAME);
+    }
+
+    /**
+     * Add Stats.
+     *
+     * @param array $stats
+     * @return self
+     */
+    public function setStats(array $stats): self
+    {
+        $this->stats = $stats;
+
+        return $this;
+    }
+
+    /**
+     * Sends metrics to the usage worker.
+     *
+     * @return string|bool
+     */
+    public function trigger(): string|bool
+    {
+        $client = new Client($this->queue, $this->connection);
+
+        return $client->enqueue([
+            'stats' => $this->stats,
+        ]);
+    }
+}

src/Appwrite/Hooks/Hooks.php

@@ -16,11 +16,14 @@ class Hooks
 
     /**
      * @param mixed[] $params
+     * @return mixed
      */
-    public function trigger(string $name, array $params = [])
+    public function trigger(string $name, array $params = []): mixed
     {
         if (isset(self::$hooks[$name])) {
-            call_user_func_array(self::$hooks[$name], $params);
+            return call_user_func_array(self::$hooks[$name], $params);
         }
+
+        return null;
     }
 }

src/Appwrite/Platform/Services/Workers.php

@@ -14,7 +14,7 @@ use Appwrite\Platform\Workers\Builds;
 use Appwrite\Platform\Workers\Deletes;
 use Appwrite\Platform\Workers\Hamster;
 use Appwrite\Platform\Workers\Usage;
-use Appwrite\Platform\Workers\UsageHook;
+use Appwrite\Platform\Workers\UsageDump;
 use Appwrite\Platform\Workers\Migrations;
 
 class Workers extends Service
@@ -33,7 +33,7 @@ class Workers extends Service
             ->addAction(Builds::getName(), new Builds())
             ->addAction(Deletes::getName(), new Deletes())
             ->addAction(Hamster::getName(), new Hamster())
-            ->addAction(UsageHook::getName(), new UsageHook())
+            ->addAction(UsageDump::getName(), new UsageDump())
             ->addAction(Usage::getName(), new Usage())
             ->addAction(Migrations::getName(), new Migrations())
 

src/Appwrite/Platform/Workers/Usage.php

@@ -3,23 +3,23 @@
 namespace Appwrite\Platform\Workers;
 
 use Exception;
+use Utopia\App;
 use Utopia\CLI\Console;
-use Utopia\Database\Database;
 use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Platform\Action;
+use Appwrite\Event\UsageDump;
 use Utopia\Queue\Message;
 
 class Usage extends Action
 {
-    protected static array $stats = [];
-    protected array $periods = [
-        '1h' => 'Y-m-d H:00',
-        '1d' => 'Y-m-d 00:00',
-        'inf' => '0000-00-00 00:00'
-    ];
+    private array $stats = [];
+    private int $lastTriggeredTime = 0;
+    private int $keys = 0;
+    private const INFINITY_PERIOD = '_inf_';
+    private const KEYS_THRESHOLD = 10000;
+
 
-    protected const INFINITY_PERIOD = '_inf_';
     public static function getName(): string
     {
         return 'usage';
@@ -35,26 +35,31 @@ class Usage extends Action
         ->desc('Usage worker')
         ->inject('message')
         ->inject('getProjectDB')
-        ->callback(function (Message $message, callable $getProjectDB) {
-             $this->action($message, $getProjectDB);
+        ->inject('queueForUsageDump')
+        ->callback(function (Message $message, callable $getProjectDB, UsageDump $queueForUsageDump) {
+             $this->action($message, $getProjectDB, $queueForUsageDump);
         });
+
+        $this->lastTriggeredTime = time();
     }
 
     /**
      * @param Message $message
      * @param callable $getProjectDB
+     * @param UsageDump $queueForUsageDump
      * @return void
      * @throws \Utopia\Database\Exception
      * @throws Exception
      */
-    public function action(Message $message, callable $getProjectDB): void
+    public function action(Message $message, callable $getProjectDB, UsageDump $queueForUsageDump): void
     {
         $payload = $message->getPayload() ?? [];
         if (empty($payload)) {
             throw new Exception('Missing payload');
         }
+        //Todo Figure out way to preserve keys when the container is being recreated @shimonewman
 
-        $payload = $message->getPayload() ?? [];
+        $aggregationInterval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '20');
         $project = new Document($payload['project'] ?? []);
         $projectId = $project->getInternalId();
         foreach ($payload['reduce'] ?? [] as $document) {
@@ -69,17 +74,35 @@ class Usage extends Action
                 getProjectDB: $getProjectDB
             );
         }
-        self::$stats[$projectId]['project'] = $project;
+
+        $this->stats[$projectId]['project'] = $project;
         foreach ($payload['metrics'] ?? [] as $metric) {
-            if (!isset(self::$stats[$projectId]['keys'][$metric['key']])) {
-                self::$stats[$projectId]['keys'][$metric['key']] = $metric['value'];
+                 $this->keys++;
+            if (!isset($this->stats[$projectId]['keys'][$metric['key']])) {
+                $this->stats[$projectId]['keys'][$metric['key']] = $metric['value'];
                 continue;
             }
-            self::$stats[$projectId]['keys'][$metric['key']] += $metric['value'];
+
+            $this->stats[$projectId]['keys'][$metric['key']] += $metric['value'];
+        }
+
+        // if keys crossed threshold or X time passed since the last send and there are some keys in the array ($this->stats)
+        if (
+            $this->keys >= self::KEYS_THRESHOLD ||
+            (time() - $this->lastTriggeredTime > $aggregationInterval  && $this->keys > 0)
+        ) {
+            console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');
+
+            $queueForUsageDump
+                ->setStats($this->stats)
+                ->trigger();
+
+            $this->stats = [];
+            $this->keys = 0;
+            $this->lastTriggeredTime = time();
         }
     }
 
-
      /**
      * On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files.
      * When we remove a parent document we need to deduct his children aggregation from the project scope.

src/Appwrite/Platform/Workers/UsageDump.php

@@ -0,0 +1,113 @@
+<?php
+
+namespace Appwrite\Platform\Workers;
+
+use Appwrite\Extend\Exception;
+use Utopia\App;
+use Utopia\Database\Document;
+use Utopia\Database\Exception\Duplicate;
+use Utopia\Platform\Action;
+use Utopia\CLI\Console;
+use Utopia\Database\DateTime;
+use Utopia\Queue\Message;
+
+class UsageDump extends Action
+{
+    protected array $stats = [];
+    protected array $periods = [
+        '1h' => 'Y-m-d H:00',
+        '1d' => 'Y-m-d 00:00',
+        'inf' => '0000-00-00 00:00'
+    ];
+
+    public static function getName(): string
+    {
+        return 'usage-dump';
+    }
+
+    /**
+     * @throws \Exception
+     */
+    public function __construct()
+    {
+
+        $this
+            ->inject('message')
+            ->inject('getProjectDB')
+            ->callback(function (Message $message, callable $getProjectDB) {
+                $this->action($message, $getProjectDB);
+            })
+        ;
+    }
+
+    /**
+     * @param Message $message
+     * @param callable $getProjectDB
+     * @return void
+     * @throws Exception
+     * @throws \Utopia\Database\Exception
+     */
+    public function action(Message $message, callable $getProjectDB): void
+    {
+
+        $payload = $message->getPayload() ?? [];
+        if (empty($payload)) {
+            throw new Exception('Missing payload');
+        }
+
+        //Todo rename both usage workers @shimonewman
+        foreach ($payload['stats'] ?? [] as $stats) {
+            $project = new Document($stats['project'] ?? []);
+            $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
+
+            if ($numberOfKeys === 0) {
+                continue;
+            }
+
+            console::log('[' . DateTime::now() . '] ProjectId [' . $project->getInternalId() . '] Database [' . $project['database'] . '] ' . $numberOfKeys . ' keys');
+
+            try {
+                $dbForProject = $getProjectDB($project);
+                foreach ($stats['keys'] ?? [] as $key => $value) {
+                    if ($value == 0) {
+                        continue;
+                    }
+
+                    foreach ($this->periods as $period => $format) {
+                        $time = 'inf' === $period ? null : date($format, time());
+                        $id = \md5("{$time}_{$period}_{$key}");
+
+                        try {
+                            $dbForProject->createDocument('stats_v2', new Document([
+                                '$id' => $id,
+                                'period' => $period,
+                                'time' => $time,
+                                'metric' => $key,
+                                'value' => $value,
+                                'region' => App::getEnv('_APP_REGION', 'default'),
+                            ]));
+                        } catch (Duplicate $th) {
+                            if ($value < 0) {
+                                $dbForProject->decreaseDocumentAttribute(
+                                    'stats_v2',
+                                    $id,
+                                    'value',
+                                    abs($value)
+                                );
+                            } else {
+                                $dbForProject->increaseDocumentAttribute(
+                                    'stats_v2',
+                                    $id,
+                                    'value',
+                                    $value
+                                );
+                            }
+                        }
+                    }
+                }
+            } catch (\Exception $e) {
+                console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage());
+            }
+        }
+    }
+}

src/Appwrite/Platform/Workers/UsageHook.php

@@ -1,103 +0,0 @@
-<?php
-
-namespace Appwrite\Platform\Workers;
-
-use Utopia\App;
-use Utopia\Database\Document;
-use Utopia\Database\Exception\Duplicate;
-use Utopia\Platform\Action;
-use Utopia\CLI\Console;
-use Swoole\Timer;
-use Utopia\Database\DateTime;
-
-class UsageHook extends Usage
-{
-    public static function getName(): string
-    {
-        return 'usageHook';
-    }
-
-    public function __construct()
-    {
-
-        $this
-            ->setType(Action::TYPE_WORKER_START)
-            ->inject('register')
-            ->inject('getProjectDB')
-            ->callback(function ($register, callable $getProjectDB) {
-                $this->action($register, $getProjectDB);
-            })
-        ;
-    }
-
-    /**
-     * @param $register
-     * @param $getProjectDB
-     * @return void
-     */
-    public function action($register, $getProjectDB): void
-    {
-
-        $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '60000');
-        Timer::tick($interval, function () use ($register, $getProjectDB) {
-
-            $offset = count(self::$stats);
-            $projects = array_slice(self::$stats, 0, $offset, true);
-            array_splice(self::$stats, 0, $offset);
-            foreach ($projects as $data) {
-                $numberOfKeys = !empty($data['keys']) ? count($data['keys']) : 0;
-                $projectInternalId = $data['project']->getInternalId();
-                $database = $data['project']['database'] ?? '';
-
-                console::warning('Ticker started ' . DateTime::now());
-
-                if ($numberOfKeys === 0) {
-                    continue;
-                }
-
-                try {
-                    $dbForProject = $getProjectDB($data['project']);
-                    foreach ($data['keys'] ?? [] as $key => $value) {
-                        if ($value == 0) {
-                            continue;
-                        }
-
-                        foreach ($this->periods as $period => $format) {
-                            $time = 'inf' === $period ? null : date($format, time());
-                            $id = \md5("{$time}_{$period}_{$key}");
-
-                            try {
-                                $dbForProject->createDocument('stats_v2', new Document([
-                                    '$id' => $id,
-                                    'period' => $period,
-                                    'time' => $time,
-                                    'metric' => $key,
-                                    'value' => $value,
-                                    'region' => App::getEnv('_APP_REGION', 'default'),
-                                ]));
-                            } catch (Duplicate $th) {
-                                if ($value < 0) {
-                                    $dbForProject->decreaseDocumentAttribute(
-                                        'stats_v2',
-                                        $id,
-                                        'value',
-                                        abs($value)
-                                    );
-                                } else {
-                                    $dbForProject->increaseDocumentAttribute(
-                                        'stats_v2',
-                                        $id,
-                                        'value',
-                                        $value
-                                    );
-                                }
-                            }
-                        }
-                    }
-                } catch (\Exception $e) {
-                    console::error(DateTime::now() . ' ' . $projectInternalId . ' ' . $e->getMessage());
-                }
-            }
-        });
-    }
-}