usage containers updated

app/tasks/usage.php

@@ -4,13 +4,16 @@ global $cli, $register;
 
 use Appwrite\Stats\Usage;
 use Appwrite\Stats\UsageDB;
+use Appwrite\Usage\Calculators\Aggregator;
+use Appwrite\Usage\Calculators\Database;
+use Appwrite\Usage\Calculators\TimeSeries;
 use InfluxDB\Database as InfluxDatabase;
 use Utopia\App;
 use Utopia\Cache\Adapter\Redis as RedisCache;
 use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
 use Utopia\Database\Adapter\MariaDB;
-use Utopia\Database\Database;
+use Utopia\Database\Database as UtopiaDatabase;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Registry\Registry;
 use Utopia\Logger\Log;
@@ -19,7 +22,7 @@ use Utopia\Validator\WhiteList;
 Authorization::disable();
 Authorization::setDefaultStatus(false);
 
-function getDatabase(Registry &$register, string $namespace): Database
+function getDatabase(Registry &$register, string $namespace): UtopiaDatabase
 {
     $attempts = 0;
 
@@ -31,7 +34,7 @@ function getDatabase(Registry &$register, string $namespace): Database
             $redis = $register->get('cache');
 
             $cache = new Cache(new RedisCache($redis));
-            $database = new Database(new MariaDB($db), $cache);
+            $database = new UtopiaDatabase(new MariaDB($db), $cache);
             $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
             $database->setNamespace($namespace);
 
@@ -39,7 +42,7 @@ function getDatabase(Registry &$register, string $namespace): Database
                 throw new Exception('Projects collection not ready');
             }
             break; // leave loop if successful
-        } catch (\Exception$e) {
+        } catch (\Exception $e) {
             Console::warning("Database not ready. Retrying connection ({$attempts})...");
             if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) {
                 throw new \Exception('Failed to connect to database: ' . $e->getMessage());
@@ -66,7 +69,7 @@ function getInfluxDB(Registry &$register): InfluxDatabase
             if (in_array('telegraf', $client->listDatabases())) {
                 break; // leave the do-while if successful
             }
-        } catch (\Throwable$th) {
+        } catch (\Throwable $th) {
             Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
             if ($attempts >= $max) {
                 throw new \Exception('InfluxDB database not ready yet');
@@ -119,48 +122,44 @@ $cli
         Console::title('Usage Aggregation V1');
         Console::success(APP_NAME . ' usage aggregation process v1 has started');
 
-        $interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default)
-
         $database = getDatabase($register, '_console');
         $influxDB = getInfluxDB($register);
 
-        $usage = new Usage($database, $influxDB, $logError);
-        $usageDB = new UsageDB($database, $logError);
+        if ($type == 'timeseries') {
 
-        $iterations = 0;
-        Console::loop(function () use ($interval, $usage, $usageDB, &$iterations) {
-            $now = date('d-m-Y H:i:s', time());
-            Console::info("[{$now}] Aggregating usage data every {$interval} seconds");
+            $interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default)
 
-            $loopStart = microtime(true);
 
-            /**
-             * Aggregate InfluxDB every 30 seconds
-             */
-            $usage->collect();
+            $usage = new TimeSeries($database, $influxDB, $logError);
+
+            Console::loop(function () use ($interval, $usage) {
+                $now = date('d-m-Y H:i:s', time());
+                Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds");
+                $loopStart = microtime(true);
+
+                $usage->collect();
 
-            if ($iterations % 30 != 0 && App::getEnv('_APP_ENV', 'production') == 'production') { // return if 30 iterations has not passed
-                $iterations++;
                 $loopTook = microtime(true) - $loopStart;
                 $now = date('d-m-Y H:i:s', time());
                 Console::info("[{$now}] Aggregation took {$loopTook} seconds");
-                return;
-            }
+            }, $interval);
+        }
 
-            $iterations = 0; // Reset iterations to prevent overflow when running for long time
-            /**
-             * Aggregate MariaDB every 15 minutes
-             * Some of the queries here might contain full-table scans.
-             */
-            $now = date('d-m-Y H:i:s', time());
-            Console::info("[{$now}] Aggregating database counters.");
+        if ($type == 'database') {
+            $interval = (int) App::getEnv('_APP_USAGE_DATABASE_INTERVAL', '900'); // 15 minutes (by default)
+            $usage = new Database($database, $logError);
+            $aggregrator = new Aggregator($database, $logError);
 
-            $usageDB->collect();
+            Console::loop(function () use ($interval, $usage, $aggregrator) {
+                $now = date('d-m-Y H:i:s', time());
+                Console::info("[{$now}] Aggregating database usage every {$interval} seconds.");
+                $loopStart = microtime(true);
+                $usage->collect();
+                $aggregrator->collect();
+                $loopTook = microtime(true) - $loopStart;
+                $now = date('d-m-Y H:i:s', time());
 
-            $iterations++;
-            $loopTook = microtime(true) - $loopStart;
-            $now = date('d-m-Y H:i:s', time());
-
-            Console::info("[{$now}] Aggregation took {$loopTook} seconds");
-        }, $interval);
+                Console::info("[{$now}] Aggregation took {$loopTook} seconds");
+            }, $interval);
+        }
     });

app/views/install/compose.phtml

@@ -548,10 +548,44 @@ services:
       - _APP_MAINTENANCE_RETENTION_ABUSE
       - _APP_MAINTENANCE_RETENTION_AUDIT
 
-  appwrite-usage:
+  appwrite-usage-timeseries:
     image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
-    entrypoint: usage
-    container_name: appwrite-usage
+    entrypoint:
+      - usage
+      - --type=timeseries
+    container_name: appwrite-usage-timeseries
+    <<: *x-logging
+    restart: unless-stopped
+    networks:
+      - appwrite
+    depends_on:
+      - influxdb
+      - mariadb
+    environment:
+      - _APP_ENV
+      - _APP_OPENSSL_KEY_V1
+      - _APP_DB_HOST
+      - _APP_DB_PORT
+      - _APP_DB_SCHEMA
+      - _APP_DB_USER
+      - _APP_DB_PASS
+      - _APP_INFLUXDB_HOST
+      - _APP_INFLUXDB_PORT
+      - _APP_USAGE_TIMESERIES_INTERVAL
+      - _APP_USAGE_DATABASE_INTERVAL
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_LOGGING_PROVIDER
+      - _APP_LOGGING_CONFIG
+
+  appwrite-usage-database:
+    image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
+    entrypoint:
+      - usage
+      - --type=database
+    container_name: appwrite-usage-database
     <<: *x-logging
     restart: unless-stopped
     networks:

docker-compose.yml

@@ -582,10 +582,50 @@ services:
       - _APP_MAINTENANCE_RETENTION_ABUSE
       - _APP_MAINTENANCE_RETENTION_AUDIT
 
-  appwrite-usage:
-    entrypoint: usage
+  appwrite-usage-timeseries:
+    entrypoint:
+      - usage
+      - --type=timeseries
     <<: *x-logging
-    container_name: appwrite-usage
+    container_name: appwrite-usage-timeseries
+    build:
+      context: .
+      args:
+        - DEBUG=false
+    networks:
+      - appwrite
+    volumes:
+      - ./app:/usr/src/code/app
+      - ./src:/usr/src/code/src
+      - ./dev:/usr/local/dev
+    depends_on:
+      - influxdb
+      - mariadb
+    environment:
+      - _APP_ENV
+      - _APP_OPENSSL_KEY_V1
+      - _APP_DB_HOST
+      - _APP_DB_PORT
+      - _APP_DB_SCHEMA
+      - _APP_DB_USER
+      - _APP_DB_PASS
+      - _APP_INFLUXDB_HOST
+      - _APP_INFLUXDB_PORT
+      - _APP_USAGE_TIMESERIES_INTERVAL
+      - _APP_USAGE_DATABASE_INTERVAL
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_LOGGING_PROVIDER
+      - _APP_LOGGING_CONFIG
+  
+  appwrite-usage-database:
+    entrypoint:
+      - usage
+      - --type=database
+    <<: *x-logging
+    container_name: appwrite-usage-database
     build:
       context: .
       args: