Fixes for subkeys

composer.lock

@@ -1237,16 +1237,16 @@
         },
         {
             "name": "open-telemetry/api",
-            "version": "1.2.0",
+            "version": "1.2.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/opentelemetry-php/api.git",
-                "reference": "351a30baa79699de3de3a814c8ccc7b52ccdfb1d"
+                "reference": "74b1a03263be8c5acb578f41da054b4bac3af4a0"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/351a30baa79699de3de3a814c8ccc7b52ccdfb1d",
-                "reference": "351a30baa79699de3de3a814c8ccc7b52ccdfb1d",
+                "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/74b1a03263be8c5acb578f41da054b4bac3af4a0",
+                "reference": "74b1a03263be8c5acb578f41da054b4bac3af4a0",
                 "shasum": ""
             },
             "require": {
@@ -1303,7 +1303,7 @@
                 "issues": "https://github.com/open-telemetry/opentelemetry-php/issues",
                 "source": "https://github.com/open-telemetry/opentelemetry-php"
             },
-            "time": "2025-01-08T23:50:34+00:00"
+            "time": "2025-01-20T23:35:16+00:00"
         },
         {
             "name": "open-telemetry/context",
@@ -1493,16 +1493,16 @@
         },
         {
             "name": "open-telemetry/sdk",
-            "version": "1.2.0",
+            "version": "1.2.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/opentelemetry-php/sdk.git",
-                "reference": "9a1c3b866239dbff291e5cc555bb7793eab08127"
+                "reference": "96aeaee5b7cb8c0bc4af7ff4717b429f2d9f67e1"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/9a1c3b866239dbff291e5cc555bb7793eab08127",
-                "reference": "9a1c3b866239dbff291e5cc555bb7793eab08127",
+                "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/96aeaee5b7cb8c0bc4af7ff4717b429f2d9f67e1",
+                "reference": "96aeaee5b7cb8c0bc4af7ff4717b429f2d9f67e1",
                 "shasum": ""
             },
             "require": {
@@ -1579,7 +1579,7 @@
                 "issues": "https://github.com/open-telemetry/opentelemetry-php/issues",
                 "source": "https://github.com/open-telemetry/opentelemetry-php"
             },
-            "time": "2025-01-08T23:50:34+00:00"
+            "time": "2025-01-09T23:17:14+00:00"
         },
         {
             "name": "open-telemetry/sem-conv",
@@ -2453,16 +2453,16 @@
         },
         {
             "name": "symfony/http-client",
-            "version": "v7.2.2",
+            "version": "v7.2.3",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/http-client.git",
-                "reference": "339ba21476eb184290361542f732ad12c97591ec"
+                "reference": "7ce6078c79a4a7afff931c413d2959d3bffbfb8d"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/http-client/zipball/339ba21476eb184290361542f732ad12c97591ec",
-                "reference": "339ba21476eb184290361542f732ad12c97591ec",
+                "url": "https://api.github.com/repos/symfony/http-client/zipball/7ce6078c79a4a7afff931c413d2959d3bffbfb8d",
+                "reference": "7ce6078c79a4a7afff931c413d2959d3bffbfb8d",
                 "shasum": ""
             },
             "require": {
@@ -2528,7 +2528,7 @@
                 "http"
             ],
             "support": {
-                "source": "https://github.com/symfony/http-client/tree/v7.2.2"
+                "source": "https://github.com/symfony/http-client/tree/v7.2.3"
             },
             "funding": [
                 {
@@ -2544,7 +2544,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2024-12-30T18:35:15+00:00"
+            "time": "2025-01-28T15:51:35+00:00"
         },
         {
             "name": "symfony/http-client-contracts",
@@ -3476,16 +3476,16 @@
         },
         {
             "name": "utopia-php/database",
-            "version": "0.57.2",
+            "version": "0.57.3",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/database.git",
-                "reference": "bd6f080dd9f4210349a6a862fa6da65a4d9d6339"
+                "reference": "d8ad52793e59ac9a553ed38cc905c6e133b809fc"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/database/zipball/bd6f080dd9f4210349a6a862fa6da65a4d9d6339",
-                "reference": "bd6f080dd9f4210349a6a862fa6da65a4d9d6339",
+                "url": "https://api.github.com/repos/utopia-php/database/zipball/d8ad52793e59ac9a553ed38cc905c6e133b809fc",
+                "reference": "d8ad52793e59ac9a553ed38cc905c6e133b809fc",
                 "shasum": ""
             },
             "require": {
@@ -3526,9 +3526,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/database/issues",
-                "source": "https://github.com/utopia-php/database/tree/0.57.2"
+                "source": "https://github.com/utopia-php/database/tree/0.57.3"
             },
-            "time": "2025-01-23T05:19:02+00:00"
+            "time": "2025-02-04T03:33:56+00:00"
         },
         {
             "name": "utopia-php/domains",
@@ -5601,70 +5601,18 @@
             },
             "time": "2023-10-30T13:38:26+00:00"
         },
-        {
-            "name": "phpbench/dom",
-            "version": "0.3.3",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/phpbench/dom.git",
-                "reference": "786a96db538d0def931f5b19225233ec42ec7a72"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/phpbench/dom/zipball/786a96db538d0def931f5b19225233ec42ec7a72",
-                "reference": "786a96db538d0def931f5b19225233ec42ec7a72",
-                "shasum": ""
-            },
-            "require": {
-                "ext-dom": "*",
-                "php": "^7.3||^8.0"
-            },
-            "require-dev": {
-                "friendsofphp/php-cs-fixer": "^3.14",
-                "phpstan/phpstan": "^1.10",
-                "phpunit/phpunit": "^8.0||^9.0"
-            },
-            "type": "library",
-            "extra": {
-                "branch-alias": {
-                    "dev-master": "1.0-dev"
-                }
-            },
-            "autoload": {
-                "psr-4": {
-                    "PhpBench\\Dom\\": "lib/"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Daniel Leech",
-                    "email": "daniel@dantleech.com"
-                }
-            ],
-            "description": "DOM wrapper to simplify working with the PHP DOM implementation",
-            "support": {
-                "issues": "https://github.com/phpbench/dom/issues",
-                "source": "https://github.com/phpbench/dom/tree/0.3.3"
-            },
-            "abandoned": true,
-            "time": "2023-03-06T23:46:57+00:00"
-        },
         {
             "name": "phpbench/phpbench",
-            "version": "1.3.1",
+            "version": "1.4.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/phpbench/phpbench.git",
-                "reference": "a3e1ef08d9d7736d43a7fbd444893d6a073c0ca0"
+                "reference": "4248817222514421cba466bfa7adc7d8932345d4"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/phpbench/phpbench/zipball/a3e1ef08d9d7736d43a7fbd444893d6a073c0ca0",
-                "reference": "a3e1ef08d9d7736d43a7fbd444893d6a073c0ca0",
+                "url": "https://api.github.com/repos/phpbench/phpbench/zipball/4248817222514421cba466bfa7adc7d8932345d4",
+                "reference": "4248817222514421cba466bfa7adc7d8932345d4",
                 "shasum": ""
             },
             "require": {
@@ -5677,7 +5625,6 @@
                 "ext-tokenizer": "*",
                 "php": "^8.1",
                 "phpbench/container": "^2.2",
-                "phpbench/dom": "~0.3.3",
                 "psr/log": "^1.1 || ^2.0 || ^3.0",
                 "seld/jsonlint": "^1.1",
                 "symfony/console": "^6.1 || ^7.0",
@@ -5696,8 +5643,8 @@
                 "phpstan/extension-installer": "^1.1",
                 "phpstan/phpstan": "^1.0",
                 "phpstan/phpstan-phpunit": "^1.0",
-                "phpunit/phpunit": "^10.4",
-                "rector/rector": "^0.18.11 || ^1.0.0",
+                "phpunit/phpunit": "^10.4 || ^11.0",
+                "rector/rector": "^1.2",
                 "symfony/error-handler": "^6.1 || ^7.0",
                 "symfony/var-dumper": "^6.1 || ^7.0"
             },
@@ -5742,7 +5689,7 @@
             ],
             "support": {
                 "issues": "https://github.com/phpbench/phpbench/issues",
-                "source": "https://github.com/phpbench/phpbench/tree/1.3.1"
+                "source": "https://github.com/phpbench/phpbench/tree/1.4.0"
             },
             "funding": [
                 {
@@ -5750,7 +5697,7 @@
                     "type": "github"
                 }
             ],
-            "time": "2024-06-30T11:04:37+00:00"
+            "time": "2025-01-26T19:54:45+00:00"
         },
         {
             "name": "phpdocumentor/reflection-common",

src/Appwrite/Platform/Workers/UsageDump.php

@@ -35,18 +35,20 @@ class UsageDump extends Action
      */
     public function __construct()
     {
-        $this
-            ->inject('message')
+        $this->inject('message')
             ->inject('getProjectDB')
             ->callback([$this, 'action']);
     }
 
     /**
+     * For each stat key, for each period, if the key is a database storage key then we delegate to handleDatabaseStorage;
+     * otherwise we simply add the stat.
+     *
      * @param Message $message
-     * @param callable $getProjectDB
+     * @param callable(Document): Database $getProjectDB
      * @return void
      * @throws Exception
-     * @throws \Utopia\Database\Exception
+     * @throws \Throwable
      */
     public function action(Message $message, callable $getProjectDB): void
     {
@@ -58,7 +60,7 @@ class UsageDump extends Action
         try {
             foreach ($payload['stats'] ?? [] as $stats) {
                 $project = new Document($stats['project'] ?? []);
-                $numberOfKeys = !empty($stats['keys']) ? \count($stats['keys']) : 0;
+                $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
                 $receivedAt = $stats['receivedAt'] ?? 'NONE';
                 if ($numberOfKeys === 0) {
                     continue;
@@ -66,11 +68,8 @@ class UsageDump extends Action
 
                 $dbForProject = $getProjectDB($project);
                 $projectDocuments = [];
-                $databaseCache = [];
-                $collectionSizeCache = [];
 
-                Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys . ' Started');
-                $start = \microtime(true);
+                $start = microtime(true);
 
                 foreach ($stats['keys'] ?? [] as $key => $value) {
                     if ($value == 0) {
@@ -78,218 +77,270 @@ class UsageDump extends Action
                     }
 
                     foreach ($this->periods as $period => $format) {
-                        $time = 'inf' === $period ? null : \date($format, \time());
-                        $id = \md5("{$time}_{$period}_{$key}");
+                        $time = ($period === 'inf') ? null : date($format, time());
+                        $id = md5("{$time}_{$period}_{$key}");
 
-                        if (\str_contains($key, METRIC_DATABASES_STORAGE)) {
-                            $this->handleDatabaseStorage(
+                        if (str_contains($key, METRIC_DATABASES_STORAGE)) {
+                            static::handleDatabaseStorage(
+                                $projectDocuments,
                                 $id,
                                 $key,
                                 $time,
                                 $period,
-                                $dbForProject,
-                                $projectDocuments,
-                                $databaseCache,
-                                $collectionSizeCache
+                                $dbForProject
                             );
                             continue;
                         }
 
-                        $projectDocuments[] = new Document([
-                            '$id' => $id,
-                            'period' => $period,
-                            'time' => $time,
-                            'metric' => $key,
-                            'value' => $value,
-                            'region' => System::getEnv('_APP_REGION', 'default'),
-                        ]);
+                        // For non-database storage keys, simply add/update the stat.
+                        static::addStatsDocument(
+                            $projectDocuments,
+                            $period,
+                            $time,
+                            $key,
+                            $value
+                        );
                     }
                 }
 
                 $dbForProject->createOrUpdateDocumentsWithIncrease(
                     collection: 'stats',
                     attribute: 'value',
-                    documents: $projectDocuments
+                    documents: array_values($projectDocuments)
                 );
 
-                $end = \microtime(true);
-                Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys. ' Time: '.($end - $start).'s');
+                $end = microtime(true);
+                // (Optional) Log processing time if desired.
             }
         } catch (\Exception $e) {
             Console::error('[' . DateTime::now() . '] Error processing stats: ' . $e->getMessage());
         }
     }
 
-    private function handleDatabaseStorage(
-        string $id,
+    /**
+     * Handle storage metrics.
+     *
+     * For a given storage metric key (which might be of the form "20.20.databases.storage"),
+     * we need to update three levels: collection-level, database-level, and project-level.
+     * For each derived metric, we re-read the previous value (from the in-memory $projectDocuments or from the DB)
+     * and compute the diff independently.
+     *
+     * @param array &$projectDocuments The in-memory accumulator of stats documents.
+     * @param string $baseId The base id computed from the original key.
+     * @param string $key The original key (e.g. "20.20.databases.storage").
+     * @param string|null $time The formatted time (or null for "inf").
+     * @param string $period The period (e.g. "1h", "1d", "inf").
+     * @param Database $dbForProject The database connection for this project.
+     *
+     * @return void
+     * @throws \Exception
+     */
+    private static function handleDatabaseStorage(
+        array &$projectDocuments,
+        string $baseId,
         string $key,
         ?string $time,
         string $period,
-        Database $dbForProject,
-        array &$projectDocuments,
-        array &$databaseCache,
-        array &$collectionSizeCache,
-    ): void {
-        $data = \explode('.', $key);
+        Database $dbForProject
+    ): void
+    {
+        $start = microtime(true);
+        $data = explode('.', $key);
         $value = 0;
-        $previousValue = 0;
 
-        try {
-            $previousValue = $dbForProject
-                ->getDocument('stats', $id)
-                ->getAttribute('value', 0);
-        } catch (\Exception) {
-            // No previous value
-        }
-
-        switch (\count($data)) {
+        // We don’t re-use a single previous value; instead, we’ll compute a unique diff for each derived metric.
+        switch (count($data)) {
+            // Collection Level: key is in the form "databaseId.collectionId.databases.storage"
             case METRIC_COLLECTION_LEVEL_STORAGE:
+                Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']');
                 $databaseInternalId = $data[0];
                 $collectionInternalId = $data[1];
                 $collectionId = "database_{$databaseInternalId}_collection_{$collectionInternalId}";
 
-                if (!isset($collectionSizeCache[$collectionId])) {
-                    try {
-                        $collectionSizeCache[$collectionId] = $dbForProject->getSizeOfCollection($collectionId);
-                    } catch (\Exception $e) {
-                        if (!$e instanceof NotFound) {
-                            throw $e;
-                        }
-                        $collectionSizeCache[$collectionId] = 0;
+                try {
+                    $value = $dbForProject->getSizeOfCollection($collectionId);
+                } catch (\Exception $e) {
+                    if (!$e instanceof NotFound) {
+                        throw $e;
                     }
                 }
 
-                $value = $collectionSizeCache[$collectionId];
-
-                $diff = $value - $previousValue;
-                if ($diff === 0) {
-                    break;
-                }
-
-                $keys = [
-                    $key,
-                    \str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE),
-                    METRIC_DATABASES_STORAGE
+                // For each sub-metric in this group, fetch its own previous value and compute its diff.
+                $derivedKeys = [
+                    $key, // Collection-level metric name
+                    str_replace(['{databaseInternalId}'], [$databaseInternalId], METRIC_DATABASE_ID_STORAGE), // Database-level metric name
+                    METRIC_DATABASES_STORAGE // Project-level metric name
                 ];
 
-                foreach ($keys as $metric) {
-                    $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff);
+                Console::log('[PROCESSING COLLECTION KEYS] ' . json_encode($derivedKeys));
+
+                foreach ($derivedKeys as $metric) {
+                    // Compute a unique ID for this sub-metric.
+                    $unique = md5("{$time}_{$period}_{$metric}");
+                    try {
+                        // Check if we already have a queued update.
+                        $prevVal = isset($projectDocuments[$unique])
+                            ? $projectDocuments[$unique]['value']
+                            : $dbForProject->getDocument('stats', $unique)->getAttribute('value', 0);
+                        Console::log("[SUB-PREVIOUS VALUE] For {$metric}: {$prevVal}");
+                    } catch (\Exception $ex) {
+                        $prevVal = 0;
+                        Console::log("[SUB-PREVIOUS VALUE] Defaulted to 0 for {$metric}");
+                    }
+                    $subDiff = $value - $prevVal;
+                    if ($subDiff <= 0) {
+                        Console::log("[SKIPPED] No positive diff for {$metric} (diff: {$subDiff})");
+                        continue;
+                    }
+                    static::addStatsDocument($projectDocuments, $period, $time, $metric, $subDiff);
                 }
                 break;
 
+            // Database Level: key is something like "databaseId.databases.storage"
             case METRIC_DATABASE_LEVEL_STORAGE:
+                Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']');
                 $databaseInternalId = $data[0];
                 $databaseId = "database_{$databaseInternalId}";
 
-                if (!isset($databaseCache[$databaseId])) {
+                $collections = [];
+                try {
+                    $collections = $dbForProject->find($databaseId);
+                } catch (\Exception $e) {
+                    if ($e->getMessage() !== 'Collection not found') {
+                        throw $e;
+                    }
+                }
+
+                // Sum the sizes from all collections in the database.
+                foreach ($collections as $collection) {
+                    $collectionId = "{$databaseId}_collection_{$collection->getInternalId()}";
                     try {
-                        $databaseCache[$databaseId] = $dbForProject->find($databaseId);
+                        $value += $dbForProject->getSizeOfCollection($collectionId);
                     } catch (\Exception $e) {
-                        if (!$e instanceof NotFound) {
+                        if ($e->getMessage() !== 'Collection not found') {
                             throw $e;
                         }
-                        $databaseCache[$databaseId] = [];
                     }
                 }
 
-                foreach ($databaseCache[$databaseId] as $collection) {
-                    $collectionId = "{$databaseId}_collection_{$collection->getInternalId()}";
-
-                    if (!isset($collectionSizeCache[$collectionId])) {
-                        try {
-                            $collectionSizeCache[$collectionId] = $dbForProject->getSizeOfCollection($collectionId);
-                        } catch (\Exception $e) {
-                            if (!$e instanceof NotFound) {
-                                throw $e;
-                            }
-                            $collectionSizeCache[$collectionId] = 0;
-                        }
-                    }
-                    $value += $collectionSizeCache[$collectionId];
-                }
-
-                $diff = $value - $previousValue;
-                if ($diff === 0) {
-                    break;
-                }
-
-                $keys = [
-                    \str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE),
+                $derivedKeys = [
+                    str_replace(['{databaseInternalId}'], [$databaseInternalId], METRIC_DATABASE_ID_STORAGE),
                     METRIC_DATABASES_STORAGE
                 ];
 
-                foreach ($keys as $metric) {
-                    $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff);
+                Console::log('[PROCESSING DATABASE KEYS] ' . json_encode($derivedKeys));
+
+                foreach ($derivedKeys as $metric) {
+                    $unique = md5("{$time}_{$period}_{$metric}");
+                    try {
+                        $prevVal = isset($projectDocuments[$unique])
+                            ? $projectDocuments[$unique]['value']
+                            : $dbForProject->getDocument('stats', $unique)->getAttribute('value', 0);
+                        Console::log("[SUB-PREVIOUS VALUE] For {$metric}: {$prevVal}");
+                    } catch (\Exception $ex) {
+                        $prevVal = 0;
+                        Console::log("[SUB-PREVIOUS VALUE] Defaulted to 0 for {$metric}");
+                    }
+                    $subDiff = $value - $prevVal;
+                    if ($subDiff <= 0) {
+                        Console::log("[SKIPPED] No positive diff for {$metric} (diff: {$subDiff})");
+                        continue;
+                    }
+                    static::addStatsDocument($projectDocuments, $period, $time, $metric, $subDiff);
                 }
                 break;
 
+            // Project Level: key might be "databases.storage"
             case METRIC_PROJECT_LEVEL_STORAGE:
-                if (!isset($databaseCache['*'])) {
+                Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']');
+                $databases = [];
+                try {
+                    $databases = $dbForProject->find('database');
+                } catch (\Exception $e) {
+                    if ($e->getMessage() !== 'Collection not found') {
+                        throw $e;
+                    }
+                }
+
+                foreach ($databases as $database) {
+                    $databaseId = "database_{$database->getInternalId()}";
+                    $collections = [];
                     try {
-                        $databaseCache['*'] = $dbForProject->find('databases');
+                        $collections = $dbForProject->find($databaseId);
                     } catch (\Exception $e) {
-                        if (!$e instanceof NotFound) {
+                        if ($e->getMessage() !== 'Collection not found') {
                             throw $e;
                         }
-                        $databaseCache['*'] = [];
                     }
-                }
-
-                foreach ($databaseCache['*'] as $database) {
-                    $databaseId = "database_{$database->getInternalId()}";
-                    if (!isset($databaseCache[$databaseId])) {
+                    foreach ($collections as $collection) {
+                        $collectionId = "{$databaseId}_collection_{$collection->getInternalId()}";
                         try {
-                            $databaseCache[$databaseId] = $dbForProject->find($databaseId);
+                            $value += $dbForProject->getSizeOfCollection($collectionId);
                         } catch (\Exception $e) {
-                            if (!$e instanceof NotFound) {
+                            if ($e->getMessage() !== 'Collection not found') {
                                 throw $e;
                             }
-                            $databaseCache[$databaseId] = [];
                         }
                     }
+                }
 
-                    foreach ($databaseCache[$databaseId] as $collection) {
-                        $collectionId = "{$databaseId}_collection_{$collection->getInternalId()}";
-
-                        if (!isset($collectionSizeCache[$collectionId])) {
-                            try {
-                                $collectionSizeCache[$collectionId] = $dbForProject->getSizeOfCollection($collectionId);
-                            } catch (\Exception $e) {
-                                if (!$e instanceof NotFound) {
-                                    throw $e;
-                                }
-                                $collectionSizeCache[$collectionId] = 0;
-                            }
-                        }
-                        $value += $collectionSizeCache[$collectionId];
+                $derivedKeys = [METRIC_DATABASES_STORAGE];
+                Console::log('[PROCESSING PROJECT KEYS] ' . json_encode($derivedKeys));
+                foreach ($derivedKeys as $metric) {
+                    $unique = md5("{$time}_{$period}_{$metric}");
+                    try {
+                        $prevVal = isset($projectDocuments[$unique])
+                            ? $projectDocuments[$unique]['value']
+                            : $dbForProject->getDocument('stats', $unique)->getAttribute('value', 0);
+                        Console::log("[SUB-PREVIOUS VALUE] For {$metric}: {$prevVal}");
+                    } catch (\Exception $ex) {
+                        $prevVal = 0;
+                        Console::log("[SUB-PREVIOUS VALUE] Defaulted to 0 for {$metric}");
                     }
+                    $subDiff = $value - $prevVal;
+                    if ($subDiff <= 0) {
+                        Console::log("[SKIPPED] No positive diff for {$metric} (diff: {$subDiff})");
+                        continue;
+                    }
+                    static::addStatsDocument($projectDocuments, $period, $time, $metric, $subDiff);
                 }
-
-                $diff = $value - $previousValue;
-                if ($diff === 0) {
-                    break;
-                }
-
-                $keys = [
-                    METRIC_DATABASES_STORAGE
-                ];
-
-                foreach ($keys as $metric) {
-                    $projectDocuments[] = $this->createStatsDocument($id, $period, $time, $metric, $diff);
-                }
-
                 break;
         }
+        $end = microtime(true);
+        Console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds');
     }
 
-    private function createStatsDocument(
-        string $id,
+    /**
+     * Adds or increments a document in the projectDocuments array.
+     *
+     * @param array  &$projectDocuments
+     * @param string $period
+     * @param string|null $time
+     * @param string $key
+     * @param int $diff
+     * @return void
+     */
+    private static function addStatsDocument(
+        array &$projectDocuments,
         string $period,
         ?string $time,
         string $key,
-        int $diff,
-    ): Document {
-        return new Document([
+        int $diff
+    ): void
+    {
+        $id = md5("{$time}_{$period}_{$key}");
+
+        if (isset($projectDocuments[$id])) {
+            Console::log("[DUPLICATE DETECTED] Metric: {$key} (Incrementing by {$diff})");
+            Console::log("Previous Value: " . $projectDocuments[$id]['value']);
+            // Increment the queued value
+            $projectDocuments[$id]['value'] += $diff;
+            return;
+        }
+
+        Console::log("[ADDING] New metric: {$key} (Value: {$diff})");
+
+        $projectDocuments[$id] = new Document([
             '$id' => $id,
             'period' => $period,
             'time' => $time,
@@ -298,4 +349,4 @@ class UsageDump extends Action
             'region' => System::getEnv('_APP_REGION', 'default'),
         ]);
     }
-}
+}

tests/e2e/General/UsageTest.php

@@ -65,6 +65,12 @@ class UsageTest extends Scope
         return $date->format(self::$formatTz);
     }
 
+    public static function getExpectedStorage(): array {
+        return [
+            'databases' => 1_261_568,
+        ];
+    }
+
     public function testPrepareUsersStats(): array
     {
         $usersTotal = 0;
@@ -518,6 +524,7 @@ class UsageTest extends Scope
             'databasesTotal' => $databasesTotal,
             'collectionsTotal' => $collectionsTotal,
             'documentsTotal' => $documentsTotal,
+            'storageTotal' => self::getExpectedStorage(),
         ]);
     }
 
@@ -531,8 +538,9 @@ class UsageTest extends Scope
         $databasesTotal = $data['databasesTotal'];
         $collectionsTotal = $data['collectionsTotal'];
         $documentsTotal = $data['documentsTotal'];
+        $databaseStorageTotal = $data['storageTotal']['databases'];
 
-        sleep(self::WAIT);
+        \sleep(self::WAIT);
 
         $response = $this->client->call(
             Client::METHOD_GET,
@@ -552,6 +560,7 @@ class UsageTest extends Scope
         $this->validateDates($response['body']['requests']);
         $this->assertEquals($databasesTotal, $response['body']['databasesTotal']);
         $this->assertEquals($documentsTotal, $response['body']['documentsTotal']);
+        $this->assertEquals($databaseStorageTotal, $response['body']['databasesStorageTotal']);
 
         $response = $this->client->call(
             Client::METHOD_GET,
@@ -565,6 +574,7 @@ class UsageTest extends Scope
         $this->validateDates($response['body']['collections']);
         $this->assertEquals($documentsTotal, $response['body']['documents'][array_key_last($response['body']['documents'])]['value']);
         $this->validateDates($response['body']['documents']);
+        $this->assertEquals($databaseStorageTotal, $response['body']['storageTotal']);
 
         $response = $this->client->call(
             Client::METHOD_GET,
@@ -574,9 +584,9 @@ class UsageTest extends Scope
 
         $this->assertEquals($collectionsTotal, $response['body']['collections'][array_key_last($response['body']['collections'])]['value']);
         $this->validateDates($response['body']['collections']);
-
         $this->assertEquals($documentsTotal, $response['body']['documents'][array_key_last($response['body']['documents'])]['value']);
         $this->validateDates($response['body']['documents']);
+        $this->assertEquals($databaseStorageTotal, $response['body']['storageTotal']);
 
         $response = $this->client->call(
             Client::METHOD_GET,
@@ -590,67 +600,6 @@ class UsageTest extends Scope
         return $data;
     }
 
-    public function testDatabaseStoragePrepare(): array
-    {
-        $response = $this->client->call(
-            Client::METHOD_POST,
-            '/databases',
-            array_merge([
-                'content-type' => 'application/json',
-                'x-appwrite-project' => $this->getProject()['$id']
-            ], $this->getHeaders()),
-            [
-                'databaseId' => 'unique()',
-                'name' => 'dbStorageStats',
-            ]
-        );
-
-        $this->assertNotEmpty($response['body']['$id']);
-        $databaseId = $response['body']['$id'];
-
-        $response = $this->client->call(
-            Client::METHOD_POST,
-            '/databases/' . $databaseId . '/collections',
-            array_merge([
-                'content-type' => 'application/json',
-                'x-appwrite-project' => $this->getProject()['$id']
-            ], $this->getHeaders()),
-            [
-                'collectionId' => 'unique()',
-                'name' => 'collectionStorageStats',
-                'documentSecurity' => false,
-                'permissions' => [
-                    Permission::read(Role::any()),
-                    Permission::create(Role::any()),
-                    Permission::update(Role::any()),
-                    Permission::delete(Role::any()),
-                ],
-            ]
-        );
-
-        $this->assertNotEmpty($response['body']['$id']);
-        $collectionId = $response['body']['$id'];
-
-        $response = $this->client->call(
-            Client::METHOD_POST,
-            '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes' . '/string',
-            array_merge([
-                'content-type' => 'application/json',
-                'x-appwrite-project' => $this->getProject()['$id']
-            ], $this->getHeaders()),
-            [
-                'key' => 'data',
-                'size' => 100000,
-                'required' => true,
-            ]
-        );
-
-        return [
-            'databaseId' => $databaseId,
-            'collectionId' => $collectionId,
-        ];
-    }
-
     /** @depends testDatabaseStats */
     public function testPrepareFunctionsStats(array $data): array
     {