handling map size

src/Appwrite/Platform/Workers/UsageDump.php

@@ -15,10 +15,16 @@ use Utopia\System\System;
 const METRIC_COLLECTION_LEVEL_STORAGE = 4;
 const METRIC_DATABASE_LEVEL_STORAGE = 3;
 const METRIC_PROJECT_LEVEL_STORAGE = 2;
+const MAP_ARRAY_SIZE_MAX = 400000;
+
 
 class UsageDump extends Action
 {
     protected array $stats = [];
+
+    protected array $map = [];
+    protected array $timeMap = [];
+
     protected array $periods = [
         '1h' => 'Y-m-d H:00',
         '1d' => 'Y-m-d 00:00',
@@ -57,6 +63,11 @@ class UsageDump extends Action
             throw new Exception('Missing payload');
         }
 
+        if (count($this->map) >= self::MAP_ARRAY_SIZE_MAX) {
+            $this->handleMapSize();
+            //$this->map = [];
+        }
+
         // TODO: rename both usage workers @shimonewman
         foreach ($payload['stats'] ?? [] as $stats) {
             $project = new Document($stats['project'] ?? []);
@@ -88,16 +99,7 @@ class UsageDump extends Action
                         $time = 'inf' === $period ? null : date($format, time());
                         $id = \md5("{$time}_{$period}_{$key}");
 
-                        try {
-                            $dbForProject->createDocument('stats', new Document([
-                                '$id' => $id,
-                                'period' => $period,
-                                'time' => $time,
-                                'metric' => $key,
-                                'value' => $value,
-                                'region' => System::getEnv('_APP_REGION', 'default'),
-                            ]));
-                        } catch (Duplicate $th) {
+                        if(array_key_exists($id, $this->map)) {
                             if ($value < 0) {
                                 $dbForProject->decreaseDocumentAttribute(
                                     'stats',
@@ -113,6 +115,38 @@ class UsageDump extends Action
                                     $value
                                 );
                             }
+                        } else {
+
+                            $this->map[$id] = null;
+                            $this->timeMap[time()] = $id;
+
+                            try {
+                                $dbForProject->createDocument('stats', new Document([
+                                    '$id' => $id,
+                                    'period' => $period,
+                                    'time' => $time,
+                                    'metric' => $key,
+                                    'value' => $value,
+                                    'region' => System::getEnv('_APP_REGION', 'default'),
+                                ]));
+
+                            } catch (Duplicate $th) {
+                                if ($value < 0) {
+                                    $dbForProject->decreaseDocumentAttribute(
+                                        'stats',
+                                        $id,
+                                        'value',
+                                        abs($value)
+                                    );
+                                } else {
+                                    $dbForProject->increaseDocumentAttribute(
+                                        'stats',
+                                        $id,
+                                        'value',
+                                        $value
+                                    );
+                                }
+                            }
                         }
                     }
                 }
@@ -122,6 +156,24 @@ class UsageDump extends Action
         }
     }
 
+    private function handleMapSize(): void
+    {
+
+        $now = time();
+        $thirtyDaysAgo = $now - (60 * 60 * 24 * 30); //30 days
+        $filteredKeys = [];
+        foreach ($this->timeMap as $key => $value) {
+            if ($key >= $thirtyDaysAgo && $key < $now) {
+                $filteredKeys[$key] = $value;
+            }
+        }
+
+        foreach ($filteredKeys as $key => $value) {
+                unset($this->map[$value]);
+                unset($this->timeMap[$key]);
+        }
+    }
+
     private function handleDatabaseStorage(string $key, Database $dbForProject): void
     {
         $data = explode('.', $key);