diff --git a/src/Appwrite/Platform/Workers/UsageDump.php b/src/Appwrite/Platform/Workers/UsageDump.php index 834e84d9b7..3f1aba1662 100644 --- a/src/Appwrite/Platform/Workers/UsageDump.php +++ b/src/Appwrite/Platform/Workers/UsageDump.php @@ -25,8 +25,6 @@ class UsageDump extends Action 'inf' => '0000-00-00 00:00' ]; - private static array $seenMetrics = []; - public static function getName(): string { return 'usage-dump'; @@ -37,13 +35,15 @@ class UsageDump extends Action */ public function __construct() { - $this - ->inject('message') + $this->inject('message') ->inject('getProjectDB') ->callback([$this, 'action']); } /** + * For each stat key, for each period, if the key is a database storage key then we delegate to handleDatabaseStorage; + * otherwise we simply add the stat. + * * @param Message $message * @param callable(Document): Database $getProjectDB * @return void @@ -59,10 +59,8 @@ class UsageDump extends Action try { foreach ($payload['stats'] ?? [] as $stats) { - static::$seenMetrics = []; - $project = new Document($stats['project'] ?? []); - $numberOfKeys = !empty($stats['keys']) ? \count($stats['keys']) : 0; + $numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0; $receivedAt = $stats['receivedAt'] ?? 'NONE'; if ($numberOfKeys === 0) { continue; @@ -71,8 +69,7 @@ class UsageDump extends Action $dbForProject = $getProjectDB($project); $projectDocuments = []; - //Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys . ' Started'); - $start = \microtime(true); + $start = microtime(true); foreach ($stats['keys'] ?? [] as $key => $value) { if ($value == 0) { @@ -80,21 +77,22 @@ class UsageDump extends Action } foreach ($this->periods as $period => $format) { - $time = 'inf' === $period ? null : \date($format, \time()); - $id = \md5("{$time}_{$period}_{$key}"); + $time = ($period === 'inf') ? null : date($format, time()); + $id = md5("{$time}_{$period}_{$key}"); - if (\str_contains($key, METRIC_DATABASES_STORAGE)) { + if (str_contains($key, METRIC_DATABASES_STORAGE)) { static::handleDatabaseStorage( $projectDocuments, $id, $key, $time, $period, - $dbForProject, + $dbForProject ); continue; } + // For non-database storage keys, simply add/update the stat. static::addStatsDocument( $projectDocuments, $period, @@ -105,53 +103,56 @@ class UsageDump extends Action } } - //\var_dump($projectDocuments); - $dbForProject->createOrUpdateDocumentsWithIncrease( collection: 'stats', attribute: 'value', - documents: \array_values($projectDocuments) + documents: array_values($projectDocuments) ); - $end = \microtime(true); - //Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys. ' Time: '.($end - $start).'s'); + $end = microtime(true); + // (Optional) Log processing time if desired. } } catch (\Exception $e) { Console::error('[' . DateTime::now() . '] Error processing stats: ' . $e->getMessage()); } } - private static function getUniqueKey(string $key, string $period, ?string $time): string - { - return "{$key}.{$period}.{$time}"; - } - + /** + * Handle storage metrics. + * + * For a given storage metric key (which might be of the form "20.20.databases.storage"), + * we need to update three levels: collection-level, database-level, and project-level. + * For each derived metric, we re-read the previous value (from the in-memory $projectDocuments or from the DB) + * and compute the diff independently. + * + * @param array &$projectDocuments The in-memory accumulator of stats documents. + * @param string $baseId The base id computed from the original key. + * @param string $key The original key (e.g. "20.20.databases.storage"). + * @param string|null $time The formatted time (or null for "inf"). + * @param string $period The period (e.g. "1h", "1d", "inf"). + * @param Database $dbForProject The database connection for this project. + * + * @return void + * @throws \Exception + */ private static function handleDatabaseStorage( array &$projectDocuments, - string $id, + string $baseId, string $key, ?string $time, string $period, - Database $dbForProject, - ): void { - $data = \explode('.', $key); + Database $dbForProject + ): void + { + $start = microtime(true); + $data = explode('.', $key); $value = 0; - try { - if (isset($projectDocuments[$id])) { - $previousValue = $projectDocuments[$id]['value']; - Console::log("[PREVIOUS VALUE] Found in projectDocuments: {$previousValue} for {$id}"); - } else { - $previousValue = $dbForProject->getDocument('stats', $id)->getAttribute('value', 0); - Console::log("[PREVIOUS VALUE] Fetched from DB: {$previousValue} for {$id}"); - } - } catch (\Exception) { - $previousValue = 0; - Console::log("[PREVIOUS VALUE] Defaulted to 0 for {$id}"); - } - - switch (\count($data)) { + // We don’t re-use a single previous value; instead, we’ll compute a unique diff for each derived metric. + switch (count($data)) { + // Collection Level: key is in the form "databaseId.collectionId.databases.storage" case METRIC_COLLECTION_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']'); $databaseInternalId = $data[0]; $collectionInternalId = $data[1]; $collectionId = "database_{$databaseInternalId}_collection_{$collectionInternalId}"; @@ -164,34 +165,40 @@ class UsageDump extends Action } } - $diff = $value - $previousValue; - - //Console::info('['.DateTime::now().'] Collection: '.$collectionId. ' Value: '.$value. ' PreviousValue: '.$previousValue. ' Diff: '.$diff); - - if ($diff <= 0) { - break; - } - - $keys = [ - $key, - \str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE), - METRIC_DATABASES_STORAGE + // For each sub-metric in this group, fetch its own previous value and compute its diff. + $derivedKeys = [ + $key, // Collection-level metric name + str_replace(['{databaseInternalId}'], [$databaseInternalId], METRIC_DATABASE_ID_STORAGE), // Database-level metric name + METRIC_DATABASES_STORAGE // Project-level metric name ]; - \var_dump('[PROCESSING COLLECTION KEYS] ' . \json_encode($keys)); + Console::log('[PROCESSING COLLECTION KEYS] ' . json_encode($derivedKeys)); - foreach ($keys as $metric) { - static::addStatsDocument( - $projectDocuments, - $period, - $time, - $metric, - $diff - ); + foreach ($derivedKeys as $metric) { + // Compute a unique ID for this sub-metric. + $unique = md5("{$time}_{$period}_{$metric}"); + try { + // Check if we already have a queued update. + $prevVal = isset($projectDocuments[$unique]) + ? $projectDocuments[$unique]['value'] + : $dbForProject->getDocument('stats', $unique)->getAttribute('value', 0); + Console::log("[SUB-PREVIOUS VALUE] For {$metric}: {$prevVal}"); + } catch (\Exception $ex) { + $prevVal = 0; + Console::log("[SUB-PREVIOUS VALUE] Defaulted to 0 for {$metric}"); + } + $subDiff = $value - $prevVal; + if ($subDiff <= 0) { + Console::log("[SKIPPED] No positive diff for {$metric} (diff: {$subDiff})"); + continue; + } + static::addStatsDocument($projectDocuments, $period, $time, $metric, $subDiff); } - break; + + // Database Level: key is something like "databaseId.databases.storage" case METRIC_DATABASE_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']'); $databaseInternalId = $data[0]; $databaseId = "database_{$databaseInternalId}"; @@ -199,154 +206,139 @@ class UsageDump extends Action try { $collections = $dbForProject->find($databaseId); } catch (\Exception $e) { - if (!$e instanceof NotFound) { - Console::error('[Error] Type: ' . get_class($e)); - Console::error('[Error] Message: ' . $e->getMessage()); - Console::error('[Error] File: ' . $e->getFile()); - Console::error('[Error] Line: ' . $e->getLine()); - Console::error('[Error] Trace: ' . $e->getTraceAsString()); - + if ($e->getMessage() !== 'Collection not found') { throw $e; } } + // Sum the sizes from all collections in the database. foreach ($collections as $collection) { $collectionId = "{$databaseId}_collection_{$collection->getInternalId()}"; - try { - $value = $dbForProject->getSizeOfCollection($collectionId); + $value += $dbForProject->getSizeOfCollection($collectionId); } catch (\Exception $e) { - if (!$e instanceof NotFound) { + if ($e->getMessage() !== 'Collection not found') { throw $e; } } } - $diff = $value - $previousValue; - - Console::info('['.DateTime::now().'] Database: '.$databaseId. ' Value: '.$value. ' PreviousValue: '.$previousValue. ' Diff: '.$diff); - - if ($diff <= 0) { - break; - } - - - $keys = [ - \str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE), + $derivedKeys = [ + str_replace(['{databaseInternalId}'], [$databaseInternalId], METRIC_DATABASE_ID_STORAGE), METRIC_DATABASES_STORAGE ]; - \var_dump('[PROCESSING DATABASE KEYS] ' . \json_encode($keys)); + Console::log('[PROCESSING DATABASE KEYS] ' . json_encode($derivedKeys)); - foreach ($keys as $metric) { - static::addStatsDocument( - $projectDocuments, - $period, - $time, - $metric, - $diff - ); + foreach ($derivedKeys as $metric) { + $unique = md5("{$time}_{$period}_{$metric}"); + try { + $prevVal = isset($projectDocuments[$unique]) + ? $projectDocuments[$unique]['value'] + : $dbForProject->getDocument('stats', $unique)->getAttribute('value', 0); + Console::log("[SUB-PREVIOUS VALUE] For {$metric}: {$prevVal}"); + } catch (\Exception $ex) { + $prevVal = 0; + Console::log("[SUB-PREVIOUS VALUE] Defaulted to 0 for {$metric}"); + } + $subDiff = $value - $prevVal; + if ($subDiff <= 0) { + Console::log("[SKIPPED] No positive diff for {$metric} (diff: {$subDiff})"); + continue; + } + static::addStatsDocument($projectDocuments, $period, $time, $metric, $subDiff); } - break; + + // Project Level: key might be "databases.storage" case METRIC_PROJECT_LEVEL_STORAGE: + Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']'); $databases = []; try { $databases = $dbForProject->find('database'); } catch (\Exception $e) { - if (!$e instanceof NotFound) { - Console::error('[Error] Type: ' . get_class($e)); - Console::error('[Error] Message: ' . $e->getMessage()); - Console::error('[Error] File: ' . $e->getFile()); - Console::error('[Error] Line: ' . $e->getLine()); - Console::error('[Error] Trace: ' . $e->getTraceAsString()); - + if ($e->getMessage() !== 'Collection not found') { throw $e; } } foreach ($databases as $database) { $databaseId = "database_{$database->getInternalId()}"; - $collections = []; try { $collections = $dbForProject->find($databaseId); } catch (\Exception $e) { - if (!$e instanceof NotFound) { - Console::error('[Error] Type: ' . get_class($e)); - Console::error('[Error] Message: ' . $e->getMessage()); - Console::error('[Error] File: ' . $e->getFile()); - Console::error('[Error] Line: ' . $e->getLine()); - Console::error('[Error] Trace: ' . $e->getTraceAsString()); - + if ($e->getMessage() !== 'Collection not found') { throw $e; } } - foreach ($collections as $collection) { $collectionId = "{$databaseId}_collection_{$collection->getInternalId()}"; - try { - $value = $dbForProject->getSizeOfCollection($collectionId); + $value += $dbForProject->getSizeOfCollection($collectionId); } catch (\Exception $e) { - if (!$e instanceof NotFound) { + if ($e->getMessage() !== 'Collection not found') { throw $e; } } } } - $diff = $value - $previousValue; - - $project = $dbForProject->getSharedTables() - ? $dbForProject->getTenant() - : $dbForProject->getNamespace(); - - //Console::info('['.DateTime::now().'] Project: '. $project . ' Value: '.$value. ' PreviousValue: '.$previousValue. ' Diff: '.$diff); - - if ($diff <= 0) { - break; + $derivedKeys = [METRIC_DATABASES_STORAGE]; + Console::log('[PROCESSING PROJECT KEYS] ' . json_encode($derivedKeys)); + foreach ($derivedKeys as $metric) { + $unique = md5("{$time}_{$period}_{$metric}"); + try { + $prevVal = isset($projectDocuments[$unique]) + ? $projectDocuments[$unique]['value'] + : $dbForProject->getDocument('stats', $unique)->getAttribute('value', 0); + Console::log("[SUB-PREVIOUS VALUE] For {$metric}: {$prevVal}"); + } catch (\Exception $ex) { + $prevVal = 0; + Console::log("[SUB-PREVIOUS VALUE] Defaulted to 0 for {$metric}"); + } + $subDiff = $value - $prevVal; + if ($subDiff <= 0) { + Console::log("[SKIPPED] No positive diff for {$metric} (diff: {$subDiff})"); + continue; + } + static::addStatsDocument($projectDocuments, $period, $time, $metric, $subDiff); } - - $keys = [ - METRIC_DATABASES_STORAGE - ]; - - \var_dump('[PROCESSING PROJECT KEYS] ' . \json_encode($keys)); - - foreach ($keys as $metric) { - static::addStatsDocument( - $projectDocuments, - $period, - $time, - $metric, - $diff - ); - } - break; } + $end = microtime(true); + Console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds'); } + /** + * Adds or increments a document in the projectDocuments array. + * + * @param array &$projectDocuments + * @param string $period + * @param string|null $time + * @param string $key + * @param int $diff + * @return void + */ private static function addStatsDocument( array &$projectDocuments, string $period, ?string $time, string $key, int $diff - ): void { - $id = \md5("{$time}_{$period}_{$key}"); + ): void + { + $id = md5("{$time}_{$period}_{$key}"); if (isset($projectDocuments[$id])) { - Console::log("[DUPLICATE DETECTED] Metric: {$id} (Incrementing by {$diff})"); + Console::log("[DUPLICATE DETECTED] Metric: {$key} (Incrementing by {$diff})"); Console::log("Previous Value: " . $projectDocuments[$id]['value']); - \debug_print_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS); - + // Increment the queued value $projectDocuments[$id]['value'] += $diff; return; } - Console::log("[ADDING] New metric: {$id} (Value: {$diff})"); + Console::log("[ADDING] New metric: {$key} (Value: {$diff})"); $projectDocuments[$id] = new Document([ '$id' => $id, @@ -357,4 +349,4 @@ class UsageDump extends Action 'region' => System::getEnv('_APP_REGION', 'default'), ]); } -} +} \ No newline at end of file