mirror of
https://github.com/appwrite/appwrite.git
synced 2026-05-26 13:51:13 +00:00
Fixes for subkeys
This commit is contained in:
Jake Barnby
@@ -25,8 +25,6 @@ class UsageDump extends Action
|
||||
'inf' => '0000-00-00 00:00'
|
||||
];
|
||||
|
||||
private static array $seenMetrics = [];
|
||||
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'usage-dump';
|
||||
@@ -37,13 +35,15 @@ class UsageDump extends Action
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->inject('message')
|
||||
$this->inject('message')
|
||||
->inject('getProjectDB')
|
||||
->callback([$this, 'action']);
|
||||
}
|
||||
|
||||
/**
|
||||
* For each stat key, for each period, if the key is a database storage key then we delegate to handleDatabaseStorage;
|
||||
* otherwise we simply add the stat.
|
||||
*
|
||||
* @param Message $message
|
||||
* @param callable(Document): Database $getProjectDB
|
||||
* @return void
|
||||
@@ -59,10 +59,8 @@ class UsageDump extends Action
|
||||
|
||||
try {
|
||||
foreach ($payload['stats'] ?? [] as $stats) {
|
||||
static::$seenMetrics = [];
|
||||
|
||||
$project = new Document($stats['project'] ?? []);
|
||||
$numberOfKeys = !empty($stats['keys']) ? \count($stats['keys']) : 0;
|
||||
$numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
|
||||
$receivedAt = $stats['receivedAt'] ?? 'NONE';
|
||||
if ($numberOfKeys === 0) {
|
||||
continue;
|
||||
@@ -71,8 +69,7 @@ class UsageDump extends Action
|
||||
$dbForProject = $getProjectDB($project);
|
||||
$projectDocuments = [];
|
||||
|
||||
//Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys . ' Started');
|
||||
$start = \microtime(true);
|
||||
$start = microtime(true);
|
||||
|
||||
foreach ($stats['keys'] ?? [] as $key => $value) {
|
||||
if ($value == 0) {
|
||||
@@ -80,21 +77,22 @@ class UsageDump extends Action
|
||||
}
|
||||
|
||||
foreach ($this->periods as $period => $format) {
|
||||
$time = 'inf' === $period ? null : \date($format, \time());
|
||||
$id = \md5("{$time}_{$period}_{$key}");
|
||||
$time = ($period === 'inf') ? null : date($format, time());
|
||||
$id = md5("{$time}_{$period}_{$key}");
|
||||
|
||||
if (\str_contains($key, METRIC_DATABASES_STORAGE)) {
|
||||
if (str_contains($key, METRIC_DATABASES_STORAGE)) {
|
||||
static::handleDatabaseStorage(
|
||||
$projectDocuments,
|
||||
$id,
|
||||
$key,
|
||||
$time,
|
||||
$period,
|
||||
$dbForProject,
|
||||
$dbForProject
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// For non-database storage keys, simply add/update the stat.
|
||||
static::addStatsDocument(
|
||||
$projectDocuments,
|
||||
$period,
|
||||
@@ -105,53 +103,56 @@ class UsageDump extends Action
|
||||
}
|
||||
}
|
||||
|
||||
//\var_dump($projectDocuments);
|
||||
|
||||
$dbForProject->createOrUpdateDocumentsWithIncrease(
|
||||
collection: 'stats',
|
||||
attribute: 'value',
|
||||
documents: \array_values($projectDocuments)
|
||||
documents: array_values($projectDocuments)
|
||||
);
|
||||
|
||||
$end = \microtime(true);
|
||||
//Console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys. ' Time: '.($end - $start).'s');
|
||||
$end = microtime(true);
|
||||
// (Optional) Log processing time if desired.
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
Console::error('[' . DateTime::now() . '] Error processing stats: ' . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private static function getUniqueKey(string $key, string $period, ?string $time): string
|
||||
{
|
||||
return "{$key}.{$period}.{$time}";
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle storage metrics.
|
||||
*
|
||||
* For a given storage metric key (which might be of the form "20.20.databases.storage"),
|
||||
* we need to update three levels: collection-level, database-level, and project-level.
|
||||
* For each derived metric, we re-read the previous value (from the in-memory $projectDocuments or from the DB)
|
||||
* and compute the diff independently.
|
||||
*
|
||||
* @param array &$projectDocuments The in-memory accumulator of stats documents.
|
||||
* @param string $baseId The base id computed from the original key.
|
||||
* @param string $key The original key (e.g. "20.20.databases.storage").
|
||||
* @param string|null $time The formatted time (or null for "inf").
|
||||
* @param string $period The period (e.g. "1h", "1d", "inf").
|
||||
* @param Database $dbForProject The database connection for this project.
|
||||
*
|
||||
* @return void
|
||||
* @throws \Exception
|
||||
*/
|
||||
private static function handleDatabaseStorage(
|
||||
array &$projectDocuments,
|
||||
string $id,
|
||||
string $baseId,
|
||||
string $key,
|
||||
?string $time,
|
||||
string $period,
|
||||
Database $dbForProject,
|
||||
): void {
|
||||
$data = \explode('.', $key);
|
||||
Database $dbForProject
|
||||
): void
|
||||
{
|
||||
$start = microtime(true);
|
||||
$data = explode('.', $key);
|
||||
$value = 0;
|
||||
|
||||
try {
|
||||
if (isset($projectDocuments[$id])) {
|
||||
$previousValue = $projectDocuments[$id]['value'];
|
||||
Console::log("[PREVIOUS VALUE] Found in projectDocuments: {$previousValue} for {$id}");
|
||||
} else {
|
||||
$previousValue = $dbForProject->getDocument('stats', $id)->getAttribute('value', 0);
|
||||
Console::log("[PREVIOUS VALUE] Fetched from DB: {$previousValue} for {$id}");
|
||||
}
|
||||
} catch (\Exception) {
|
||||
$previousValue = 0;
|
||||
Console::log("[PREVIOUS VALUE] Defaulted to 0 for {$id}");
|
||||
}
|
||||
|
||||
switch (\count($data)) {
|
||||
// We don’t re-use a single previous value; instead, we’ll compute a unique diff for each derived metric.
|
||||
switch (count($data)) {
|
||||
// Collection Level: key is in the form "databaseId.collectionId.databases.storage"
|
||||
case METRIC_COLLECTION_LEVEL_STORAGE:
|
||||
Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']');
|
||||
$databaseInternalId = $data[0];
|
||||
$collectionInternalId = $data[1];
|
||||
$collectionId = "database_{$databaseInternalId}_collection_{$collectionInternalId}";
|
||||
@@ -164,34 +165,40 @@ class UsageDump extends Action
|
||||
}
|
||||
}
|
||||
|
||||
$diff = $value - $previousValue;
|
||||
|
||||
//Console::info('['.DateTime::now().'] Collection: '.$collectionId. ' Value: '.$value. ' PreviousValue: '.$previousValue. ' Diff: '.$diff);
|
||||
|
||||
if ($diff <= 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
$keys = [
|
||||
$key,
|
||||
\str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE),
|
||||
METRIC_DATABASES_STORAGE
|
||||
// For each sub-metric in this group, fetch its own previous value and compute its diff.
|
||||
$derivedKeys = [
|
||||
$key, // Collection-level metric name
|
||||
str_replace(['{databaseInternalId}'], [$databaseInternalId], METRIC_DATABASE_ID_STORAGE), // Database-level metric name
|
||||
METRIC_DATABASES_STORAGE // Project-level metric name
|
||||
];
|
||||
|
||||
\var_dump('[PROCESSING COLLECTION KEYS] ' . \json_encode($keys));
|
||||
Console::log('[PROCESSING COLLECTION KEYS] ' . json_encode($derivedKeys));
|
||||
|
||||
foreach ($keys as $metric) {
|
||||
static::addStatsDocument(
|
||||
$projectDocuments,
|
||||
$period,
|
||||
$time,
|
||||
$metric,
|
||||
$diff
|
||||
);
|
||||
foreach ($derivedKeys as $metric) {
|
||||
// Compute a unique ID for this sub-metric.
|
||||
$unique = md5("{$time}_{$period}_{$metric}");
|
||||
try {
|
||||
// Check if we already have a queued update.
|
||||
$prevVal = isset($projectDocuments[$unique])
|
||||
? $projectDocuments[$unique]['value']
|
||||
: $dbForProject->getDocument('stats', $unique)->getAttribute('value', 0);
|
||||
Console::log("[SUB-PREVIOUS VALUE] For {$metric}: {$prevVal}");
|
||||
} catch (\Exception $ex) {
|
||||
$prevVal = 0;
|
||||
Console::log("[SUB-PREVIOUS VALUE] Defaulted to 0 for {$metric}");
|
||||
}
|
||||
$subDiff = $value - $prevVal;
|
||||
if ($subDiff <= 0) {
|
||||
Console::log("[SKIPPED] No positive diff for {$metric} (diff: {$subDiff})");
|
||||
continue;
|
||||
}
|
||||
static::addStatsDocument($projectDocuments, $period, $time, $metric, $subDiff);
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
// Database Level: key is something like "databaseId.databases.storage"
|
||||
case METRIC_DATABASE_LEVEL_STORAGE:
|
||||
Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']');
|
||||
$databaseInternalId = $data[0];
|
||||
$databaseId = "database_{$databaseInternalId}";
|
||||
|
||||
@@ -199,154 +206,139 @@ class UsageDump extends Action
|
||||
try {
|
||||
$collections = $dbForProject->find($databaseId);
|
||||
} catch (\Exception $e) {
|
||||
if (!$e instanceof NotFound) {
|
||||
Console::error('[Error] Type: ' . get_class($e));
|
||||
Console::error('[Error] Message: ' . $e->getMessage());
|
||||
Console::error('[Error] File: ' . $e->getFile());
|
||||
Console::error('[Error] Line: ' . $e->getLine());
|
||||
Console::error('[Error] Trace: ' . $e->getTraceAsString());
|
||||
|
||||
if ($e->getMessage() !== 'Collection not found') {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
// Sum the sizes from all collections in the database.
|
||||
foreach ($collections as $collection) {
|
||||
$collectionId = "{$databaseId}_collection_{$collection->getInternalId()}";
|
||||
|
||||
try {
|
||||
$value = $dbForProject->getSizeOfCollection($collectionId);
|
||||
$value += $dbForProject->getSizeOfCollection($collectionId);
|
||||
} catch (\Exception $e) {
|
||||
if (!$e instanceof NotFound) {
|
||||
if ($e->getMessage() !== 'Collection not found') {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$diff = $value - $previousValue;
|
||||
|
||||
Console::info('['.DateTime::now().'] Database: '.$databaseId. ' Value: '.$value. ' PreviousValue: '.$previousValue. ' Diff: '.$diff);
|
||||
|
||||
if ($diff <= 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
$keys = [
|
||||
\str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE),
|
||||
$derivedKeys = [
|
||||
str_replace(['{databaseInternalId}'], [$databaseInternalId], METRIC_DATABASE_ID_STORAGE),
|
||||
METRIC_DATABASES_STORAGE
|
||||
];
|
||||
|
||||
\var_dump('[PROCESSING DATABASE KEYS] ' . \json_encode($keys));
|
||||
Console::log('[PROCESSING DATABASE KEYS] ' . json_encode($derivedKeys));
|
||||
|
||||
foreach ($keys as $metric) {
|
||||
static::addStatsDocument(
|
||||
$projectDocuments,
|
||||
$period,
|
||||
$time,
|
||||
$metric,
|
||||
$diff
|
||||
);
|
||||
foreach ($derivedKeys as $metric) {
|
||||
$unique = md5("{$time}_{$period}_{$metric}");
|
||||
try {
|
||||
$prevVal = isset($projectDocuments[$unique])
|
||||
? $projectDocuments[$unique]['value']
|
||||
: $dbForProject->getDocument('stats', $unique)->getAttribute('value', 0);
|
||||
Console::log("[SUB-PREVIOUS VALUE] For {$metric}: {$prevVal}");
|
||||
} catch (\Exception $ex) {
|
||||
$prevVal = 0;
|
||||
Console::log("[SUB-PREVIOUS VALUE] Defaulted to 0 for {$metric}");
|
||||
}
|
||||
$subDiff = $value - $prevVal;
|
||||
if ($subDiff <= 0) {
|
||||
Console::log("[SKIPPED] No positive diff for {$metric} (diff: {$subDiff})");
|
||||
continue;
|
||||
}
|
||||
static::addStatsDocument($projectDocuments, $period, $time, $metric, $subDiff);
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
// Project Level: key might be "databases.storage"
|
||||
case METRIC_PROJECT_LEVEL_STORAGE:
|
||||
Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']');
|
||||
$databases = [];
|
||||
try {
|
||||
$databases = $dbForProject->find('database');
|
||||
} catch (\Exception $e) {
|
||||
if (!$e instanceof NotFound) {
|
||||
Console::error('[Error] Type: ' . get_class($e));
|
||||
Console::error('[Error] Message: ' . $e->getMessage());
|
||||
Console::error('[Error] File: ' . $e->getFile());
|
||||
Console::error('[Error] Line: ' . $e->getLine());
|
||||
Console::error('[Error] Trace: ' . $e->getTraceAsString());
|
||||
|
||||
if ($e->getMessage() !== 'Collection not found') {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($databases as $database) {
|
||||
$databaseId = "database_{$database->getInternalId()}";
|
||||
|
||||
$collections = [];
|
||||
try {
|
||||
$collections = $dbForProject->find($databaseId);
|
||||
} catch (\Exception $e) {
|
||||
if (!$e instanceof NotFound) {
|
||||
Console::error('[Error] Type: ' . get_class($e));
|
||||
Console::error('[Error] Message: ' . $e->getMessage());
|
||||
Console::error('[Error] File: ' . $e->getFile());
|
||||
Console::error('[Error] Line: ' . $e->getLine());
|
||||
Console::error('[Error] Trace: ' . $e->getTraceAsString());
|
||||
|
||||
if ($e->getMessage() !== 'Collection not found') {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($collections as $collection) {
|
||||
$collectionId = "{$databaseId}_collection_{$collection->getInternalId()}";
|
||||
|
||||
try {
|
||||
$value = $dbForProject->getSizeOfCollection($collectionId);
|
||||
$value += $dbForProject->getSizeOfCollection($collectionId);
|
||||
} catch (\Exception $e) {
|
||||
if (!$e instanceof NotFound) {
|
||||
if ($e->getMessage() !== 'Collection not found') {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$diff = $value - $previousValue;
|
||||
|
||||
$project = $dbForProject->getSharedTables()
|
||||
? $dbForProject->getTenant()
|
||||
: $dbForProject->getNamespace();
|
||||
|
||||
//Console::info('['.DateTime::now().'] Project: '. $project . ' Value: '.$value. ' PreviousValue: '.$previousValue. ' Diff: '.$diff);
|
||||
|
||||
if ($diff <= 0) {
|
||||
break;
|
||||
$derivedKeys = [METRIC_DATABASES_STORAGE];
|
||||
Console::log('[PROCESSING PROJECT KEYS] ' . json_encode($derivedKeys));
|
||||
foreach ($derivedKeys as $metric) {
|
||||
$unique = md5("{$time}_{$period}_{$metric}");
|
||||
try {
|
||||
$prevVal = isset($projectDocuments[$unique])
|
||||
? $projectDocuments[$unique]['value']
|
||||
: $dbForProject->getDocument('stats', $unique)->getAttribute('value', 0);
|
||||
Console::log("[SUB-PREVIOUS VALUE] For {$metric}: {$prevVal}");
|
||||
} catch (\Exception $ex) {
|
||||
$prevVal = 0;
|
||||
Console::log("[SUB-PREVIOUS VALUE] Defaulted to 0 for {$metric}");
|
||||
}
|
||||
$subDiff = $value - $prevVal;
|
||||
if ($subDiff <= 0) {
|
||||
Console::log("[SKIPPED] No positive diff for {$metric} (diff: {$subDiff})");
|
||||
continue;
|
||||
}
|
||||
static::addStatsDocument($projectDocuments, $period, $time, $metric, $subDiff);
|
||||
}
|
||||
|
||||
$keys = [
|
||||
METRIC_DATABASES_STORAGE
|
||||
];
|
||||
|
||||
\var_dump('[PROCESSING PROJECT KEYS] ' . \json_encode($keys));
|
||||
|
||||
foreach ($keys as $metric) {
|
||||
static::addStatsDocument(
|
||||
$projectDocuments,
|
||||
$period,
|
||||
$time,
|
||||
$metric,
|
||||
$diff
|
||||
);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
$end = microtime(true);
|
||||
Console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds');
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds or increments a document in the projectDocuments array.
|
||||
*
|
||||
* @param array &$projectDocuments
|
||||
* @param string $period
|
||||
* @param string|null $time
|
||||
* @param string $key
|
||||
* @param int $diff
|
||||
* @return void
|
||||
*/
|
||||
private static function addStatsDocument(
|
||||
array &$projectDocuments,
|
||||
string $period,
|
||||
?string $time,
|
||||
string $key,
|
||||
int $diff
|
||||
): void {
|
||||
$id = \md5("{$time}_{$period}_{$key}");
|
||||
): void
|
||||
{
|
||||
$id = md5("{$time}_{$period}_{$key}");
|
||||
|
||||
if (isset($projectDocuments[$id])) {
|
||||
Console::log("[DUPLICATE DETECTED] Metric: {$id} (Incrementing by {$diff})");
|
||||
Console::log("[DUPLICATE DETECTED] Metric: {$key} (Incrementing by {$diff})");
|
||||
Console::log("Previous Value: " . $projectDocuments[$id]['value']);
|
||||
\debug_print_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS);
|
||||
|
||||
// Increment the queued value
|
||||
$projectDocuments[$id]['value'] += $diff;
|
||||
return;
|
||||
}
|
||||
|
||||
Console::log("[ADDING] New metric: {$id} (Value: {$diff})");
|
||||
Console::log("[ADDING] New metric: {$key} (Value: {$diff})");
|
||||
|
||||
$projectDocuments[$id] = new Document([
|
||||
'$id' => $id,
|
||||
@@ -357,4 +349,4 @@ class UsageDump extends Action
|
||||
'region' => System::getEnv('_APP_REGION', 'default'),
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user