diff --git a/app/console b/app/console index fae048b917..ac4181aea4 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit fae048b91787d0b372c13caf27f14fc8b780ef60 +Subproject commit ac4181aea403d888e63cb527c700e80013c68ea8 diff --git a/src/Appwrite/Platform/Tasks/UsageCreateMetrics.php b/src/Appwrite/Platform/Tasks/UsageCreateMetrics.php index 0cc050d227..b6e6fa6042 100644 --- a/src/Appwrite/Platform/Tasks/UsageCreateMetrics.php +++ b/src/Appwrite/Platform/Tasks/UsageCreateMetrics.php @@ -9,6 +9,13 @@ use Utopia\Platform\Action; use Utopia\CLI\Console; use Utopia\Database\Validator\Authorization; +/** + * 1. docker compose exec -t appwrite usage-update-metrics + * 2. docker compose exec -t appwrite usage-create-metrics + * 3. docker compose exec -t appwrite usage-update-metrics commit=true + **. docker compose exec -t appwrite usage-update-metrics rollback=true (in case somthing went wrong) + */ + class UsageCreateMetrics extends Action { public static function getName(): string @@ -19,12 +26,11 @@ class UsageCreateMetrics extends Action public function __construct() { $this - ->desc('Usage inf calculation')->inject('dbForConsole') + ->desc('Usage inf metric calculation')->inject('dbForConsole') ->inject('getProjectDB') ->callback(fn(Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB)); } - /** * @throws \Exception */ @@ -36,181 +42,221 @@ class UsageCreateMetrics extends Action function createInfMetric($dbForProject, $key, $value): void { $id = \md5("null_inf_{$key}"); - $stats = $dbForProject->getDocument('stats', $id); if ($stats->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => 'inf', - 'time' => null, - 'metric' => '', - 'metricV2' => $key, - 'value' => (int)$value, - 'region' => 'default', + $stats = $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => 'inf', + 'time' => null, + 'metric' => '', + 'metricV2' => $key, + 'value' => (int)$value, + 'region' => 'default', ])); } else { $stats->setAttribute('value', (int)($stats['value'] + $value)); $dbForProject->updateDocument('stats', $stats->getId(), $stats); } + + Console::log($stats->getId() . ', ' . $key . ', inf,' . $stats->getAttribute('value')); } - $projects = $dbForConsole->find('projects'); - if (count($projects) === 0) { - Console::info("Could not found any projects"); - return; - } - foreach ($projects as $project) { - Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getInternalId() . ")"); - $dbForProject = $getProjectDB($project); - /** - * Network - inbound, outbound, requests - */ - $metrics = []; - $metrics[] = ['from' => 'project.$all.network.outbound', 'to' => 'network.outbound']; - $metrics[] = ['from' => 'project.$all.network.inbound', 'to' => 'network.inbound']; - $metrics[] = ['from' => 'project.$all.network.requests', 'to' => 'network.requests']; + $limit = 100; + $projectCursor = null; + while (true) { + $projectsQueries = [Query::limit($limit)]; + if ($projectCursor !== null) { + $projectsQueries[] = Query::cursorAfter($projectCursor); + } + $projects = $dbForConsole->find('projects', $projectsQueries); - foreach ($metrics as $metric) { - $stats = $dbForProject->find('stats', [ - Query::equal('metric', [$metric['from']]), - Query::equal('period', ['1d']), - Query::greaterThan('value', 0) - ]); - foreach ($stats as $stat) { - createInfMetric($dbForProject, $metric['to'], $stat['value']); - } + if (count($projects) === 0) { + Console::info("**** Could not found any projects ****"); + break; } - /** - * users - */ - $users = $dbForProject->count('users', [ - Query::equal('status', [1]), - ]); - createInfMetric($dbForProject, 'users', $users); - /** - * Sessions - */ - $sessions = $dbForProject->count('sessions', []); - createInfMetric($dbForProject, 'sessions', $sessions); + foreach ($projects as $project) { + Console::info(PHP_EOL . "Project " . $project->getAttribute('name') . " (" . $project->getInternalId() . ")"); + /** + * Delete (reset) inf metric if exists (in case of multiple executions) + */ + $dbForProject = $getProjectDB($project); + $statsQueries[] = Query::equal('period', ['inf']); + $statsCursor = null; + while (true) { + $statsQueries = [Query::limit($limit)]; + $statsQueries[] = Query::equal('period', ['inf']); + if ($statsCursor !== null) { + $statsQueries[] = Query::cursorAfter($statsCursor); + } - /** - * Buckets, Files - */ - $filesStoragePerProject = 0; - $filesCountPerProject = 0; - $bucketsCountPerProject = 0; - $buckets = $dbForProject->find('buckets', []); - foreach ($buckets as $bucket) { - $filesCount = 0; - $filesStorage = 0; - $files = $dbForProject->find('bucket_' . $bucket->getInternalId(), []); - foreach ($files as $file) { - $filesStorage += $file['sizeOriginal']; - $filesStoragePerProject += $file['sizeOriginal']; - $filesCount++; - $filesCountPerProject++; + $stats = $dbForProject->find('stats', $statsQueries); + if (count($stats) === 0) { + Console::info("**** Could not found any stats to delete ****"); + break; + } + + foreach ($stats as $stat) { + $dbForProject->deleteDocument('stats', $stat->getId()); + Console::error('deleting ' . $stat->getId() . ',' . $stat['metricV2'] . ',' . $stat['period']); + } + + $statsCursor = $stats[array_key_last($stats)]; } - createInfMetric($dbForProject, $bucket->getInternalId() . '.files.storage', $filesStoragePerProject); - createInfMetric($dbForProject, $bucket->getInternalId() . '.files', $filesCountPerProject); - $bucketsCountPerProject++; - } - createInfMetric($dbForProject, 'files.storage', $filesStoragePerProject); - createInfMetric($dbForProject, 'files', $filesCountPerProject); - createInfMetric($dbForProject, 'buckets', $bucketsCountPerProject); - /** - * Databases, collections, documents - */ - $databasesCountPerProject = 0; - $collectionsCountPerProject = 0; - $documentsCountPerProject = 0; - $databases = $dbForProject->find('databases', []); - foreach ($databases as $database) { - $databasesCountPerProject++; - $documentsCountPerDatabase = 0; - $collectionCountPerDatabase = 0; - $collections = $dbForProject->find('database_' . $database->getInternalId(), []); - foreach ($collections as $collection) { - $documentsCountPerCollection = 0; - $collectionsCountPerProject++; - $collectionCountPerDatabase++; - $documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), []); - $documentsCountPerProject += $documents; - $documentsCountPerDatabase += $documents; - $documentsCountPerCollection += $documents; - createInfMetric($dbForProject, $database->getInternalId() . '.' . $collection->getInternalId() . '.documents', $documentsCountPerCollection); - } - createInfMetric($dbForProject, $database->getInternalId() . '.documents', $documentsCountPerDatabase); - } - createInfMetric($dbForProject, 'databases', $databasesCountPerProject); - createInfMetric($dbForProject, 'collections', $collectionsCountPerProject); - createInfMetric($dbForProject, 'documents', $documentsCountPerProject); + /** + * Network - inbound, outbound, requests + */ + $metrics = []; + $metrics[] = ['from' => 'project.$all.network.outbound', 'to' => 'network.outbound']; + $metrics[] = ['from' => 'project.$all.network.inbound', 'to' => 'network.inbound']; + $metrics[] = ['from' => 'project.$all.network.requests', 'to' => 'network.requests']; - /** - * Functions, deployments, builds, executions - */ - $deploymentSizePerProject = 0; - $deploymentCountPerProject = 0; - $buildDurationPerProject = 0; - $buildCountPerProject = 0; - $executionCountPerProject = 0; - $executionDurationPerProject = 0 ; - - $functions = $dbForProject->find('functions', []); - foreach ($functions as $function) { - $deploymentSize = 0; - $deploymentCount = 0; - $buildDuration = 0; - $executionDuration = 0; - $executionCount = 0; - $buildCount = 0; - - $deployments = $dbForProject->find('deployments', [ - Query::equal('resourceId', [$function->getId()]), - ]); - foreach ($deployments as $deployment) { - $deploymentSize += $deployment['size']; - $deploymentSizePerProject += $deployment['size']; - $deploymentCount++; - $deploymentCountPerProject++; - - $build = $dbForProject->findOne('builds', [ - Query::equal('deploymentId', [$deployment->getId()]), - ]); - - if (!empty($build)) { - $buildDuration += $build['duration']; - $buildDurationPerProject += $build['duration']; - $buildCountPerProject++; - $buildCount++; + foreach ($metrics as $metric) { + $stats = $dbForProject->find('stats', [ + Query::equal('metric', [$metric['from']]), + Query::equal('period', ['1d']), + Query::greaterThan('value', 0) + ]); + foreach ($stats as $stat) { + createInfMetric($dbForProject, $metric['to'], $stat['value']); } } - createInfMetric($dbForProject, $function->getInternalId() . '.builds', $buildCount); - createInfMetric($dbForProject, $function->getInternalId() . '.builds.compute', (int)($buildDuration * 1000)); - createInfMetric($dbForProject, $function->getInternalId() . '.deployments', $deploymentCount); - createInfMetric($dbForProject, $function->getInternalId() . '.deployments.storage', $deploymentSize); - - $executions = $dbForProject->find('executions', [ - Query::equal('functionId', [$function->getId()]) + /** + * users + */ + $users = $dbForProject->count('users', [ + Query::equal('status', [1]), ]); - foreach ($executions as $execution) { - $executionDuration += $execution['duration']; - $executionDurationPerProject += $execution['duration']; - $executionCount++; - $executionCountPerProject++; - } + createInfMetric($dbForProject, 'users', $users); - createInfMetric($dbForProject, $function->getInternalId() . '.executions.compute', (int)($executionDuration * 1000)); - createInfMetric($dbForProject, $function->getInternalId() . '.executions', $executionCount); + /** + * Sessions + */ + $sessions = $dbForProject->count('sessions', []); + createInfMetric($dbForProject, 'sessions', $sessions); + + /** + * Buckets, Files + */ + $filesStoragePerProject = 0; + $filesCountPerProject = 0; + $bucketsCountPerProject = 0; + $buckets = $dbForProject->find('buckets', []); + foreach ($buckets as $bucket) { + $filesCount = 0; + $filesStorage = 0; + $files = $dbForProject->find('bucket_' . $bucket->getInternalId(), []); + foreach ($files as $file) { + $filesStorage += $file['sizeOriginal']; + $filesStoragePerProject += $file['sizeOriginal']; + $filesCount++; + $filesCountPerProject++; + } + + createInfMetric($dbForProject, $bucket->getInternalId() . '.files.storage', $filesStoragePerProject); + createInfMetric($dbForProject, $bucket->getInternalId() . '.files', $filesCountPerProject); + $bucketsCountPerProject++; + } + createInfMetric($dbForProject, 'files.storage', $filesStoragePerProject); + createInfMetric($dbForProject, 'files', $filesCountPerProject); + createInfMetric($dbForProject, 'buckets', $bucketsCountPerProject); + + /** + * Databases, collections, documents + */ + $databasesCountPerProject = 0; + $collectionsCountPerProject = 0; + $documentsCountPerProject = 0; + $databases = $dbForProject->find('databases', []); + foreach ($databases as $database) { + $databasesCountPerProject++; + $documentsCountPerDatabase = 0; + $collectionCountPerDatabase = 0; + $collections = $dbForProject->find('database_' . $database->getInternalId(), []); + foreach ($collections as $collection) { + $documentsCountPerCollection = 0; + $collectionsCountPerProject++; + $collectionCountPerDatabase++; + $documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), []); + $documentsCountPerProject += $documents; + $documentsCountPerDatabase += $documents; + $documentsCountPerCollection += $documents; + createInfMetric($dbForProject, $database->getInternalId() . '.' . $collection->getInternalId() . '.documents', $documentsCountPerCollection); + } + createInfMetric($dbForProject, $database->getInternalId() . '.documents', $documentsCountPerDatabase); + } + createInfMetric($dbForProject, 'databases', $databasesCountPerProject); + createInfMetric($dbForProject, 'collections', $collectionsCountPerProject); + createInfMetric($dbForProject, 'documents', $documentsCountPerProject); + + /** + * Functions, deployments, builds, executions + */ + $deploymentSizePerProject = 0; + $deploymentCountPerProject = 0; + $buildDurationPerProject = 0; + $buildCountPerProject = 0; + $executionCountPerProject = 0; + $executionDurationPerProject = 0 ; + + $functions = $dbForProject->find('functions', []); + foreach ($functions as $function) { + $deploymentSize = 0; + $deploymentCount = 0; + $buildDuration = 0; + $executionDuration = 0; + $executionCount = 0; + $buildCount = 0; + + $deployments = $dbForProject->find('deployments', [ + Query::equal('resourceId', [$function->getId()]), + ]); + foreach ($deployments as $deployment) { + $deploymentSize += $deployment['size']; + $deploymentSizePerProject += $deployment['size']; + $deploymentCount++; + $deploymentCountPerProject++; + + $build = $dbForProject->findOne('builds', [ + Query::equal('deploymentId', [$deployment->getId()]), + ]); + + if (!empty($build)) { + $buildDuration += $build['duration']; + $buildDurationPerProject += $build['duration']; + $buildCountPerProject++; + $buildCount++; + } + } + + createInfMetric($dbForProject, $function->getInternalId() . '.builds', $buildCount); + createInfMetric($dbForProject, $function->getInternalId() . '.builds.compute', (int)($buildDuration * 1000)); + createInfMetric($dbForProject, $function->getInternalId() . '.deployments', $deploymentCount); + createInfMetric($dbForProject, $function->getInternalId() . '.deployments.storage', $deploymentSize); + + $executions = $dbForProject->find('executions', [ + Query::equal('functionId', [$function->getId()]) + ]); + foreach ($executions as $execution) { + $executionDuration += $execution['duration']; + $executionDurationPerProject += $execution['duration']; + $executionCount++; + $executionCountPerProject++; + } + + createInfMetric($dbForProject, $function->getInternalId() . '.executions.compute', (int)($executionDuration * 1000)); + createInfMetric($dbForProject, $function->getInternalId() . '.executions', $executionCount); + } + createInfMetric($dbForProject, 'deployments', $deploymentCountPerProject); + createInfMetric($dbForProject, 'builds', $buildCountPerProject); + createInfMetric($dbForProject, 'executions', $executionCountPerProject); + createInfMetric($dbForProject, 'deployments.storage', $deploymentSizePerProject); + createInfMetric($dbForProject, 'builds.compute', (int)($buildDurationPerProject * 1000)); + createInfMetric($dbForProject, 'executions.compute', (int)($executionDurationPerProject * 1000)); } - createInfMetric($dbForProject, 'deployments', $deploymentCountPerProject); - createInfMetric($dbForProject, 'builds', $buildCountPerProject); - createInfMetric($dbForProject, 'executions', $executionCountPerProject); - createInfMetric($dbForProject, 'deployments.storage', $deploymentSizePerProject); - createInfMetric($dbForProject, 'builds.compute', (int)($buildDurationPerProject * 1000)); - createInfMetric($dbForProject, 'executions.compute', (int)($executionDurationPerProject * 1000)); + $projectCursor = $projects[array_key_last($projects)]; } } } diff --git a/src/Appwrite/Platform/Tasks/UsageUpdateMetrics.php b/src/Appwrite/Platform/Tasks/UsageUpdateMetrics.php index ab44bf89c0..917ad076d5 100644 --- a/src/Appwrite/Platform/Tasks/UsageUpdateMetrics.php +++ b/src/Appwrite/Platform/Tasks/UsageUpdateMetrics.php @@ -33,177 +33,190 @@ class UsageUpdateMetrics extends Action Authorization::disable(); Authorization::setDefaultStatus(false); + $limit = 100; + $projectCursor = null; + while (true) { + $projectsQueries = [Query::limit($limit)]; + if ($projectCursor !== null) { + $projectsQueries[] = Query::cursorAfter($projectCursor); + } + $projects = $dbForConsole->find('projects', $projectsQueries); - $projects = $dbForConsole->find('projects'); - if (count($projects) === 0) { - Console::info("Could not found any projects"); - return; - } - - foreach ($projects as $project) { - Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getInternalId() . ")"); - $dbForProject = $getProjectDB($project); - - if ($commit) { - try { - $dbForProject->renameAttribute('stats', 'metric', 'metricOld'); - $dbForProject->renameAttribute('stats', 'metricV2', 'metric'); - } catch (\Throwable $th) { - Console::Error("Error while trying to rename col: {$th->getMessage()}"); - } - - Console::log("Commit -Renaming stats column metric to metricOld, metricV2 to metric"); - continue; + if (count($projects) === 0) { + Console::info("**** Could not found any projects ****"); + break; } - if ($rollback) { - try { - $dbForProject->renameAttribute('stats', 'metric', 'metricV2'); - $dbForProject->renameAttribute('stats', 'metricOld', 'metric'); - } catch (\Throwable $th) { - Console::Error("Error while trying to rename column: {$th->getMessage()}"); - } + foreach ($projects as $project) { + Console::info(PHP_EOL . "Project " . $project->getAttribute('name') . " (" . $project->getInternalId() . ")"); + $dbForProject = $getProjectDB($project); - Console::log("Rollback - Renaming stats column metric to metricV2, metricOld to metric"); - continue; - } - - /** - * Project level - */ - $metrics[] = ['from' => 'project.$all.network.outbound', 'to' => 'network.outbound']; - $metrics[] = ['from' => 'project.$all.network.inbound', 'to' => 'network.inbound']; - $metrics[] = ['from' => 'project.$all.network.requests', 'to' => 'network.requests']; - $metrics[] = ['from' => 'executions.$all.compute.total', 'to' => 'executions']; - $metrics[] = ['from' => 'executions.$all.compute.time', 'to' => 'executions.compote']; - $metrics[] = ['from' => 'builds.$all.compute.total', 'to' => 'builds']; - $metrics[] = ['from' => 'builds.$all.compute.time', 'to' => 'builds.compute']; - $metrics[] = ['from' => 'files.$all.storage.size', 'to' => 'files.storage']; - $metrics[] = ['from' => 'files.$all.count.total', 'to' => 'files']; - $metrics[] = ['from' => 'buckets.$all.count.total', 'to' => 'buckets']; - $metrics[] = ['from' => 'collections.$all.count.total', 'to' => 'collections']; - $metrics[] = ['from' => 'databases.$all.count.total', 'to' => 'databases']; - $metrics[] = ['from' => 'documents.$all.count.total', 'to' => 'documents']; - $metrics[] = ['from' => 'users.$all.count.total ', 'to' => 'users']; - foreach ($metrics as $metric) { - $stats = $dbForProject->find('stats', [ - Query::equal('metric', [$metric['from']]), - Query::greaterThan('value', 0) - ]); - foreach ($stats as $stat) { - $stat->setAttribute('metricV2', $metric['to']); - $dbForProject->updateDocument('stats', $stat->getId(), $stat); - } - } - - /** - * Buckets, Files - */ - $buckets = $dbForProject->find('buckets', []); - foreach ($buckets as $bucket) { - $metrics = []; - - $metrics[] = [ - 'from' => str_replace('{bucketId}', $bucket->getId(), 'files.{bucketId}.count.total'), - 'to' => str_replace('{bucketInternalId}', $bucket->getInternalId(), '{bucketInternalId}.files') - ]; - - $metrics[] = [ - 'from' => str_replace('{bucketId}', $bucket->getId(), 'files.{bucketId}.storage.size'), - 'to' => str_replace('{bucketInternalId}', $bucket->getInternalId(), '{bucketInternalId}.files.storage') - ]; - - foreach ($metrics as $metric) { - $stats = $dbForProject->find('stats', [ - Query::equal('metric', [$metric['from']]), - Query::greaterThan('value', 0) - ]); - foreach ($stats as $stat) { - $stat->setAttribute('metricV2', $metric['to']); - $dbForProject->updateDocument('stats', $stat->getId(), $stat); + if ($commit) { + try { + $dbForProject->renameAttribute('stats', 'metric', 'metricOld'); + $dbForProject->renameAttribute('stats', 'metricV2', 'metric'); + Console::log("Commit -Renaming stats column metric to metricOld, metricV2 to metric"); + } catch (\Throwable $th) { + Console::Error("Error while trying to rename col: {$th->getMessage()}"); } + continue; } - } - /** - * Databases, collections, documents - */ - $databases = $dbForProject->find('databases', []); - foreach ($databases as $database) { + if ($rollback) { + try { + $dbForProject->renameAttribute('stats', 'metric', 'metricV2'); + $dbForProject->renameAttribute('stats', 'metricOld', 'metric'); + Console::log("Rollback - Renaming stats column metric to metricV2, metricOld to metric"); + } catch (\Throwable $th) { + Console::Error("Error while trying to rename column: {$th->getMessage()}"); + } + continue; + } + + /** + * Project level + */ $metrics = []; - $metrics[] = [ - 'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'), - 'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.collections')]; - $metrics[] = [ - 'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'), - 'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.documents')]; + $metrics[] = ['from' => 'project.$all.network.outbound', 'to' => 'network.outbound']; + $metrics[] = ['from' => 'project.$all.network.inbound', 'to' => 'network.inbound']; + $metrics[] = ['from' => 'project.$all.network.requests', 'to' => 'network.requests']; + $metrics[] = ['from' => 'executions.$all.compute.total', 'to' => 'executions']; + $metrics[] = ['from' => 'executions.$all.compute.time', 'to' => 'executions.compute']; + $metrics[] = ['from' => 'builds.$all.compute.total', 'to' => 'builds']; + $metrics[] = ['from' => 'builds.$all.compute.time', 'to' => 'builds.compute']; + $metrics[] = ['from' => 'files.$all.storage.size', 'to' => 'files.storage']; + $metrics[] = ['from' => 'files.$all.count.total', 'to' => 'files']; + $metrics[] = ['from' => 'buckets.$all.count.total', 'to' => 'buckets']; + $metrics[] = ['from' => 'collections.$all.count.total', 'to' => 'collections']; + $metrics[] = ['from' => 'databases.$all.count.total', 'to' => 'databases']; + $metrics[] = ['from' => 'documents.$all.count.total', 'to' => 'documents']; + $metrics[] = ['from' => 'users.$all.count.total ', 'to' => 'users']; foreach ($metrics as $metric) { $stats = $dbForProject->find('stats', [ Query::equal('metric', [$metric['from']]), Query::greaterThan('value', 0) ]); - foreach ($stats as $stat) { $stat->setAttribute('metricV2', $metric['to']); $dbForProject->updateDocument('stats', $stat->getId(), $stat); + Console::log($stat->getId() . ', ' . $metric['from'] . ', ' . $metric['to'] . ', ' . $stat['period'] . ', ' . $stat['value']); } + } - $collections = $dbForProject->find('database_' . $database->getInternalId(), []); + /** + * Buckets, Files + */ + $buckets = $dbForProject->find('buckets', []); + foreach ($buckets as $bucket) { + $metrics = []; - foreach ($collections as $collection) { - $__metric = [ - 'from' => str_replace(['{databaseId}','{collectionId}'], [$database->getId(), $collection->getId()], 'documents.{databaseId}/{collectionId}.count.total'), - 'to' => str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], '{databaseInternalId}.{collectionInternalId}.documents')]; + $metrics[] = [ + 'from' => str_replace('{bucketId}', $bucket->getId(), 'files.{bucketId}.count.total'), + 'to' => str_replace('{bucketInternalId}', $bucket->getInternalId(), '{bucketInternalId}.files') + ]; - $__stats = $dbForProject->find('stats', [ - Query::equal('metric', [$__metric['from']]), - Query::greaterThan('value', 0) + $metrics[] = [ + 'from' => str_replace('{bucketId}', $bucket->getId(), 'files.{bucketId}.storage.size'), + 'to' => str_replace('{bucketInternalId}', $bucket->getInternalId(), '{bucketInternalId}.files.storage') + ]; + + foreach ($metrics as $metric) { + $stats = $dbForProject->find('stats', [ + Query::equal('metric', [$metric['from']]), + Query::greaterThan('value', 0) + ]); + foreach ($stats as $stat) { + $stat->setAttribute('metricV2', $metric['to']); + $dbForProject->updateDocument('stats', $stat->getId(), $stat); + Console::log($stat->getId() . ', ' . $metric['from'] . ', ' . $metric['to'] . ', ' . $stat['period']); + } + } + } + + /** + * Databases, collections, documents + */ + $databases = $dbForProject->find('databases', []); + foreach ($databases as $database) { + $metrics = []; + $metrics[] = [ + 'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'), + 'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.collections')]; + $metrics[] = [ + 'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'), + 'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.documents')]; + foreach ($metrics as $metric) { + $stats = $dbForProject->find('stats', [ + Query::equal('metric', [$metric['from']]), + Query::greaterThan('value', 0) ]); - foreach ($__stats as $__stat) { - $__stat->setAttribute('metricV2', $__metric['to']); - $dbForProject->updateDocument('stats', $__stat->getId(), $__stat); + foreach ($stats as $stat) { + $stat->setAttribute('metricV2', $metric['to']); + $dbForProject->updateDocument('stats', $stat->getId(), $stat); + Console::log($stat->getId() . ', ' . $metric['from'] . ', ' . $metric['to'] . ', ' . $stat['period']); + } + + $collections = $dbForProject->find('database_' . $database->getInternalId(), []); + + foreach ($collections as $collection) { + $__metric = [ + 'from' => str_replace(['{databaseId}', '{collectionId}'], [$database->getId(), $collection->getId()], 'documents.{databaseId}/{collectionId}.count.total'), + 'to' => str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], '{databaseInternalId}.{collectionInternalId}.documents')]; + + $__stats = $dbForProject->find('stats', [ + Query::equal('metric', [$__metric['from']]), + Query::greaterThan('value', 0) + ]); + + foreach ($__stats as $__stat) { + $__stat->setAttribute('metricV2', $__metric['to']); + $dbForProject->updateDocument('stats', $__stat->getId(), $__stat); + Console::log($__stat->getId() . ', ' . $__metric['from'] . ', ' . $__metric['to'] . ', ' . $__stat['period']); + } + } + } + } + + /** + * executions + */ + $functions = $dbForProject->find('functions', []); + foreach ($functions as $function) { + $metrics = []; + $metrics[] = [ + 'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.time'), + 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions.compute') + ]; + $metrics[] = [ + 'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.total'), + 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions') + ]; + + $metrics[] = [ + 'from' => str_replace('{functionId}', $function->getId(), 'builds.{functionId}.compute.total'), + 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.builds') + ]; + + $metrics[] = [ + 'from' => str_replace('{functionId}', $function->getId(), 'builds.{functionId}.compute.time'), + 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.compute') + ]; + foreach ($metrics as $metric) { + $stats = $dbForProject->find('stats', [ + Query::equal('metric', [$metric['from']]), + Query::greaterThan('value', 0) + ]); + + foreach ($stats as $stat) { + $stat->setAttribute('metricV2', $metric['to']); + $dbForProject->updateDocument('stats', $stat->getId(), $stat); + Console::log($stat->getId() . ', ' . $metric['from'] . ', ' . $metric['to'] . ', ' . $stat['period']); } } } } - - /** - * executions - */ - $functions = $dbForProject->find('functions', []); - foreach ($functions as $function) { - $metrics = []; - $metrics[] = [ - 'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.time'), - 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions.compute') - ]; - $metrics[] = [ - 'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.total'), - 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions') - ]; - - $metrics[] = [ - 'from' => str_replace('{functionId}', $function->getId(), 'builds.{functionId}.compute.total'), - 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.builds') - ]; - - $metrics[] = [ - 'from' => str_replace('{functionId}', $function->getId(), 'builds.{functionId}.compute.time'), - 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.compute') - ]; - foreach ($metrics as $metric) { - $stats = $dbForProject->find('stats', [ - Query::equal('metric', [$metric['from']]), - Query::greaterThan('value', 0) - ]); - - foreach ($stats as $stat) { - $stat->setAttribute('metricV2', $metric['to']); - $dbForProject->updateDocument('stats', $stat->getId(), $stat); - } - } - } + $projectCursor = $projects[array_key_last($projects)]; } } }