From c44bb27a353a6f2713f494ffbc46cba6d67048a4 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 19 Mar 2023 15:06:01 +0200 Subject: [PATCH] usage migration --- Dockerfile | 4 +- bin/migrate-usage | 3 - bin/usage-create-metrics | 3 + bin/usage-inf-calc | 3 - bin/usage-update-metrics | 3 + src/Appwrite/Platform/Services/Tasks.php | 8 +- src/Appwrite/Platform/Tasks/MigrateUsage.php | 166 -------------- .../Platform/Tasks/UsageCreateMetrics.php | 216 ++++++++++++++++++ src/Appwrite/Platform/Tasks/UsageInfCalc.php | 185 --------------- .../Platform/Tasks/UsageUpdateMetrics.php | 209 +++++++++++++++++ 10 files changed, 437 insertions(+), 363 deletions(-) delete mode 100644 bin/migrate-usage create mode 100644 bin/usage-create-metrics delete mode 100644 bin/usage-inf-calc create mode 100644 bin/usage-update-metrics delete mode 100644 src/Appwrite/Platform/Tasks/MigrateUsage.php create mode 100644 src/Appwrite/Platform/Tasks/UsageCreateMetrics.php delete mode 100644 src/Appwrite/Platform/Tasks/UsageInfCalc.php create mode 100644 src/Appwrite/Platform/Tasks/UsageUpdateMetrics.php diff --git a/Dockerfile b/Dockerfile index af804e9f79..a8d42013ec 100755 --- a/Dockerfile +++ b/Dockerfile @@ -139,8 +139,8 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-mails && \ chmod +x /usr/local/bin/worker-messaging && \ chmod +x /usr/local/bin/worker-webhooks && \ - chmod +x /usr/local/bin/migrate-usage && \ - chmod +x /usr/local/bin/usage-inf-calc + chmod +x /usr/local/bin/usage-create-metrics && \ + chmod +x /usr/local/bin/usage-update-metrics # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/bin/migrate-usage b/bin/migrate-usage deleted file mode 100644 index 242e69a88d..0000000000 --- a/bin/migrate-usage +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php migrate-usage $@ \ No newline at end of file diff --git a/bin/usage-create-metrics b/bin/usage-create-metrics new file mode 100644 index 0000000000..82579f9bae --- /dev/null +++ b/bin/usage-create-metrics @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php usage-create-metrics $@ \ No newline at end of file diff --git a/bin/usage-inf-calc b/bin/usage-inf-calc deleted file mode 100644 index 43c9acade3..0000000000 --- a/bin/usage-inf-calc +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -php /usr/src/code/app/cli.php usage-inf-calc $@ \ No newline at end of file diff --git a/bin/usage-update-metrics b/bin/usage-update-metrics new file mode 100644 index 0000000000..78ae8dafe9 --- /dev/null +++ b/bin/usage-update-metrics @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php usage-update-metrics $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 98eb6542ed..ce7e0c9200 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -2,8 +2,8 @@ namespace Appwrite\Platform\Services; -use Appwrite\Platform\Tasks\MigrateUsage; -use Appwrite\Platform\Tasks\UsageInfCalc; +use Appwrite\Platform\Tasks\UsageUpdateMetrics; +use Appwrite\Platform\Tasks\UsageCreateMetrics; use Utopia\Platform\Service; use Appwrite\Platform\Tasks\Doctor; use Appwrite\Platform\Tasks\Install; @@ -40,7 +40,7 @@ class Tasks extends Service ->addAction(SDKs::getName(), new SDKs()) ->addAction(VolumeSync::getName(), new VolumeSync()) ->addAction(Specs::getName(), new Specs()) - ->addAction(MigrateUsage::getName(), new MigrateUsage()) - ->addAction(UsageInfCalc::getName(), new UsageInfCalc()); + ->addAction(UsageUpdateMetrics::getName(), new UsageUpdateMetrics()) + ->addAction(UsageCreateMetrics::getName(), new UsageCreateMetrics()); } } diff --git a/src/Appwrite/Platform/Tasks/MigrateUsage.php b/src/Appwrite/Platform/Tasks/MigrateUsage.php deleted file mode 100644 index 100713c53d..0000000000 --- a/src/Appwrite/Platform/Tasks/MigrateUsage.php +++ /dev/null @@ -1,166 +0,0 @@ -desc('Migrate old usage metric pattern to new') - ->param('structure', false, new Boolean(), 'Manipulate usage cols structure', true) - ->inject('dbForConsole') - ->inject('getProjectDB') - ->callback(fn(bool $structure, Database $dbForConsole, callable $getProjectDB) => $this->action($structure, $dbForConsole, $getProjectDB)); - } - - - public function action(bool $structure, Database $dbForConsole, callable $getProjectDB): void - { - Authorization::disable(); - Authorization::setDefaultStatus(false); - - - $limit = 100; - $projectCursor = null; - while (true) { - $projectsQueries = [Query::limit($limit)]; - if ($projectCursor !== null) { - $projectsQueries[] = Query::cursorAfter($projectCursor); - } - $projects = $dbForConsole->find('projects', $projectsQueries); - - if (count($projects) === 0) { - break; - } - - foreach ($projects as $project) { - Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getInternalId() . ")"); - $dbForProject = $getProjectDB($project); - - if ($structure) { - try { - $dbForProject->renameAttribute('stats', 'metric', 'metricOld'); - $dbForProject->renameAttribute('stats', 'metricV2', 'metric'); - } catch (\Throwable $th) { - Console::Error("Error while trying yo rename col: {$th->getMessage()}"); - } - - Console::log("Renaming stats col metric to metricOld, metricV2 to metric"); - continue; - } - - - /** - * Project level - */ - $metrics[] = ['from' => 'project.$all.network.outbound', 'to' => 'network.outbound']; - $metrics[] = ['from' => 'project.$all.network.inbound', 'to' => 'network.inbound']; - $metrics[] = ['from' => 'project.$all.network.requests', 'to' => 'network.requests']; - $metrics[] = ['from' => 'executions.$all.compute.total', 'to' => 'executions']; - $metrics[] = ['from' => 'builds.$all.compute.time', 'to' => 'builds.compute']; - $metrics[] = ['from' => 'files.$all.storage.size', 'to' => 'files.storage']; - $metrics[] = ['from' => 'files.$all.count.total', 'to' => 'files']; - $metrics[] = ['from' => 'buckets.$all.count.total', 'to' => 'buckets']; - $metrics[] = ['from' => 'collections.$all.count.total', 'to' => 'collections']; - $metrics[] = ['from' => 'databases.$all.count.total', 'to' => 'databases']; - $metrics[] = ['from' => 'users.$all.count.total ', 'to' => 'users']; - $metrics[] = ['from' => 'documents.$all.count.total', 'to' => 'documents']; - $metrics[] = ['from' => 'documents.$all.count.total', 'to' => 'documents']; - - foreach ($metrics as $metric) { - $stats = $dbForProject->find('stats', [ - Query::equal('metric', [$metric['from']]), - Query::greaterThan('value', 0) - ]); - foreach ($stats as $stat) { - $stat->setAttribute('metricV2', $metric['to']); - $dbForProject->updateDocument('stats', $stat->getId(), $stat); - } - } - - /** - * Databases, collections, documents - */ - $databases = $dbForProject->find('databases', []); - foreach ($databases as $database) { - $metrics = []; - $metrics[] = [ - 'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'), - 'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.collections')]; - $metrics[] = [ - 'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'), - 'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.documents')]; - foreach ($metrics as $metric) { - $stats = $dbForProject->find('stats', [ - Query::equal('metric', [$metric['from']]), - Query::greaterThan('value', 0) - ]); - - foreach ($stats as $stat) { - $stat->setAttribute('metricV2', $metric['to']); - $dbForProject->updateDocument('stats', $stat->getId(), $stat); - } - - $collections = $dbForProject->find('database_' . $database->getInternalId(), []); - - foreach ($collections as $collection) { - $__metric = [ - 'from' => str_replace(['{databaseId}','{collectionId}'], [$database->getId(), $collection->getId()], 'documents.{databaseId}/{collectionId}.count.total'), - 'to' => str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], '{databaseInternalId}.{collectionInternalId}.documents')]; - - $__stats = $dbForProject->find('stats', [ - Query::equal('metric', [$__metric['from']]), - Query::greaterThan('value', 0) - ]); - - foreach ($__stats as $__stat) { - $__stat->setAttribute('metricV2', $__metric['to']); - $dbForProject->updateDocument('stats', $__stat->getId(), $__stat); - } - } - } - } - - /** - * executions - */ - $functions = $dbForProject->find('functions', []); - foreach ($functions as $function) { - $metrics = []; - $metrics[] = [ - 'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.time'), - 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions.compute')]; - $metrics[] = [ - 'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.total'), - 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions')]; - foreach ($metrics as $metric) { - $stats = $dbForProject->find('stats', [ - Query::equal('metric', [$metric['from']]), - Query::greaterThan('value', 0) - ]); - - foreach ($stats as $stat) { - $stat->setAttribute('metricV2', $metric['to']); - $dbForProject->updateDocument('stats', $stat->getId(), $stat); - } - } - } - $projectCursor = $projects[array_key_last($projects)]; - } - } - } -} diff --git a/src/Appwrite/Platform/Tasks/UsageCreateMetrics.php b/src/Appwrite/Platform/Tasks/UsageCreateMetrics.php new file mode 100644 index 0000000000..0cc050d227 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/UsageCreateMetrics.php @@ -0,0 +1,216 @@ +desc('Usage inf calculation')->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn(Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB)); + } + + + /** + * @throws \Exception + */ + public function action(Database $dbForConsole, callable $getProjectDB): void + { + Authorization::disable(); + Authorization::setDefaultStatus(false); + + function createInfMetric($dbForProject, $key, $value): void + { + $id = \md5("null_inf_{$key}"); + + $stats = $dbForProject->getDocument('stats', $id); + if ($stats->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => 'inf', + 'time' => null, + 'metric' => '', + 'metricV2' => $key, + 'value' => (int)$value, + 'region' => 'default', + ])); + } else { + $stats->setAttribute('value', (int)($stats['value'] + $value)); + $dbForProject->updateDocument('stats', $stats->getId(), $stats); + } + } + + $projects = $dbForConsole->find('projects'); + if (count($projects) === 0) { + Console::info("Could not found any projects"); + return; + } + foreach ($projects as $project) { + Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getInternalId() . ")"); + $dbForProject = $getProjectDB($project); + /** + * Network - inbound, outbound, requests + */ + $metrics = []; + $metrics[] = ['from' => 'project.$all.network.outbound', 'to' => 'network.outbound']; + $metrics[] = ['from' => 'project.$all.network.inbound', 'to' => 'network.inbound']; + $metrics[] = ['from' => 'project.$all.network.requests', 'to' => 'network.requests']; + + foreach ($metrics as $metric) { + $stats = $dbForProject->find('stats', [ + Query::equal('metric', [$metric['from']]), + Query::equal('period', ['1d']), + Query::greaterThan('value', 0) + ]); + foreach ($stats as $stat) { + createInfMetric($dbForProject, $metric['to'], $stat['value']); + } + } + + /** + * users + */ + $users = $dbForProject->count('users', [ + Query::equal('status', [1]), + ]); + createInfMetric($dbForProject, 'users', $users); + /** + * Sessions + */ + $sessions = $dbForProject->count('sessions', []); + createInfMetric($dbForProject, 'sessions', $sessions); + + /** + * Buckets, Files + */ + $filesStoragePerProject = 0; + $filesCountPerProject = 0; + $bucketsCountPerProject = 0; + $buckets = $dbForProject->find('buckets', []); + foreach ($buckets as $bucket) { + $filesCount = 0; + $filesStorage = 0; + $files = $dbForProject->find('bucket_' . $bucket->getInternalId(), []); + foreach ($files as $file) { + $filesStorage += $file['sizeOriginal']; + $filesStoragePerProject += $file['sizeOriginal']; + $filesCount++; + $filesCountPerProject++; + } + createInfMetric($dbForProject, $bucket->getInternalId() . '.files.storage', $filesStoragePerProject); + createInfMetric($dbForProject, $bucket->getInternalId() . '.files', $filesCountPerProject); + $bucketsCountPerProject++; + } + createInfMetric($dbForProject, 'files.storage', $filesStoragePerProject); + createInfMetric($dbForProject, 'files', $filesCountPerProject); + createInfMetric($dbForProject, 'buckets', $bucketsCountPerProject); + + /** + * Databases, collections, documents + */ + $databasesCountPerProject = 0; + $collectionsCountPerProject = 0; + $documentsCountPerProject = 0; + $databases = $dbForProject->find('databases', []); + foreach ($databases as $database) { + $databasesCountPerProject++; + $documentsCountPerDatabase = 0; + $collectionCountPerDatabase = 0; + $collections = $dbForProject->find('database_' . $database->getInternalId(), []); + foreach ($collections as $collection) { + $documentsCountPerCollection = 0; + $collectionsCountPerProject++; + $collectionCountPerDatabase++; + $documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), []); + $documentsCountPerProject += $documents; + $documentsCountPerDatabase += $documents; + $documentsCountPerCollection += $documents; + createInfMetric($dbForProject, $database->getInternalId() . '.' . $collection->getInternalId() . '.documents', $documentsCountPerCollection); + } + createInfMetric($dbForProject, $database->getInternalId() . '.documents', $documentsCountPerDatabase); + } + createInfMetric($dbForProject, 'databases', $databasesCountPerProject); + createInfMetric($dbForProject, 'collections', $collectionsCountPerProject); + createInfMetric($dbForProject, 'documents', $documentsCountPerProject); + + /** + * Functions, deployments, builds, executions + */ + $deploymentSizePerProject = 0; + $deploymentCountPerProject = 0; + $buildDurationPerProject = 0; + $buildCountPerProject = 0; + $executionCountPerProject = 0; + $executionDurationPerProject = 0 ; + + $functions = $dbForProject->find('functions', []); + foreach ($functions as $function) { + $deploymentSize = 0; + $deploymentCount = 0; + $buildDuration = 0; + $executionDuration = 0; + $executionCount = 0; + $buildCount = 0; + + $deployments = $dbForProject->find('deployments', [ + Query::equal('resourceId', [$function->getId()]), + ]); + foreach ($deployments as $deployment) { + $deploymentSize += $deployment['size']; + $deploymentSizePerProject += $deployment['size']; + $deploymentCount++; + $deploymentCountPerProject++; + + $build = $dbForProject->findOne('builds', [ + Query::equal('deploymentId', [$deployment->getId()]), + ]); + + if (!empty($build)) { + $buildDuration += $build['duration']; + $buildDurationPerProject += $build['duration']; + $buildCountPerProject++; + $buildCount++; + } + } + + createInfMetric($dbForProject, $function->getInternalId() . '.builds', $buildCount); + createInfMetric($dbForProject, $function->getInternalId() . '.builds.compute', (int)($buildDuration * 1000)); + createInfMetric($dbForProject, $function->getInternalId() . '.deployments', $deploymentCount); + createInfMetric($dbForProject, $function->getInternalId() . '.deployments.storage', $deploymentSize); + + $executions = $dbForProject->find('executions', [ + Query::equal('functionId', [$function->getId()]) + ]); + foreach ($executions as $execution) { + $executionDuration += $execution['duration']; + $executionDurationPerProject += $execution['duration']; + $executionCount++; + $executionCountPerProject++; + } + + createInfMetric($dbForProject, $function->getInternalId() . '.executions.compute', (int)($executionDuration * 1000)); + createInfMetric($dbForProject, $function->getInternalId() . '.executions', $executionCount); + } + createInfMetric($dbForProject, 'deployments', $deploymentCountPerProject); + createInfMetric($dbForProject, 'builds', $buildCountPerProject); + createInfMetric($dbForProject, 'executions', $executionCountPerProject); + createInfMetric($dbForProject, 'deployments.storage', $deploymentSizePerProject); + createInfMetric($dbForProject, 'builds.compute', (int)($buildDurationPerProject * 1000)); + createInfMetric($dbForProject, 'executions.compute', (int)($executionDurationPerProject * 1000)); + } + } +} diff --git a/src/Appwrite/Platform/Tasks/UsageInfCalc.php b/src/Appwrite/Platform/Tasks/UsageInfCalc.php deleted file mode 100644 index 4e9c291d77..0000000000 --- a/src/Appwrite/Platform/Tasks/UsageInfCalc.php +++ /dev/null @@ -1,185 +0,0 @@ -desc('Usage inf calculation')->inject('dbForConsole') - ->inject('getProjectDB') - ->callback(fn(Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB)); - } - - - public function action(Database $dbForConsole, callable $getProjectDB): void - { - Authorization::disable(); - Authorization::setDefaultStatus(false); - - function createInfMetric($dbForProject, $key, $value): void - { - $id = \md5("null_inf_{$key}"); - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => 'inf', - 'time' => null, - 'metric' => '', - 'metricV2' => $key, - 'value' => $value, - 'region' => 'default', - ])); - } - - - $limit = 100; - $projectCursor = null; - while (true) { - $projectsQueries = [Query::limit($limit)]; - if ($projectCursor !== null) { - $projectsQueries[] = Query::cursorAfter($projectCursor); - } - $projects = $dbForConsole->find('projects', $projectsQueries); - - if (count($projects) === 0) { - break; - } - - foreach ($projects as $project) { - Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getInternalId() . ")"); - $dbForProject = $getProjectDB($project); - /** - * Buckets, Files - */ - $filesStoragePerProject = 0; - $filesCountPerProject = 0; - $bucketsCountPerProject = 0; - $buckets = $dbForProject->find('buckets', []); - foreach ($buckets as $bucket) { - $filesCount = 0; - $filesStorage = 0; - $files = $dbForProject->find('bucket_' . $bucket->getInternalId(), []); - foreach ($files as $file) { - $filesStorage += $file['sizeOriginal']; - $filesStoragePerProject += $file['sizeOriginal']; - $filesCount++; - $filesCountPerProject++; - } - createInfMetric($dbForProject, '{' . $bucket->getInternalId() . '}.files.storage', $filesStoragePerProject); - createInfMetric($dbForProject, '{' . $bucket->getInternalId() . '}.files', $filesCountPerProject); - $bucketsCountPerProject++; - } - createInfMetric($dbForProject, 'files.storage', $filesStoragePerProject); - createInfMetric($dbForProject, 'files', $filesCountPerProject); - createInfMetric($dbForProject, 'buckets', $bucketsCountPerProject); - - /** - * Databases, collections, documents - */ - $databasesCountPerProject = 0; - $collectionsCountPerProject = 0; - $documentsCountPerProject = 0; - $databases = $dbForProject->find('databases', []); - foreach ($databases as $database) { - $databasesCountPerProject++; - $documentsCountPerDatabase = 0; - $collectionCountPerDatabase = 0; - $collections = $dbForProject->find('database_' . $database->getInternalId(), []); - foreach ($collections as $collection) { - $documentsCountPerCollection = 0; - $collectionsCountPerProject++; - $collectionCountPerDatabase++; - $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), []); - foreach ($documents as $document) { - $documentsCountPerProject++; - $documentsCountPerDatabase++; - $documentsCountPerCollection++; - createInfMetric($dbForProject, '{' . $bucket->getInternalId() . '}.files.storage', $filesStoragePerProject); - createInfMetric($dbForProject, '{' . $bucket->getInternalId() . '}.files', $filesCountPerProject); - } - createInfMetric($dbForProject, '{' . $database->getInternalId() . '}.' . $collection->getId() . '.documents', $documentsCountPerCollection); - } - createInfMetric($dbForProject, '{' . $database->getInternalId() . '}.documents', $documentsCountPerDatabase); - } - createInfMetric($dbForProject, 'databases', $databasesCountPerProject); - createInfMetric($dbForProject, 'collections', $collectionsCountPerProject); - createInfMetric($dbForProject, 'documents', $documentsCountPerProject); - - /** - * Functions, deployments, builds, executions - */ - $deploymentSizePerProject = 0; - $deploymentCountPerProject = 0; - $buildDurationPerProject = 0; - $buildCountPerProject = 0; - $executionCountPerProject = 0; - $executionDurationPerProject = 0 ; - $functions = $dbForProject->find('functions', []); - foreach ($functions as $function) { - $deploymentSize = 0; - $deploymentCount = 0; - $buildDuration = 0; - $executionDuration = 0; - $executionCount = 0; - $buildCount = 0; - $deployments = $dbForProject->find('deployments', [ - Query::equal('resourceId', [$function->getId()]), - ]); - foreach ($deployments as $deployment) { - $deploymentSize += $deployment['size']; - $deploymentSizePerProject += $deployment['size']; - $deploymentCount++; - $deploymentCountPerProject++; - - $build = $dbForProject->findOne('builds', [ - Query::equal('deploymentId', [$deployment->getId()]), - ]); - - if (!empty($build)) { - $buildDuration += $build['duration'] * 1000; - $buildDurationPerProject += $build['duration'] * 1000; - $buildCountPerProject++; - $buildCount++; - } - } - - createInfMetric($dbForProject, $function->getInternalId() . '.builds', $buildCount); - createInfMetric($dbForProject, $function->getInternalId() . '.builds.compute', $buildDuration); - createInfMetric($dbForProject, $function->getInternalId() . '.deployments', $deploymentCount); - createInfMetric($dbForProject, $function->getInternalId() . '.deployments.storage', $deploymentSize); - - $executions = $dbForProject->find('executions', []); - foreach ($executions as $execution) { - $executionDuration += $execution['duration'] * 1000; - $executionDurationPerProject += $execution['duration'] * 1000; - $executionCount++; - $executionCountPerProject++; - } - createInfMetric($dbForProject, $function->getInternalId() . 'executions.compute', $executionDuration); - createInfMetric($dbForProject, $function->getInternalId() . '.executions', $executionCount); - } - createInfMetric($dbForProject, 'deployments', $deploymentCountPerProject); - createInfMetric($dbForProject, 'builds', $buildCountPerProject); - createInfMetric($dbForProject, 'executions', $executionCountPerProject); - createInfMetric($dbForProject, 'deployments.storage', $deploymentSizePerProject); - createInfMetric($dbForProject, 'builds.compute', $buildDurationPerProject); - createInfMetric($dbForProject, 'executions.compute', $executionDurationPerProject); - - $projectCursor = $projects[array_key_last($projects)]; - } - } - } -} diff --git a/src/Appwrite/Platform/Tasks/UsageUpdateMetrics.php b/src/Appwrite/Platform/Tasks/UsageUpdateMetrics.php new file mode 100644 index 0000000000..ab44bf89c0 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/UsageUpdateMetrics.php @@ -0,0 +1,209 @@ +desc('Migrate old usage metric pattern to new') + ->param('commit', false, new Boolean(true), 'Switch usage metric columns', true) + ->param('rollback', false, new Boolean(true), 'Rollback usage metric columns ', true) + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn(bool $commit, $rollback, Database $dbForConsole, callable $getProjectDB) => $this->action($commit, $rollback, $dbForConsole, $getProjectDB)); + } + + + public function action(bool $commit, $rollback, Database $dbForConsole, callable $getProjectDB): void + { + Authorization::disable(); + Authorization::setDefaultStatus(false); + + + $projects = $dbForConsole->find('projects'); + if (count($projects) === 0) { + Console::info("Could not found any projects"); + return; + } + + foreach ($projects as $project) { + Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getInternalId() . ")"); + $dbForProject = $getProjectDB($project); + + if ($commit) { + try { + $dbForProject->renameAttribute('stats', 'metric', 'metricOld'); + $dbForProject->renameAttribute('stats', 'metricV2', 'metric'); + } catch (\Throwable $th) { + Console::Error("Error while trying to rename col: {$th->getMessage()}"); + } + + Console::log("Commit -Renaming stats column metric to metricOld, metricV2 to metric"); + continue; + } + + if ($rollback) { + try { + $dbForProject->renameAttribute('stats', 'metric', 'metricV2'); + $dbForProject->renameAttribute('stats', 'metricOld', 'metric'); + } catch (\Throwable $th) { + Console::Error("Error while trying to rename column: {$th->getMessage()}"); + } + + Console::log("Rollback - Renaming stats column metric to metricV2, metricOld to metric"); + continue; + } + + /** + * Project level + */ + $metrics[] = ['from' => 'project.$all.network.outbound', 'to' => 'network.outbound']; + $metrics[] = ['from' => 'project.$all.network.inbound', 'to' => 'network.inbound']; + $metrics[] = ['from' => 'project.$all.network.requests', 'to' => 'network.requests']; + $metrics[] = ['from' => 'executions.$all.compute.total', 'to' => 'executions']; + $metrics[] = ['from' => 'executions.$all.compute.time', 'to' => 'executions.compote']; + $metrics[] = ['from' => 'builds.$all.compute.total', 'to' => 'builds']; + $metrics[] = ['from' => 'builds.$all.compute.time', 'to' => 'builds.compute']; + $metrics[] = ['from' => 'files.$all.storage.size', 'to' => 'files.storage']; + $metrics[] = ['from' => 'files.$all.count.total', 'to' => 'files']; + $metrics[] = ['from' => 'buckets.$all.count.total', 'to' => 'buckets']; + $metrics[] = ['from' => 'collections.$all.count.total', 'to' => 'collections']; + $metrics[] = ['from' => 'databases.$all.count.total', 'to' => 'databases']; + $metrics[] = ['from' => 'documents.$all.count.total', 'to' => 'documents']; + $metrics[] = ['from' => 'users.$all.count.total ', 'to' => 'users']; + foreach ($metrics as $metric) { + $stats = $dbForProject->find('stats', [ + Query::equal('metric', [$metric['from']]), + Query::greaterThan('value', 0) + ]); + foreach ($stats as $stat) { + $stat->setAttribute('metricV2', $metric['to']); + $dbForProject->updateDocument('stats', $stat->getId(), $stat); + } + } + + /** + * Buckets, Files + */ + $buckets = $dbForProject->find('buckets', []); + foreach ($buckets as $bucket) { + $metrics = []; + + $metrics[] = [ + 'from' => str_replace('{bucketId}', $bucket->getId(), 'files.{bucketId}.count.total'), + 'to' => str_replace('{bucketInternalId}', $bucket->getInternalId(), '{bucketInternalId}.files') + ]; + + $metrics[] = [ + 'from' => str_replace('{bucketId}', $bucket->getId(), 'files.{bucketId}.storage.size'), + 'to' => str_replace('{bucketInternalId}', $bucket->getInternalId(), '{bucketInternalId}.files.storage') + ]; + + foreach ($metrics as $metric) { + $stats = $dbForProject->find('stats', [ + Query::equal('metric', [$metric['from']]), + Query::greaterThan('value', 0) + ]); + foreach ($stats as $stat) { + $stat->setAttribute('metricV2', $metric['to']); + $dbForProject->updateDocument('stats', $stat->getId(), $stat); + } + } + } + + /** + * Databases, collections, documents + */ + $databases = $dbForProject->find('databases', []); + foreach ($databases as $database) { + $metrics = []; + $metrics[] = [ + 'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'), + 'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.collections')]; + $metrics[] = [ + 'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'), + 'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.documents')]; + foreach ($metrics as $metric) { + $stats = $dbForProject->find('stats', [ + Query::equal('metric', [$metric['from']]), + Query::greaterThan('value', 0) + ]); + + foreach ($stats as $stat) { + $stat->setAttribute('metricV2', $metric['to']); + $dbForProject->updateDocument('stats', $stat->getId(), $stat); + } + + $collections = $dbForProject->find('database_' . $database->getInternalId(), []); + + foreach ($collections as $collection) { + $__metric = [ + 'from' => str_replace(['{databaseId}','{collectionId}'], [$database->getId(), $collection->getId()], 'documents.{databaseId}/{collectionId}.count.total'), + 'to' => str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], '{databaseInternalId}.{collectionInternalId}.documents')]; + + $__stats = $dbForProject->find('stats', [ + Query::equal('metric', [$__metric['from']]), + Query::greaterThan('value', 0) + ]); + + foreach ($__stats as $__stat) { + $__stat->setAttribute('metricV2', $__metric['to']); + $dbForProject->updateDocument('stats', $__stat->getId(), $__stat); + } + } + } + } + + /** + * executions + */ + $functions = $dbForProject->find('functions', []); + foreach ($functions as $function) { + $metrics = []; + $metrics[] = [ + 'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.time'), + 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions.compute') + ]; + $metrics[] = [ + 'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.total'), + 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions') + ]; + + $metrics[] = [ + 'from' => str_replace('{functionId}', $function->getId(), 'builds.{functionId}.compute.total'), + 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.builds') + ]; + + $metrics[] = [ + 'from' => str_replace('{functionId}', $function->getId(), 'builds.{functionId}.compute.time'), + 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.compute') + ]; + foreach ($metrics as $metric) { + $stats = $dbForProject->find('stats', [ + Query::equal('metric', [$metric['from']]), + Query::greaterThan('value', 0) + ]); + + foreach ($stats as $stat) { + $stat->setAttribute('metricV2', $metric['to']); + $dbForProject->updateDocument('stats', $stat->getId(), $stat); + } + } + } + } + } +}