mirror of
https://github.com/appwrite/appwrite.git
synced 2026-05-26 13:51:13 +00:00
usage migration
This commit is contained in:
shimon
+2
-2
@@ -139,8 +139,8 @@ RUN chmod +x /usr/local/bin/doctor && \
|
||||
chmod +x /usr/local/bin/worker-mails && \
|
||||
chmod +x /usr/local/bin/worker-messaging && \
|
||||
chmod +x /usr/local/bin/worker-webhooks && \
|
||||
chmod +x /usr/local/bin/migrate-usage && \
|
||||
chmod +x /usr/local/bin/usage-inf-calc
|
||||
chmod +x /usr/local/bin/usage-create-metrics && \
|
||||
chmod +x /usr/local/bin/usage-update-metrics
|
||||
|
||||
# Letsencrypt Permissions
|
||||
RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
php /usr/src/code/app/cli.php migrate-usage $@
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
|
||||
php /usr/src/code/app/cli.php usage-create-metrics $@
|
||||
@@ -1,3 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
php /usr/src/code/app/cli.php usage-inf-calc $@
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
|
||||
php /usr/src/code/app/cli.php usage-update-metrics $@
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
namespace Appwrite\Platform\Services;
|
||||
|
||||
use Appwrite\Platform\Tasks\MigrateUsage;
|
||||
use Appwrite\Platform\Tasks\UsageInfCalc;
|
||||
use Appwrite\Platform\Tasks\UsageUpdateMetrics;
|
||||
use Appwrite\Platform\Tasks\UsageCreateMetrics;
|
||||
use Utopia\Platform\Service;
|
||||
use Appwrite\Platform\Tasks\Doctor;
|
||||
use Appwrite\Platform\Tasks\Install;
|
||||
@@ -40,7 +40,7 @@ class Tasks extends Service
|
||||
->addAction(SDKs::getName(), new SDKs())
|
||||
->addAction(VolumeSync::getName(), new VolumeSync())
|
||||
->addAction(Specs::getName(), new Specs())
|
||||
->addAction(MigrateUsage::getName(), new MigrateUsage())
|
||||
->addAction(UsageInfCalc::getName(), new UsageInfCalc());
|
||||
->addAction(UsageUpdateMetrics::getName(), new UsageUpdateMetrics())
|
||||
->addAction(UsageCreateMetrics::getName(), new UsageCreateMetrics());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,166 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Tasks;
|
||||
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Validator\Boolean;
|
||||
use Utopia\Validator\Text;
|
||||
|
||||
class MigrateUsage extends Action
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'migrate-usage';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->desc('Migrate old usage metric pattern to new')
|
||||
->param('structure', false, new Boolean(), 'Manipulate usage cols structure', true)
|
||||
->inject('dbForConsole')
|
||||
->inject('getProjectDB')
|
||||
->callback(fn(bool $structure, Database $dbForConsole, callable $getProjectDB) => $this->action($structure, $dbForConsole, $getProjectDB));
|
||||
}
|
||||
|
||||
|
||||
public function action(bool $structure, Database $dbForConsole, callable $getProjectDB): void
|
||||
{
|
||||
Authorization::disable();
|
||||
Authorization::setDefaultStatus(false);
|
||||
|
||||
|
||||
$limit = 100;
|
||||
$projectCursor = null;
|
||||
while (true) {
|
||||
$projectsQueries = [Query::limit($limit)];
|
||||
if ($projectCursor !== null) {
|
||||
$projectsQueries[] = Query::cursorAfter($projectCursor);
|
||||
}
|
||||
$projects = $dbForConsole->find('projects', $projectsQueries);
|
||||
|
||||
if (count($projects) === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
foreach ($projects as $project) {
|
||||
Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getInternalId() . ")");
|
||||
$dbForProject = $getProjectDB($project);
|
||||
|
||||
if ($structure) {
|
||||
try {
|
||||
$dbForProject->renameAttribute('stats', 'metric', 'metricOld');
|
||||
$dbForProject->renameAttribute('stats', 'metricV2', 'metric');
|
||||
} catch (\Throwable $th) {
|
||||
Console::Error("Error while trying yo rename col: {$th->getMessage()}");
|
||||
}
|
||||
|
||||
Console::log("Renaming stats col metric to metricOld, metricV2 to metric");
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Project level
|
||||
*/
|
||||
$metrics[] = ['from' => 'project.$all.network.outbound', 'to' => 'network.outbound'];
|
||||
$metrics[] = ['from' => 'project.$all.network.inbound', 'to' => 'network.inbound'];
|
||||
$metrics[] = ['from' => 'project.$all.network.requests', 'to' => 'network.requests'];
|
||||
$metrics[] = ['from' => 'executions.$all.compute.total', 'to' => 'executions'];
|
||||
$metrics[] = ['from' => 'builds.$all.compute.time', 'to' => 'builds.compute'];
|
||||
$metrics[] = ['from' => 'files.$all.storage.size', 'to' => 'files.storage'];
|
||||
$metrics[] = ['from' => 'files.$all.count.total', 'to' => 'files'];
|
||||
$metrics[] = ['from' => 'buckets.$all.count.total', 'to' => 'buckets'];
|
||||
$metrics[] = ['from' => 'collections.$all.count.total', 'to' => 'collections'];
|
||||
$metrics[] = ['from' => 'databases.$all.count.total', 'to' => 'databases'];
|
||||
$metrics[] = ['from' => 'users.$all.count.total ', 'to' => 'users'];
|
||||
$metrics[] = ['from' => 'documents.$all.count.total', 'to' => 'documents'];
|
||||
$metrics[] = ['from' => 'documents.$all.count.total', 'to' => 'documents'];
|
||||
|
||||
foreach ($metrics as $metric) {
|
||||
$stats = $dbForProject->find('stats', [
|
||||
Query::equal('metric', [$metric['from']]),
|
||||
Query::greaterThan('value', 0)
|
||||
]);
|
||||
foreach ($stats as $stat) {
|
||||
$stat->setAttribute('metricV2', $metric['to']);
|
||||
$dbForProject->updateDocument('stats', $stat->getId(), $stat);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Databases, collections, documents
|
||||
*/
|
||||
$databases = $dbForProject->find('databases', []);
|
||||
foreach ($databases as $database) {
|
||||
$metrics = [];
|
||||
$metrics[] = [
|
||||
'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'),
|
||||
'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.collections')];
|
||||
$metrics[] = [
|
||||
'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'),
|
||||
'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.documents')];
|
||||
foreach ($metrics as $metric) {
|
||||
$stats = $dbForProject->find('stats', [
|
||||
Query::equal('metric', [$metric['from']]),
|
||||
Query::greaterThan('value', 0)
|
||||
]);
|
||||
|
||||
foreach ($stats as $stat) {
|
||||
$stat->setAttribute('metricV2', $metric['to']);
|
||||
$dbForProject->updateDocument('stats', $stat->getId(), $stat);
|
||||
}
|
||||
|
||||
$collections = $dbForProject->find('database_' . $database->getInternalId(), []);
|
||||
|
||||
foreach ($collections as $collection) {
|
||||
$__metric = [
|
||||
'from' => str_replace(['{databaseId}','{collectionId}'], [$database->getId(), $collection->getId()], 'documents.{databaseId}/{collectionId}.count.total'),
|
||||
'to' => str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], '{databaseInternalId}.{collectionInternalId}.documents')];
|
||||
|
||||
$__stats = $dbForProject->find('stats', [
|
||||
Query::equal('metric', [$__metric['from']]),
|
||||
Query::greaterThan('value', 0)
|
||||
]);
|
||||
|
||||
foreach ($__stats as $__stat) {
|
||||
$__stat->setAttribute('metricV2', $__metric['to']);
|
||||
$dbForProject->updateDocument('stats', $__stat->getId(), $__stat);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* executions
|
||||
*/
|
||||
$functions = $dbForProject->find('functions', []);
|
||||
foreach ($functions as $function) {
|
||||
$metrics = [];
|
||||
$metrics[] = [
|
||||
'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.time'),
|
||||
'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions.compute')];
|
||||
$metrics[] = [
|
||||
'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.total'),
|
||||
'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions')];
|
||||
foreach ($metrics as $metric) {
|
||||
$stats = $dbForProject->find('stats', [
|
||||
Query::equal('metric', [$metric['from']]),
|
||||
Query::greaterThan('value', 0)
|
||||
]);
|
||||
|
||||
foreach ($stats as $stat) {
|
||||
$stat->setAttribute('metricV2', $metric['to']);
|
||||
$dbForProject->updateDocument('stats', $stat->getId(), $stat);
|
||||
}
|
||||
}
|
||||
}
|
||||
$projectCursor = $projects[array_key_last($projects)];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Tasks;
|
||||
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
|
||||
class UsageCreateMetrics extends Action
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'usage-create-metrics';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->desc('Usage inf calculation')->inject('dbForConsole')
|
||||
->inject('getProjectDB')
|
||||
->callback(fn(Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @throws \Exception
|
||||
*/
|
||||
public function action(Database $dbForConsole, callable $getProjectDB): void
|
||||
{
|
||||
Authorization::disable();
|
||||
Authorization::setDefaultStatus(false);
|
||||
|
||||
function createInfMetric($dbForProject, $key, $value): void
|
||||
{
|
||||
$id = \md5("null_inf_{$key}");
|
||||
|
||||
$stats = $dbForProject->getDocument('stats', $id);
|
||||
if ($stats->isEmpty()) {
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
'$id' => $id,
|
||||
'period' => 'inf',
|
||||
'time' => null,
|
||||
'metric' => '',
|
||||
'metricV2' => $key,
|
||||
'value' => (int)$value,
|
||||
'region' => 'default',
|
||||
]));
|
||||
} else {
|
||||
$stats->setAttribute('value', (int)($stats['value'] + $value));
|
||||
$dbForProject->updateDocument('stats', $stats->getId(), $stats);
|
||||
}
|
||||
}
|
||||
|
||||
$projects = $dbForConsole->find('projects');
|
||||
if (count($projects) === 0) {
|
||||
Console::info("Could not found any projects");
|
||||
return;
|
||||
}
|
||||
foreach ($projects as $project) {
|
||||
Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getInternalId() . ")");
|
||||
$dbForProject = $getProjectDB($project);
|
||||
/**
|
||||
* Network - inbound, outbound, requests
|
||||
*/
|
||||
$metrics = [];
|
||||
$metrics[] = ['from' => 'project.$all.network.outbound', 'to' => 'network.outbound'];
|
||||
$metrics[] = ['from' => 'project.$all.network.inbound', 'to' => 'network.inbound'];
|
||||
$metrics[] = ['from' => 'project.$all.network.requests', 'to' => 'network.requests'];
|
||||
|
||||
foreach ($metrics as $metric) {
|
||||
$stats = $dbForProject->find('stats', [
|
||||
Query::equal('metric', [$metric['from']]),
|
||||
Query::equal('period', ['1d']),
|
||||
Query::greaterThan('value', 0)
|
||||
]);
|
||||
foreach ($stats as $stat) {
|
||||
createInfMetric($dbForProject, $metric['to'], $stat['value']);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* users
|
||||
*/
|
||||
$users = $dbForProject->count('users', [
|
||||
Query::equal('status', [1]),
|
||||
]);
|
||||
createInfMetric($dbForProject, 'users', $users);
|
||||
/**
|
||||
* Sessions
|
||||
*/
|
||||
$sessions = $dbForProject->count('sessions', []);
|
||||
createInfMetric($dbForProject, 'sessions', $sessions);
|
||||
|
||||
/**
|
||||
* Buckets, Files
|
||||
*/
|
||||
$filesStoragePerProject = 0;
|
||||
$filesCountPerProject = 0;
|
||||
$bucketsCountPerProject = 0;
|
||||
$buckets = $dbForProject->find('buckets', []);
|
||||
foreach ($buckets as $bucket) {
|
||||
$filesCount = 0;
|
||||
$filesStorage = 0;
|
||||
$files = $dbForProject->find('bucket_' . $bucket->getInternalId(), []);
|
||||
foreach ($files as $file) {
|
||||
$filesStorage += $file['sizeOriginal'];
|
||||
$filesStoragePerProject += $file['sizeOriginal'];
|
||||
$filesCount++;
|
||||
$filesCountPerProject++;
|
||||
}
|
||||
createInfMetric($dbForProject, $bucket->getInternalId() . '.files.storage', $filesStoragePerProject);
|
||||
createInfMetric($dbForProject, $bucket->getInternalId() . '.files', $filesCountPerProject);
|
||||
$bucketsCountPerProject++;
|
||||
}
|
||||
createInfMetric($dbForProject, 'files.storage', $filesStoragePerProject);
|
||||
createInfMetric($dbForProject, 'files', $filesCountPerProject);
|
||||
createInfMetric($dbForProject, 'buckets', $bucketsCountPerProject);
|
||||
|
||||
/**
|
||||
* Databases, collections, documents
|
||||
*/
|
||||
$databasesCountPerProject = 0;
|
||||
$collectionsCountPerProject = 0;
|
||||
$documentsCountPerProject = 0;
|
||||
$databases = $dbForProject->find('databases', []);
|
||||
foreach ($databases as $database) {
|
||||
$databasesCountPerProject++;
|
||||
$documentsCountPerDatabase = 0;
|
||||
$collectionCountPerDatabase = 0;
|
||||
$collections = $dbForProject->find('database_' . $database->getInternalId(), []);
|
||||
foreach ($collections as $collection) {
|
||||
$documentsCountPerCollection = 0;
|
||||
$collectionsCountPerProject++;
|
||||
$collectionCountPerDatabase++;
|
||||
$documents = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), []);
|
||||
$documentsCountPerProject += $documents;
|
||||
$documentsCountPerDatabase += $documents;
|
||||
$documentsCountPerCollection += $documents;
|
||||
createInfMetric($dbForProject, $database->getInternalId() . '.' . $collection->getInternalId() . '.documents', $documentsCountPerCollection);
|
||||
}
|
||||
createInfMetric($dbForProject, $database->getInternalId() . '.documents', $documentsCountPerDatabase);
|
||||
}
|
||||
createInfMetric($dbForProject, 'databases', $databasesCountPerProject);
|
||||
createInfMetric($dbForProject, 'collections', $collectionsCountPerProject);
|
||||
createInfMetric($dbForProject, 'documents', $documentsCountPerProject);
|
||||
|
||||
/**
|
||||
* Functions, deployments, builds, executions
|
||||
*/
|
||||
$deploymentSizePerProject = 0;
|
||||
$deploymentCountPerProject = 0;
|
||||
$buildDurationPerProject = 0;
|
||||
$buildCountPerProject = 0;
|
||||
$executionCountPerProject = 0;
|
||||
$executionDurationPerProject = 0 ;
|
||||
|
||||
$functions = $dbForProject->find('functions', []);
|
||||
foreach ($functions as $function) {
|
||||
$deploymentSize = 0;
|
||||
$deploymentCount = 0;
|
||||
$buildDuration = 0;
|
||||
$executionDuration = 0;
|
||||
$executionCount = 0;
|
||||
$buildCount = 0;
|
||||
|
||||
$deployments = $dbForProject->find('deployments', [
|
||||
Query::equal('resourceId', [$function->getId()]),
|
||||
]);
|
||||
foreach ($deployments as $deployment) {
|
||||
$deploymentSize += $deployment['size'];
|
||||
$deploymentSizePerProject += $deployment['size'];
|
||||
$deploymentCount++;
|
||||
$deploymentCountPerProject++;
|
||||
|
||||
$build = $dbForProject->findOne('builds', [
|
||||
Query::equal('deploymentId', [$deployment->getId()]),
|
||||
]);
|
||||
|
||||
if (!empty($build)) {
|
||||
$buildDuration += $build['duration'];
|
||||
$buildDurationPerProject += $build['duration'];
|
||||
$buildCountPerProject++;
|
||||
$buildCount++;
|
||||
}
|
||||
}
|
||||
|
||||
createInfMetric($dbForProject, $function->getInternalId() . '.builds', $buildCount);
|
||||
createInfMetric($dbForProject, $function->getInternalId() . '.builds.compute', (int)($buildDuration * 1000));
|
||||
createInfMetric($dbForProject, $function->getInternalId() . '.deployments', $deploymentCount);
|
||||
createInfMetric($dbForProject, $function->getInternalId() . '.deployments.storage', $deploymentSize);
|
||||
|
||||
$executions = $dbForProject->find('executions', [
|
||||
Query::equal('functionId', [$function->getId()])
|
||||
]);
|
||||
foreach ($executions as $execution) {
|
||||
$executionDuration += $execution['duration'];
|
||||
$executionDurationPerProject += $execution['duration'];
|
||||
$executionCount++;
|
||||
$executionCountPerProject++;
|
||||
}
|
||||
|
||||
createInfMetric($dbForProject, $function->getInternalId() . '.executions.compute', (int)($executionDuration * 1000));
|
||||
createInfMetric($dbForProject, $function->getInternalId() . '.executions', $executionCount);
|
||||
}
|
||||
createInfMetric($dbForProject, 'deployments', $deploymentCountPerProject);
|
||||
createInfMetric($dbForProject, 'builds', $buildCountPerProject);
|
||||
createInfMetric($dbForProject, 'executions', $executionCountPerProject);
|
||||
createInfMetric($dbForProject, 'deployments.storage', $deploymentSizePerProject);
|
||||
createInfMetric($dbForProject, 'builds.compute', (int)($buildDurationPerProject * 1000));
|
||||
createInfMetric($dbForProject, 'executions.compute', (int)($executionDurationPerProject * 1000));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,185 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Tasks;
|
||||
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
|
||||
class UsageInfCalc extends Action
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'usage-inf-calc';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->desc('Usage inf calculation')->inject('dbForConsole')
|
||||
->inject('getProjectDB')
|
||||
->callback(fn(Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB));
|
||||
}
|
||||
|
||||
|
||||
public function action(Database $dbForConsole, callable $getProjectDB): void
|
||||
{
|
||||
Authorization::disable();
|
||||
Authorization::setDefaultStatus(false);
|
||||
|
||||
function createInfMetric($dbForProject, $key, $value): void
|
||||
{
|
||||
$id = \md5("null_inf_{$key}");
|
||||
$dbForProject->createDocument('stats', new Document([
|
||||
'$id' => $id,
|
||||
'period' => 'inf',
|
||||
'time' => null,
|
||||
'metric' => '',
|
||||
'metricV2' => $key,
|
||||
'value' => $value,
|
||||
'region' => 'default',
|
||||
]));
|
||||
}
|
||||
|
||||
|
||||
$limit = 100;
|
||||
$projectCursor = null;
|
||||
while (true) {
|
||||
$projectsQueries = [Query::limit($limit)];
|
||||
if ($projectCursor !== null) {
|
||||
$projectsQueries[] = Query::cursorAfter($projectCursor);
|
||||
}
|
||||
$projects = $dbForConsole->find('projects', $projectsQueries);
|
||||
|
||||
if (count($projects) === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
foreach ($projects as $project) {
|
||||
Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getInternalId() . ")");
|
||||
$dbForProject = $getProjectDB($project);
|
||||
/**
|
||||
* Buckets, Files
|
||||
*/
|
||||
$filesStoragePerProject = 0;
|
||||
$filesCountPerProject = 0;
|
||||
$bucketsCountPerProject = 0;
|
||||
$buckets = $dbForProject->find('buckets', []);
|
||||
foreach ($buckets as $bucket) {
|
||||
$filesCount = 0;
|
||||
$filesStorage = 0;
|
||||
$files = $dbForProject->find('bucket_' . $bucket->getInternalId(), []);
|
||||
foreach ($files as $file) {
|
||||
$filesStorage += $file['sizeOriginal'];
|
||||
$filesStoragePerProject += $file['sizeOriginal'];
|
||||
$filesCount++;
|
||||
$filesCountPerProject++;
|
||||
}
|
||||
createInfMetric($dbForProject, '{' . $bucket->getInternalId() . '}.files.storage', $filesStoragePerProject);
|
||||
createInfMetric($dbForProject, '{' . $bucket->getInternalId() . '}.files', $filesCountPerProject);
|
||||
$bucketsCountPerProject++;
|
||||
}
|
||||
createInfMetric($dbForProject, 'files.storage', $filesStoragePerProject);
|
||||
createInfMetric($dbForProject, 'files', $filesCountPerProject);
|
||||
createInfMetric($dbForProject, 'buckets', $bucketsCountPerProject);
|
||||
|
||||
/**
|
||||
* Databases, collections, documents
|
||||
*/
|
||||
$databasesCountPerProject = 0;
|
||||
$collectionsCountPerProject = 0;
|
||||
$documentsCountPerProject = 0;
|
||||
$databases = $dbForProject->find('databases', []);
|
||||
foreach ($databases as $database) {
|
||||
$databasesCountPerProject++;
|
||||
$documentsCountPerDatabase = 0;
|
||||
$collectionCountPerDatabase = 0;
|
||||
$collections = $dbForProject->find('database_' . $database->getInternalId(), []);
|
||||
foreach ($collections as $collection) {
|
||||
$documentsCountPerCollection = 0;
|
||||
$collectionsCountPerProject++;
|
||||
$collectionCountPerDatabase++;
|
||||
$documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), []);
|
||||
foreach ($documents as $document) {
|
||||
$documentsCountPerProject++;
|
||||
$documentsCountPerDatabase++;
|
||||
$documentsCountPerCollection++;
|
||||
createInfMetric($dbForProject, '{' . $bucket->getInternalId() . '}.files.storage', $filesStoragePerProject);
|
||||
createInfMetric($dbForProject, '{' . $bucket->getInternalId() . '}.files', $filesCountPerProject);
|
||||
}
|
||||
createInfMetric($dbForProject, '{' . $database->getInternalId() . '}.' . $collection->getId() . '.documents', $documentsCountPerCollection);
|
||||
}
|
||||
createInfMetric($dbForProject, '{' . $database->getInternalId() . '}.documents', $documentsCountPerDatabase);
|
||||
}
|
||||
createInfMetric($dbForProject, 'databases', $databasesCountPerProject);
|
||||
createInfMetric($dbForProject, 'collections', $collectionsCountPerProject);
|
||||
createInfMetric($dbForProject, 'documents', $documentsCountPerProject);
|
||||
|
||||
/**
|
||||
* Functions, deployments, builds, executions
|
||||
*/
|
||||
$deploymentSizePerProject = 0;
|
||||
$deploymentCountPerProject = 0;
|
||||
$buildDurationPerProject = 0;
|
||||
$buildCountPerProject = 0;
|
||||
$executionCountPerProject = 0;
|
||||
$executionDurationPerProject = 0 ;
|
||||
$functions = $dbForProject->find('functions', []);
|
||||
foreach ($functions as $function) {
|
||||
$deploymentSize = 0;
|
||||
$deploymentCount = 0;
|
||||
$buildDuration = 0;
|
||||
$executionDuration = 0;
|
||||
$executionCount = 0;
|
||||
$buildCount = 0;
|
||||
$deployments = $dbForProject->find('deployments', [
|
||||
Query::equal('resourceId', [$function->getId()]),
|
||||
]);
|
||||
foreach ($deployments as $deployment) {
|
||||
$deploymentSize += $deployment['size'];
|
||||
$deploymentSizePerProject += $deployment['size'];
|
||||
$deploymentCount++;
|
||||
$deploymentCountPerProject++;
|
||||
|
||||
$build = $dbForProject->findOne('builds', [
|
||||
Query::equal('deploymentId', [$deployment->getId()]),
|
||||
]);
|
||||
|
||||
if (!empty($build)) {
|
||||
$buildDuration += $build['duration'] * 1000;
|
||||
$buildDurationPerProject += $build['duration'] * 1000;
|
||||
$buildCountPerProject++;
|
||||
$buildCount++;
|
||||
}
|
||||
}
|
||||
|
||||
createInfMetric($dbForProject, $function->getInternalId() . '.builds', $buildCount);
|
||||
createInfMetric($dbForProject, $function->getInternalId() . '.builds.compute', $buildDuration);
|
||||
createInfMetric($dbForProject, $function->getInternalId() . '.deployments', $deploymentCount);
|
||||
createInfMetric($dbForProject, $function->getInternalId() . '.deployments.storage', $deploymentSize);
|
||||
|
||||
$executions = $dbForProject->find('executions', []);
|
||||
foreach ($executions as $execution) {
|
||||
$executionDuration += $execution['duration'] * 1000;
|
||||
$executionDurationPerProject += $execution['duration'] * 1000;
|
||||
$executionCount++;
|
||||
$executionCountPerProject++;
|
||||
}
|
||||
createInfMetric($dbForProject, $function->getInternalId() . 'executions.compute', $executionDuration);
|
||||
createInfMetric($dbForProject, $function->getInternalId() . '.executions', $executionCount);
|
||||
}
|
||||
createInfMetric($dbForProject, 'deployments', $deploymentCountPerProject);
|
||||
createInfMetric($dbForProject, 'builds', $buildCountPerProject);
|
||||
createInfMetric($dbForProject, 'executions', $executionCountPerProject);
|
||||
createInfMetric($dbForProject, 'deployments.storage', $deploymentSizePerProject);
|
||||
createInfMetric($dbForProject, 'builds.compute', $buildDurationPerProject);
|
||||
createInfMetric($dbForProject, 'executions.compute', $executionDurationPerProject);
|
||||
|
||||
$projectCursor = $projects[array_key_last($projects)];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Tasks;
|
||||
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\CLI\Console;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Validator\Boolean;
|
||||
|
||||
class UsageUpdateMetrics extends Action
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'usage-update-metrics';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->desc('Migrate old usage metric pattern to new')
|
||||
->param('commit', false, new Boolean(true), 'Switch usage metric columns', true)
|
||||
->param('rollback', false, new Boolean(true), 'Rollback usage metric columns ', true)
|
||||
->inject('dbForConsole')
|
||||
->inject('getProjectDB')
|
||||
->callback(fn(bool $commit, $rollback, Database $dbForConsole, callable $getProjectDB) => $this->action($commit, $rollback, $dbForConsole, $getProjectDB));
|
||||
}
|
||||
|
||||
|
||||
public function action(bool $commit, $rollback, Database $dbForConsole, callable $getProjectDB): void
|
||||
{
|
||||
Authorization::disable();
|
||||
Authorization::setDefaultStatus(false);
|
||||
|
||||
|
||||
$projects = $dbForConsole->find('projects');
|
||||
if (count($projects) === 0) {
|
||||
Console::info("Could not found any projects");
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($projects as $project) {
|
||||
Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getInternalId() . ")");
|
||||
$dbForProject = $getProjectDB($project);
|
||||
|
||||
if ($commit) {
|
||||
try {
|
||||
$dbForProject->renameAttribute('stats', 'metric', 'metricOld');
|
||||
$dbForProject->renameAttribute('stats', 'metricV2', 'metric');
|
||||
} catch (\Throwable $th) {
|
||||
Console::Error("Error while trying to rename col: {$th->getMessage()}");
|
||||
}
|
||||
|
||||
Console::log("Commit -Renaming stats column metric to metricOld, metricV2 to metric");
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($rollback) {
|
||||
try {
|
||||
$dbForProject->renameAttribute('stats', 'metric', 'metricV2');
|
||||
$dbForProject->renameAttribute('stats', 'metricOld', 'metric');
|
||||
} catch (\Throwable $th) {
|
||||
Console::Error("Error while trying to rename column: {$th->getMessage()}");
|
||||
}
|
||||
|
||||
Console::log("Rollback - Renaming stats column metric to metricV2, metricOld to metric");
|
||||
continue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Project level
|
||||
*/
|
||||
$metrics[] = ['from' => 'project.$all.network.outbound', 'to' => 'network.outbound'];
|
||||
$metrics[] = ['from' => 'project.$all.network.inbound', 'to' => 'network.inbound'];
|
||||
$metrics[] = ['from' => 'project.$all.network.requests', 'to' => 'network.requests'];
|
||||
$metrics[] = ['from' => 'executions.$all.compute.total', 'to' => 'executions'];
|
||||
$metrics[] = ['from' => 'executions.$all.compute.time', 'to' => 'executions.compote'];
|
||||
$metrics[] = ['from' => 'builds.$all.compute.total', 'to' => 'builds'];
|
||||
$metrics[] = ['from' => 'builds.$all.compute.time', 'to' => 'builds.compute'];
|
||||
$metrics[] = ['from' => 'files.$all.storage.size', 'to' => 'files.storage'];
|
||||
$metrics[] = ['from' => 'files.$all.count.total', 'to' => 'files'];
|
||||
$metrics[] = ['from' => 'buckets.$all.count.total', 'to' => 'buckets'];
|
||||
$metrics[] = ['from' => 'collections.$all.count.total', 'to' => 'collections'];
|
||||
$metrics[] = ['from' => 'databases.$all.count.total', 'to' => 'databases'];
|
||||
$metrics[] = ['from' => 'documents.$all.count.total', 'to' => 'documents'];
|
||||
$metrics[] = ['from' => 'users.$all.count.total ', 'to' => 'users'];
|
||||
foreach ($metrics as $metric) {
|
||||
$stats = $dbForProject->find('stats', [
|
||||
Query::equal('metric', [$metric['from']]),
|
||||
Query::greaterThan('value', 0)
|
||||
]);
|
||||
foreach ($stats as $stat) {
|
||||
$stat->setAttribute('metricV2', $metric['to']);
|
||||
$dbForProject->updateDocument('stats', $stat->getId(), $stat);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Buckets, Files
|
||||
*/
|
||||
$buckets = $dbForProject->find('buckets', []);
|
||||
foreach ($buckets as $bucket) {
|
||||
$metrics = [];
|
||||
|
||||
$metrics[] = [
|
||||
'from' => str_replace('{bucketId}', $bucket->getId(), 'files.{bucketId}.count.total'),
|
||||
'to' => str_replace('{bucketInternalId}', $bucket->getInternalId(), '{bucketInternalId}.files')
|
||||
];
|
||||
|
||||
$metrics[] = [
|
||||
'from' => str_replace('{bucketId}', $bucket->getId(), 'files.{bucketId}.storage.size'),
|
||||
'to' => str_replace('{bucketInternalId}', $bucket->getInternalId(), '{bucketInternalId}.files.storage')
|
||||
];
|
||||
|
||||
foreach ($metrics as $metric) {
|
||||
$stats = $dbForProject->find('stats', [
|
||||
Query::equal('metric', [$metric['from']]),
|
||||
Query::greaterThan('value', 0)
|
||||
]);
|
||||
foreach ($stats as $stat) {
|
||||
$stat->setAttribute('metricV2', $metric['to']);
|
||||
$dbForProject->updateDocument('stats', $stat->getId(), $stat);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Databases, collections, documents
|
||||
*/
|
||||
$databases = $dbForProject->find('databases', []);
|
||||
foreach ($databases as $database) {
|
||||
$metrics = [];
|
||||
$metrics[] = [
|
||||
'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'),
|
||||
'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.collections')];
|
||||
$metrics[] = [
|
||||
'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'),
|
||||
'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.documents')];
|
||||
foreach ($metrics as $metric) {
|
||||
$stats = $dbForProject->find('stats', [
|
||||
Query::equal('metric', [$metric['from']]),
|
||||
Query::greaterThan('value', 0)
|
||||
]);
|
||||
|
||||
foreach ($stats as $stat) {
|
||||
$stat->setAttribute('metricV2', $metric['to']);
|
||||
$dbForProject->updateDocument('stats', $stat->getId(), $stat);
|
||||
}
|
||||
|
||||
$collections = $dbForProject->find('database_' . $database->getInternalId(), []);
|
||||
|
||||
foreach ($collections as $collection) {
|
||||
$__metric = [
|
||||
'from' => str_replace(['{databaseId}','{collectionId}'], [$database->getId(), $collection->getId()], 'documents.{databaseId}/{collectionId}.count.total'),
|
||||
'to' => str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], '{databaseInternalId}.{collectionInternalId}.documents')];
|
||||
|
||||
$__stats = $dbForProject->find('stats', [
|
||||
Query::equal('metric', [$__metric['from']]),
|
||||
Query::greaterThan('value', 0)
|
||||
]);
|
||||
|
||||
foreach ($__stats as $__stat) {
|
||||
$__stat->setAttribute('metricV2', $__metric['to']);
|
||||
$dbForProject->updateDocument('stats', $__stat->getId(), $__stat);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* executions
|
||||
*/
|
||||
$functions = $dbForProject->find('functions', []);
|
||||
foreach ($functions as $function) {
|
||||
$metrics = [];
|
||||
$metrics[] = [
|
||||
'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.time'),
|
||||
'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions.compute')
|
||||
];
|
||||
$metrics[] = [
|
||||
'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.total'),
|
||||
'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions')
|
||||
];
|
||||
|
||||
$metrics[] = [
|
||||
'from' => str_replace('{functionId}', $function->getId(), 'builds.{functionId}.compute.total'),
|
||||
'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.builds')
|
||||
];
|
||||
|
||||
$metrics[] = [
|
||||
'from' => str_replace('{functionId}', $function->getId(), 'builds.{functionId}.compute.time'),
|
||||
'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.compute')
|
||||
];
|
||||
foreach ($metrics as $metric) {
|
||||
$stats = $dbForProject->find('stats', [
|
||||
Query::equal('metric', [$metric['from']]),
|
||||
Query::greaterThan('value', 0)
|
||||
]);
|
||||
|
||||
foreach ($stats as $stat) {
|
||||
$stat->setAttribute('metricV2', $metric['to']);
|
||||
$dbForProject->updateDocument('stats', $stat->getId(), $stat);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user