diff --git a/Dockerfile b/Dockerfile index abe9b7128f..9569561a6b 100755 --- a/Dockerfile +++ b/Dockerfile @@ -138,7 +138,8 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-builds && \ chmod +x /usr/local/bin/worker-mails && \ chmod +x /usr/local/bin/worker-messaging && \ - chmod +x /usr/local/bin/worker-webhooks + chmod +x /usr/local/bin/worker-webhooks && \ + chmod +x /usr/local/bin/migrate-usage # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/app/config/collections.php b/app/config/collections.php index afde813fa1..7e423040bb 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -3104,6 +3104,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('metricV2'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => ID::custom('region'), 'type' => Database::VAR_STRING, diff --git a/app/console b/app/console index cad6f3b1bf..fae048b917 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit cad6f3b1bfdae4d423ba6f0735ba2a5cd5a58551 +Subproject commit fae048b91787d0b372c13caf27f14fc8b780ef60 diff --git a/composer.lock b/composer.lock index 4c23baa2b5..e9da115e05 100644 --- a/composer.lock +++ b/composer.lock @@ -620,16 +620,16 @@ }, { "name": "guzzlehttp/psr7", - "version": "2.4.3", + "version": "2.4.4", "source": { "type": "git", "url": "https://github.com/guzzle/psr7.git", - "reference": "67c26b443f348a51926030c83481b85718457d3d" + "reference": "3cf1b6d4f0c820a2cf8bcaec39fc698f3443b5cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/psr7/zipball/67c26b443f348a51926030c83481b85718457d3d", - "reference": "67c26b443f348a51926030c83481b85718457d3d", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/3cf1b6d4f0c820a2cf8bcaec39fc698f3443b5cf", + "reference": "3cf1b6d4f0c820a2cf8bcaec39fc698f3443b5cf", "shasum": "" }, "require": { @@ -719,7 +719,7 @@ ], "support": { "issues": "https://github.com/guzzle/psr7/issues", - "source": "https://github.com/guzzle/psr7/tree/2.4.3" + "source": "https://github.com/guzzle/psr7/tree/2.4.4" }, "funding": [ { @@ -735,7 +735,7 @@ "type": "tidelift" } ], - "time": "2022-10-26T14:07:24+00:00" + "time": "2023-03-09T13:19:02+00:00" }, { "name": "influxdb/influxdb-php", @@ -872,34 +872,37 @@ }, { "name": "league/csv", - "version": "9.8.0", + "version": "9.9.0", "source": { "type": "git", "url": "https://github.com/thephpleague/csv.git", - "reference": "9d2e0265c5d90f5dd601bc65ff717e05cec19b47" + "reference": "b4418ede47fbd88facc34e40a16c8ce9153b961b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/csv/zipball/9d2e0265c5d90f5dd601bc65ff717e05cec19b47", - "reference": "9d2e0265c5d90f5dd601bc65ff717e05cec19b47", + "url": "https://api.github.com/repos/thephpleague/csv/zipball/b4418ede47fbd88facc34e40a16c8ce9153b961b", + "reference": "b4418ede47fbd88facc34e40a16c8ce9153b961b", "shasum": "" }, "require": { "ext-json": "*", "ext-mbstring": "*", - "php": "^7.4 || ^8.0" + "php": "^8.1.2" }, "require-dev": { - "ext-curl": "*", + "doctrine/collections": "^2.1.2", "ext-dom": "*", - "friendsofphp/php-cs-fixer": "^v3.4.0", - "phpstan/phpstan": "^1.3.0", - "phpstan/phpstan-phpunit": "^1.0.0", - "phpstan/phpstan-strict-rules": "^1.1.0", - "phpunit/phpunit": "^9.5.11" + "ext-xdebug": "*", + "friendsofphp/php-cs-fixer": "^v3.14.3", + "phpbench/phpbench": "^1.2.8", + "phpstan/phpstan": "^1.10.4", + "phpstan/phpstan-deprecation-rules": "^1.1.2", + "phpstan/phpstan-phpunit": "^1.3.10", + "phpstan/phpstan-strict-rules": "^1.5.0", + "phpunit/phpunit": "^10.0.14" }, "suggest": { - "ext-dom": "Required to use the XMLConverter and or the HTMLConverter classes", + "ext-dom": "Required to use the XMLConverter and the HTMLConverter classes", "ext-iconv": "Needed to ease transcoding CSV using iconv stream filters" }, "type": "library", @@ -952,7 +955,7 @@ "type": "github" } ], - "time": "2022-01-04T00:13:07+00:00" + "time": "2023-03-11T15:57:12+00:00" }, { "name": "matomo/device-detector", @@ -5473,5 +5476,5 @@ "platform-overrides": { "php": "8.0" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 00cf3f89c5..9563bc3a2b 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Services; +use Appwrite\Platform\Tasks\MigrateUsage; use Utopia\Platform\Service; use Appwrite\Platform\Tasks\Doctor; use Appwrite\Platform\Tasks\Install; @@ -37,6 +38,7 @@ class Tasks extends Service ->addAction(Migrate::getName(), new Migrate()) ->addAction(SDKs::getName(), new SDKs()) ->addAction(VolumeSync::getName(), new VolumeSync()) - ->addAction(Specs::getName(), new Specs()); + ->addAction(Specs::getName(), new Specs()) + ->addAction(MigrateUsage::getName(), new MigrateUsage()); } } diff --git a/src/Appwrite/Platform/Tasks/MigrateUsage.php b/src/Appwrite/Platform/Tasks/MigrateUsage.php new file mode 100644 index 0000000000..a78a45ad52 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/MigrateUsage.php @@ -0,0 +1,171 @@ +desc('Migrate old usage metric pattern to new') + ->param('structure', false, new Boolean(), 'Manipulate usage cols structure', true) + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn(bool $structure, Database $dbForConsole, callable $getProjectDB) => $this->action($structure, $dbForConsole, $getProjectDB)); + } + + /** + * Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule. + */ + public function action(bool $structure, Database $dbForConsole, callable $getProjectDB): void + { + Authorization::disable(); + Authorization::setDefaultStatus(false); + + + $limit = 100; + $projectCursor = null; + while (true) { + $projectsQueries = [Query::limit($limit)]; + if ($projectCursor !== null) { + $projectsQueries[] = Query::cursorAfter($projectCursor); + } + $projects = $dbForConsole->find('projects', $projectsQueries); + + if (count($projects) === 0) { + break; + } + + foreach ($projects as $project) { + Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getInternalId() . ")"); + $dbForProject = $getProjectDB($project); + + if ($structure) { + try { + /** + * Update 'mimeType' attribute size (127->255) + */ + $dbForProject->renameAttribute('stats', 'metric', 'metricOld'); + $dbForProject->renameAttribute('stats', 'metricV2', 'metric'); + } catch (\Throwable $th) { + Console::Error("Error while trying yo rename col: {$th->getMessage()}"); + } + + Console::log("Renaming stats col metric to metricOld, metricV2 to metric"); + continue; + } + + + /** + * Project level + */ + $metrics[] = ['from' => 'project.$all.network.outbound', 'to' => 'network.outbound']; + $metrics[] = ['from' => 'project.$all.network.inbound', 'to' => 'network.inbound']; + $metrics[] = ['from' => 'project.$all.network.requests', 'to' => 'network.requests']; + $metrics[] = ['from' => 'executions.$all.compute.total', 'to' => 'executions']; + $metrics[] = ['from' => 'builds.$all.compute.time', 'to' => 'builds.compute']; + $metrics[] = ['from' => 'files.$all.storage.size', 'to' => 'files.storage']; + $metrics[] = ['from' => 'files.$all.count.total', 'to' => 'files']; + $metrics[] = ['from' => 'buckets.$all.count.total', 'to' => 'buckets']; + $metrics[] = ['from' => 'collections.$all.count.total', 'to' => 'collections']; + $metrics[] = ['from' => 'databases.$all.count.total', 'to' => 'databases']; + $metrics[] = ['from' => 'users.$all.count.total ', 'to' => 'users']; + $metrics[] = ['from' => 'documents.$all.count.total', 'to' => 'documents']; + $metrics[] = ['from' => 'documents.$all.count.total', 'to' => 'documents']; + + foreach ($metrics as $metric) { + $stats = $dbForProject->find('stats', [ + Query::equal('metric', [$metric['from']]), + Query::greaterThan('value', 0) + ]); + foreach ($stats as $stat) { + $stat->setAttribute('metricV2', $metric['to']); + $dbForProject->updateDocument('stats', $stat->getId(), $stat); + } + } + + /** + * Databases, collections, documents + */ + $databases = $dbForProject->find('databases', []); + foreach ($databases as $database) { + $metrics = []; + $metrics[] = [ + 'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'), + 'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.collections')]; + $metrics[] = [ + 'from' => str_replace('{databaseId}', $database->getId(), 'documents.{databaseId}.count.total'), + 'to' => str_replace('{databaseInternalId}', $database->getInternalId(), '{databaseInternalId}.documents')]; + foreach ($metrics as $metric) { + $stats = $dbForProject->find('stats', [ + Query::equal('metric', [$metric['from']]), + Query::greaterThan('value', 0) + ]); + + foreach ($stats as $stat) { + $stat->setAttribute('metricV2', $metric['to']); + $dbForProject->updateDocument('stats', $stat->getId(), $stat); + } + + $collections = $dbForProject->find('database_' . $database->getInternalId(), []); + + foreach ($collections as $collection) { + $__metric = [ + 'from' => str_replace(['{databaseId}','{collectionId}'], [$database->getId(), $collection->getId()], 'documents.{databaseId}/{collectionId}.count.total'), + 'to' => str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], '{databaseInternalId}.{collectionInternalId}.documents')]; + + $__stats = $dbForProject->find('stats', [ + Query::equal('metric', [$__metric['from']]), + Query::greaterThan('value', 0) + ]); + + foreach ($__stats as $__stat) { + $__stat->setAttribute('metricV2', $__metric['to']); + $dbForProject->updateDocument('stats', $__stat->getId(), $__stat); + } + } + } + } + + /** + * executions + */ + $functions = $dbForProject->find('functions', []); + foreach ($functions as $function) { + $metrics = []; + $metrics[] = [ + 'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.time'), + 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions.compute')]; + $metrics[] = [ + 'from' => str_replace('{functionId}', $function->getId(), 'executions.{functionId}.compute.total'), + 'to' => str_replace('{functionInternalId}', $function->getInternalId(), '{functionInternalId}.executions')]; + foreach ($metrics as $metric) { + $stats = $dbForProject->find('stats', [ + Query::equal('metric', [$metric['from']]), + Query::greaterThan('value', 0) + ]); + + foreach ($stats as $stat) { + $stat->setAttribute('metricV2', $metric['to']); + $dbForProject->updateDocument('stats', $stat->getId(), $stat); + } + } + } + $projectCursor = $projects[array_key_last($projects)]; + } + } + } +}