diff --git a/.gitmodules b/.gitmodules index c939b9ee2c..dad7c9ed3e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "app/console"] path = app/console url = https://github.com/appwrite/console - branch = 3.2.8 + branch = 3.2.9 diff --git a/Dockerfile b/Dockerfile index 059c499bd9..8721301269 100755 --- a/Dockerfile +++ b/Dockerfile @@ -105,7 +105,8 @@ RUN chmod +x /usr/local/bin/hamster && \ chmod +x /usr/local/bin/delete-orphaned-projects && \ chmod +x /usr/local/bin/clear-card-cache && \ chmod +x /usr/local/bin/calc-users-stats && \ - chmod +x /usr/local/bin/calc-tier-stats + chmod +x /usr/local/bin/calc-tier-stats && \ + chmod +x /usr/local/bin/get-migration-stats # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/app/console b/app/console index 6bf696aeb3..212b742992 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 6bf696aeb329d23e4ebe9d1e71edbd144cf2d58e +Subproject commit 212b7429926d097a31ed71d2410e39c600c56f3b diff --git a/app/init.php b/app/init.php index 4e8e8103c9..4cbf08cf2f 100644 --- a/app/init.php +++ b/app/init.php @@ -767,6 +767,25 @@ $register->set('pools', function () { return $group; }); +$register->set('db', function () { + // This is usually for our workers or CLI commands scope + $dbHost = App::getEnv('_APP_DB_HOST', ''); + $dbPort = App::getEnv('_APP_DB_PORT', ''); + $dbUser = App::getEnv('_APP_DB_USER', ''); + $dbPass = App::getEnv('_APP_DB_PASS', ''); + $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); + + $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true, + )); + + return $pdo; +}); $register->set('influxdb', function () { // Register DB connection diff --git a/bin/get-migration-stats b/bin/get-migration-stats new file mode 100644 index 0000000000..efc1934e15 --- /dev/null +++ b/bin/get-migration-stats @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php get-migration-stats $@ \ No newline at end of file diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 0a2dd2f186..9ee4d9236a 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -195,17 +195,24 @@ abstract class Migration * @return iterable * @throws \Exception */ - public function documentsIterator(string $collectionId): iterable + public function documentsIterator(string $collectionId, $queries = []): iterable { $sum = 0; $nextDocument = null; $collectionCount = $this->projectDB->count($collectionId); + $queries[] = Query::limit($this->limit); do { - $queries = [Query::limit($this->limit)]; if ($nextDocument !== null) { - $queries[] = Query::cursorAfter($nextDocument); + $cursorQueryIndex = \array_search('cursorAfter', \array_map(fn (Query $query) => $query->getMethod(), $queries)); + + if ($cursorQueryIndex !== false) { + $queries[$cursorQueryIndex] = Query::cursorAfter($nextDocument); + } else { + $queries[] = Query::cursorAfter($nextDocument); + } } + $documents = $this->projectDB->find($collectionId, $queries); $count = count($documents); $sum += $count; diff --git a/src/Appwrite/Migration/Version/V19.php b/src/Appwrite/Migration/Version/V19.php index b9ccf3c302..46a0a073a6 100644 --- a/src/Appwrite/Migration/Version/V19.php +++ b/src/Appwrite/Migration/Version/V19.php @@ -10,6 +10,7 @@ use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception; +use Utopia\Database\Query; class V19 extends Migration { @@ -41,6 +42,11 @@ class V19 extends Migration Console::info('Migrating Buckets'); $this->migrateBuckets(); + if ($this->project->getId() !== 'console') { + Console::info('Migrating Enum Attribute Size'); + $this->migrateEnumAttributeSize(); + } + Console::info('Migrating Documents'); $this->forEachDocument([$this, 'fixDocument']); @@ -640,6 +646,22 @@ class V19 extends Migration return $commands; } + private function migrateEnumAttributeSize(): void + { + foreach ( + $this->documentsIterator('attributes', [ + Query::equal('format', ['enum']), + Query::lessThan('size', Database::LENGTH_KEY) + ]) as $attribute + ) { + $attribute->setAttribute('size', Database::LENGTH_KEY); + $this->projectDB->updateDocument('attributes', $attribute->getId(), $attribute); + $databaseInternalId = $attribute->getAttribute('databaseInternalId'); + $collectionInternalId = $attribute->getAttribute('collectionInternalId'); + $this->projectDB->updateAttribute('database_' . $databaseInternalId . '_collection_' . $collectionInternalId, $attribute->getAttribute('key'), size: 255); + } + } + /** * Fix run on each document * diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 28d7046dd1..dc6ddc1a5b 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -19,6 +19,7 @@ use Appwrite\Platform\Tasks\VolumeSync; use Appwrite\Platform\Tasks\CalcTierStats; use Appwrite\Platform\Tasks\Upgrade; use Appwrite\Platform\Tasks\DeleteOrphanedProjects; +use Appwrite\Platform\Tasks\GetMigrationStats; use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments; class Tasks extends Service @@ -44,6 +45,7 @@ class Tasks extends Service ->addAction(CalcTierStats::getName(), new CalcTierStats()) ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects()) ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments()) + ->addAction(GetMigrationStats::getName(), new GetMigrationStats()) ; } diff --git a/src/Appwrite/Platform/Tasks/CalcTierStats.php b/src/Appwrite/Platform/Tasks/CalcTierStats.php index 2a2bc20af9..e6559a05d7 100644 --- a/src/Appwrite/Platform/Tasks/CalcTierStats.php +++ b/src/Appwrite/Platform/Tasks/CalcTierStats.php @@ -41,6 +41,7 @@ class CalcTierStats extends Action 'Functions', 'Deployments', 'Executions', + 'Migrations', ]; protected string $directory = '/usr/local'; @@ -99,8 +100,8 @@ class CalcTierStats extends Action $projects = [$console]; $count = 0; - $limit = 30; - $sum = 30; + $limit = 100; + $sum = 100; $offset = 0; while (!empty($projects)) { foreach ($projects as $project) { @@ -200,7 +201,7 @@ class CalcTierStats extends Action try { /** Get Domains */ - $stats['Domains'] = $dbForConsole->count('domains', [ + $stats['Domains'] = $dbForConsole->count('rules', [ Query::equal('projectInternalId', [$project->getInternalId()]), ]); } catch (\Throwable) { @@ -290,6 +291,13 @@ class CalcTierStats extends Action $stats['Executions'] = 0; } + /** Get Total Migrations */ + try { + $stats['Migrations'] = $dbForProject->count('migrations', []); + } catch (\Throwable) { + $stats['Migrations'] = 0; + } + $csv->insertOne(array_values($stats)); } catch (\Throwable $th) { Console::error('Failed on project ("' . $project->getId() . '") version with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage()); diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php new file mode 100644 index 0000000000..b76e0428d7 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/GetMigrationStats.php @@ -0,0 +1,187 @@ +desc('Get stats for projects') + ->inject('pools') + ->inject('cache') + ->inject('dbForConsole') + ->inject('register') + ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { + $this->action($pools, $cache, $dbForConsole, $register); + }); + } + + /** + * @throws \Utopia\Exception + * @throws CannotInsertRecord + */ + public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void + { + //docker compose exec -t appwrite get-migration-stats + + Console::title('Migration stats calculation V1'); + Console::success(APP_NAME . ' Migration stats calculation has started'); + + /* Initialise new Utopia app */ + $app = new App('UTC'); + $console = $app->getResource('console'); + + /** CSV stuff */ + $this->date = date('Y-m-d'); + $this->path = "{$this->directory}/migration_stats_{$this->date}.csv"; + $csv = Writer::createFromPath($this->path, 'w'); + $csv->insertOne($this->columns); + + /** Database connections */ + $totalProjects = $dbForConsole->count('projects'); + Console::success("Found a total of: {$totalProjects} projects"); + + $projects = [$console]; + $count = 0; + $limit = 100; + $sum = 100; + $offset = 0; + while (!empty($projects)) { + foreach ($projects as $project) { + + /** + * Skip user projects with id 'console' + */ + if ($project->getId() === 'console') { + continue; + } + + Console::info("Getting stats for {$project->getId()}"); + + try { + $db = $project->getAttribute('database'); + $adapter = $pools + ->get($db) + ->pop() + ->getResource(); + + $dbForProject = new Database($adapter, $cache); + $dbForProject->setDefaultDatabase('appwrite'); + $dbForProject->setNamespace('_' . $project->getInternalId()); + + /** Get Project ID */ + $stats['Project ID'] = $project->getId(); + + /** Get Migration details */ + $migrations = $dbForProject->find('migrations', [ + Query::limit(500) + ]); + + $migrations = array_map(function ($migration) use ($project) { + return [ + $project->getId(), + $migration->getAttribute('$id'), + $migration->getAttribute('$createdAt'), + $migration->getAttribute('status'), + $migration->getAttribute('stage'), + $migration->getAttribute('source'), + ]; + }, $migrations); + + if (!empty($migrations)) { + $csv->insertAll($migrations); + } + } catch (\Throwable $th) { + Console::error('Failed on project ("' . $project->getId() . '") with error on File: ' . $th->getFile() . ' line no: ' . $th->getline() . ' with message: ' . $th->getMessage()); + } finally { + $pools + ->get($db) + ->reclaim(); + } + } + + $sum = \count($projects); + + $projects = $dbForConsole->find('projects', [ + Query::limit($limit), + Query::offset($offset), + ]); + + $offset = $offset + $limit; + $count = $count + $sum; + } + + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...'); + + $pools + ->get('console') + ->reclaim(); + + /** @var PHPMailer $mail */ + $mail = $register->get('smtp'); + + $mail->clearAddresses(); + $mail->clearAllRecipients(); + $mail->clearReplyTos(); + $mail->clearAttachments(); + $mail->clearBCCs(); + $mail->clearCCs(); + + try { + /** Addresses */ + $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster'); + $recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', '')); + + foreach ($recipients as $recipient) { + $mail->addAddress($recipient); + } + + /** Attachments */ + $mail->addAttachment($this->path); + + /** Content */ + $mail->Subject = "Migration Report for {$this->date}"; + $mail->Body = "Please find the migration report atttached"; + $mail->send(); + Console::success('Email has been sent!'); + } catch (Exception $e) { + Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}"); + } + } +} diff --git a/src/Appwrite/Platform/Tasks/Hamster.php b/src/Appwrite/Platform/Tasks/Hamster.php index 1d5d3b0b26..2e2c2d466e 100644 --- a/src/Appwrite/Platform/Tasks/Hamster.php +++ b/src/Appwrite/Platform/Tasks/Hamster.php @@ -149,6 +149,9 @@ class Hamster extends Action /** Get Total Teams */ $statsPerProject['custom_teams'] = $dbForProject->count('teams', [], APP_LIMIT_COUNT); + /** Get Total Migrations */ + $statsPerProject['custom_migrations'] = $dbForProject->count('migrations', [], APP_LIMIT_COUNT); + /** Get Total Members */ $teamInternalId = $project->getAttribute('teamInternalId', null); if ($teamInternalId) { diff --git a/src/Appwrite/Platform/Tasks/Migrate.php b/src/Appwrite/Platform/Tasks/Migrate.php index 38e5c8aa46..682b9b5559 100644 --- a/src/Appwrite/Platform/Tasks/Migrate.php +++ b/src/Appwrite/Platform/Tasks/Migrate.php @@ -11,6 +11,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; +use Utopia\Registry\Registry; use Utopia\Validator\Text; class Migrate extends Action @@ -29,7 +30,8 @@ class Migrate extends Action ->inject('cache') ->inject('dbForConsole') ->inject('getProjectDB') - ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB) => $this->action($version, $cache, $dbForConsole, $getProjectDB)); + ->inject('register') + ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB, Registry $register) => $this->action($version, $cache, $dbForConsole, $getProjectDB, $register)); } private function clearProjectsCache(Cache $cache, Document $project) @@ -41,7 +43,7 @@ class Migrate extends Action } } - public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB) + public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB, Registry $register) { Authorization::disable(); if (!array_key_exists($version, Migration::$versions)) { @@ -89,9 +91,11 @@ class Migrate extends Action try { // TODO: Iterate through all project DBs + /** @var Database $projectDB */ $projectDB = $getProjectDB($project); $migration ->setProject($project, $projectDB, $dbForConsole) + ->setPDO($register->get('db', true)) ->execute(); } catch (\Throwable $th) { Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());