From ccff540e70f2e572b0def7163d4d0e4d496bbf5a Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 17 Feb 2022 19:24:50 +0100 Subject: [PATCH] feat: migration --- app/tasks/migrate.php | 8 +- src/Appwrite/Migration/Migration.php | 28 +++- src/Appwrite/Migration/Version/V12.php | 170 +++++++++++++++++++++++++ 3 files changed, 200 insertions(+), 6 deletions(-) diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index 416f936d4c..ff0705eb32 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -46,7 +46,13 @@ $cli $offset = 0; $projects = [$console]; $count = 0; - $totalProjects = $consoleDB->count('projects') + 1; + + try { + $totalProjects = $consoleDB->count('projects') + 1; + } catch (\Throwable $th) { + $consoleDB->setNamespace('_console'); + $totalProjects = $consoleDB->count('projects') + 1; + } $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; $migration = new $class(); diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index d00318f212..19dba9a4fa 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -38,6 +38,26 @@ abstract class Migration '0.13.0' => 'V12', ]; + /** + * @var array + */ + protected array $collections; + + public function __construct() + { + $this->collections = array_merge([ + '_metadata' => [ + '$id' => '_metadata' + ], + 'audit' => [ + '$id' => 'audit' + ], + 'abuse' => [ + '$id' => 'abuse' + ] + ], Config::getParam('collections', [])); + } + /** * Set project for migration. * @@ -51,7 +71,7 @@ abstract class Migration { $this->project = $project; $this->projectDB = $projectDB; - $this->projectDB->setNamespace('_project_' . $this->project->getId()); + $this->projectDB->setNamespace('_' . $this->project->getId()); $this->consoleDB = $consoleDB; @@ -67,10 +87,7 @@ abstract class Migration { Runtime::enableCoroutine(SWOOLE_HOOK_ALL); - /** @var array $collections */ - $collections = Config::getParam('collections', []); - - foreach ($collections as $collection) { + foreach ($this->collections as $collection) { $sum = 0; $nextDocument = null; $collectionCount = $this->projectDB->count($collection['$id']); @@ -114,6 +131,7 @@ abstract class Migration try { $new = $this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document); } catch (\Throwable $th) { + var_dump($th->getTraceAsString()); Console::error('Failed to update document: ' . $th->getMessage()); return; diff --git a/src/Appwrite/Migration/Version/V12.php b/src/Appwrite/Migration/Version/V12.php index 9b9376abac..5835f66dac 100644 --- a/src/Appwrite/Migration/Version/V12.php +++ b/src/Appwrite/Migration/Version/V12.php @@ -4,17 +4,178 @@ namespace Appwrite\Migration\Version; use Appwrite\Migration\Migration; use Utopia\CLI\Console; +use Utopia\Database\Database; use Utopia\Database\Document; class V12 extends Migration { + /** + * @var \PDO $pdo + */ + private $pdo; + public function execute(): void { + global $register; Console::log('Migrating project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')'); + $this->pdo = $register->get('db'); + + Console::info('Migrating Project Schemas'); + $this->migrateProjectSchema($this->project->getId()); + + /** + * Switch to migrated Console Project + */ + if ($this->project->getId() === 'console') { + $this->consoleDB->setNamespace('_console'); + $this->projectDB->setNamespace('_console'); + } + + Console::info('Migrating Permissions'); + $this->fixPermissions(); + Console::info('Migrating Collections'); + $this->fixCollections(); + Console::info('Migrating Documents'); $this->forEachDocument([$this, 'fixDocument']); } + /** + * Migrate Project Tables. + * + * @param string $projectId + * @return void + * @throws \Exception + * @throws \PDOException + */ + private function migrateProjectSchema(string $projectId): void + { + /** + * Remove empty generated Console Project. + */ + if ($this->projectDB->getNamespace() === '_project_console') { + $all = []; + foreach ($this->collections as $collection) { + $all[] = "_{$projectId}_{$collection['$id']}"; + $all[] = "_{$projectId}_{$collection['$id']}_perms"; + } + $this->pdo->prepare('DROP TABLE IF EXISTS ' . implode(', ', $all) . ';')->execute(); + } elseif ($this->projectDB->getNamespace() === '_console') { + return; + } + + /** + * Rename Database Tables. + */ + foreach ($this->collections as $collection) { + $id = $collection['$id']; + + $this->pdo->prepare("ALTER TABLE IF EXISTS _project_{$projectId}_{$id} RENAME TO _{$projectId}_{$id}")->execute(); + $this->pdo->prepare("CREATE TABLE IF NOT EXISTS _{$projectId}_{$id}_perms ( + `_id` int(11) unsigned NOT NULL AUTO_INCREMENT, + `_type` VARCHAR(12) NOT NULL, + `_permission` VARCHAR(255) NOT NULL, + `_document` VARCHAR(255) NOT NULL, + PRIMARY KEY (`_id`), + UNIQUE INDEX `_index1` (`_type`,`_document`,`_permission`), + INDEX `_index2` (`_permission`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;")->execute(); + } + } + + /** + * Migrate all Collection Structure. + * + * @return void + */ + protected function fixCollections(): void + { + foreach ($this->collections as $collection) { + $id = $collection['$id']; + Console::log("- {$id}"); + switch ($id) { + case 'sessions': + try { + $this->projectDB->renameAttribute($id, 'providerToken', 'providerAccessToken'); + } catch (\Throwable $th) { + Console::warning("'providerAccessToken' from {$id}: {$th->getMessage()}"); + } + try { + $this->projectDB->createAttribute(collection: $id, id: 'providerRefreshToken', type: Database::VAR_STRING, size: 16384, signed: true, required: true, filters: ['encrypt']); + } catch (\Throwable $th) { + Console::warning("'providerRefreshToken' from {$id}: {$th->getMessage()}"); + } + try { + $this->projectDB->createAttribute(collection: $id, id: 'providerAccessTokenExpiry', type: Database::VAR_INTEGER, size: 0, required: true); + } catch (\Throwable $th) { + Console::warning("'providerAccessTokenExpiry' from {$id}: {$th->getMessage()}"); + } + break; + } + usleep(100000); + } + } + + /** + * Migrate all Permission to new System with dedicated Table. + * @return void + * @throws \Exception + */ + protected function fixPermissions() + { + foreach ($this->collections as $collection) { + $id = $collection['$id']; + Console::log("- {$collection['$id']}"); + $nextDocument = null; + + do { + $documents = $this->projectDB->find($id, limit: $this->limit, cursor: $nextDocument); + $count = count($documents); + + \Co\run(function (array $documents) { + foreach ($documents as $document) { + go(function (Document $document) { + $sql = "SELECT _read, _write FROM `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$document->getCollection()}` WHERE _uid = {$this->pdo->quote($document->getid())}"; + $stmt = $this->pdo->prepare($sql); + $stmt->execute(); + + $permissions = $stmt->fetch(); + + $read = json_decode($permissions['_read'] ?? null) ?? []; + $write = json_decode($permissions['_write'] ?? null) ?? []; + + $permissions = []; + foreach ($read as $permission) { + $permissions[] = "('read', '{$permission}', '{$document->getId()}')"; + } + + foreach ($write as $permission) { + $permissions[] = "('write', '{$permission}', '{$document->getId()}')"; + } + + if (!empty($permissions)) { + $queryPermissions = "INSERT IGNORE INTO `{$this->projectDB->getDefaultDatabase()}`.`{$this->projectDB->getNamespace()}_{$document->getCollection()}_perms` (_type, _permission, _document) VALUES " . implode(', ', $permissions); + $stmtPermissions = $this->pdo->prepare($queryPermissions); + $stmtPermissions->execute(); + } + }, $document); + } + }, $documents); + + if ($count !== $this->limit) { + $nextDocument = null; + } else { + $nextDocument = end($documents); + } + } while (!is_null($nextDocument)); + } + + /** + * Timeout to give MariaDB some room to breath + */ + usleep(100000); + } + protected function fixDocument(Document $document) { switch ($document->getCollection()) { @@ -92,6 +253,15 @@ class V12 extends Migration } break; + + case 'sessions': + $document + ->setAttribute('providerRefreshToken', '') + ->setAttribute('providerAccessTokenExpiry', 0) + ->setAttribute('providerAccessToken', $document->getAttribute('providerToken', '')) + ->removeAttribute('providerToken'); + + break; } return $document;