From 9d47280d7847219a02cc0ed72ffabd95f45e34fb Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 9 Dec 2021 11:42:49 +0100 Subject: [PATCH] feat(migratio): add options --- app/tasks/migrate.php | 52 +++- src/Appwrite/Migration/Migration.php | 8 +- src/Appwrite/Migration/Version/V11.php | 357 ++++++++++++++----------- 3 files changed, 245 insertions(+), 172 deletions(-) diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index 61557a4c19..3506226bbb 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -11,7 +11,7 @@ use Appwrite\Database\Adapter\Redis as RedisAdapter; use Appwrite\Migration\Migration; use Utopia\Validator\Text; -Config::load('collections.old', __DIR__.'/../config/collections.old.php'); +Config::load('collections.old', __DIR__ . '/../config/collections.old.php'); $cli ->task('migrate') @@ -23,22 +23,54 @@ $cli Console::exit(1); return; } - + $options = []; if (str_starts_with($version, '0.12.')) { Console::error('WARNING'); Console::warning('Migrating to Version 0.12.x introduces a major breaking change within the Database Service!'); Console::warning('Before migrating, please read about the breaking changes here:'); Console::info('https://appwrite.io/guide-to-db-migration'); $confirm = Console::confirm("If you want to proceed, type 'yes':"); - if($confirm != 'yes') { + if ($confirm != 'yes') { + Console::exit(1); + return; + } + + Console::log(''); + Console::log('Collections'); + Console::log('--------------------'); + Console::warning('Be aware that following actions will happen during the migration:'); + Console::warning('- Nested Document rules will be migrated to String attributes'); + Console::warning('- Numeric rules will be migrated to float attributes'); + Console::info("Do you want to migrate your Database Collections?"); + $options['migrateCollections'] = Console::confirm("Type 'yes' or 'no':"); + + if ($options['migrateCollections'] === 'yes') { + Console::log(''); + Console::log('Documents'); + Console::log('------------------'); + Console::warning('Be aware that following actions will happen during the migration:'); + Console::warning('- Nested Documents will be stored as JSON values'); + Console::warning('- All Numeric values will be converted to float'); + Console::info("Do you want to migrate your Database Documents?"); + $options['migrateDocuments'] = Console::confirm("Type 'yes' or 'no':"); + } else { + $options['migrateDocuments'] = 'no'; + } + + + if ( + !in_array($options['migrateDocuments'], ['yes', 'no']) + || !in_array($options['migrateCollections'], ['yes', 'no']) + ) { + Console::error("You must reply with 'yes' or 'no'!"); Console::exit(1); return; } } - Config::load('collectionsold' , __DIR__.'/../config/collections.old.php'); + Config::load('collectionsold', __DIR__ . '/../config/collections.old.php'); - Console::success('Starting Data Migration to version '.$version); + Console::success('Starting Data Migration to version ' . $version); $db = $register->get('db', true); $cache = $register->get('cache', true); @@ -63,8 +95,8 @@ $cli $projects = [$console]; $count = 0; - $class = 'Appwrite\\Migration\\Version\\'.Migration::$versions[$version]; - $migration = new $class($register->get('db'), $register->get('cache')); + $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; + $migration = new $class($register->get('db'), $register->get('cache'), $options); while ($sum > 0) { foreach ($projects as $project) { @@ -74,7 +106,7 @@ $cli ->execute(); } catch (\Throwable $th) { throw $th; - Console::error('Failed to update project ("'.$project->getId().'") version with error: '.$th->getMessage()); + Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); } } @@ -82,7 +114,7 @@ $cli 'limit' => $limit, 'offset' => $offset, 'filters' => [ - '$collection='.Database::SYSTEM_COLLECTION_PROJECTS, + '$collection=' . Database::SYSTEM_COLLECTION_PROJECTS, ], ]); @@ -91,7 +123,7 @@ $cli $count = $count + $sum; if ($sum > 0) { - Console::log('Fetched '.$count.'/'.$consoleDB->getSum().' projects...'); + Console::log('Fetched ' . $count . '/' . $consoleDB->getSum() . ' projects...'); } } $cache->flushAll(); diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 0e078dee5b..d4b8d5fe01 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -12,6 +12,11 @@ use Utopia\Exception; abstract class Migration { + /** + * @var array + */ + protected array $options; + /** * @var PDO */ @@ -68,8 +73,9 @@ abstract class Migration * * @param PDO $pdo */ - public function __construct(PDO $db, Redis $cache = null) + public function __construct(PDO $db, Redis $cache = null, array $options = []) { + $this->options = $options; $this->db = $db; if(!is_null($cache)) { $this->cache = $cache; diff --git a/src/Appwrite/Migration/Version/V11.php b/src/Appwrite/Migration/Version/V11.php index f03c0eda90..b56b3f5617 100644 --- a/src/Appwrite/Migration/Version/V11.php +++ b/src/Appwrite/Migration/Version/V11.php @@ -9,6 +9,7 @@ use Exception; use PDO; use Redis; use Swoole\Runtime; +use Throwable; use Utopia\Abuse\Adapters\TimeLimit; use Utopia\Audit\Audit; use Utopia\Cache\Cache; @@ -18,6 +19,9 @@ use Utopia\Config\Config; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Exception\Limit; +use Utopia\Database\Exception\Authorization as ExceptionAuthorization; +use Utopia\Database\Exception\Structure; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; @@ -32,9 +36,10 @@ class V11 extends Migration protected array $oldCollections; protected array $newCollections; - public function __construct(PDO $db, Redis $cache = null) + public function __construct(PDO $db, Redis $cache = null, array $options = []) { - parent::__construct($db, $cache); + parent::__construct($db, $cache, $options); + $this->options = array_map(fn($option) => $option === 'yes' ? true : false, $this->options); if (!is_null($cache)) { $cacheAdapter = new Cache(new RedisCache($this->cache)); @@ -151,163 +156,8 @@ class V11 extends Migration $this->dbInternal->createCollection($key, $attributes, $indexes); } - - $sum = $this->limit; - $offset = 0; - - /** - * Migrate external collections for Project - */ - while ($sum >= $this->limit) { - $databaseCollections = $this->oldProjectDB->getCollection([ - 'limit' => $this->limit, - 'offset' => $offset, - 'orderType' => 'DESC', - 'filters' => [ - '$collection=' . OldDatabase::SYSTEM_COLLECTION_COLLECTIONS, - ] - ]); - - $sum = \count($databaseCollections); - Console::log('Migrating Collections: ' . $offset . ' / ' . $this->oldProjectDB->getSum()); - - foreach ($databaseCollections as $oldCollection) { - $id = $oldCollection->getId(); - $permissions = $oldCollection->getPermissions(); - $name = $oldCollection->getAttribute('name'); - $newCollection = $this->dbExternal->getCollection($id); - - if ($newCollection->isEmpty()) { - $this->dbExternal->createCollection($id); - /** - * Migrate permissions - */ - $read = $this->migrateWildcardPermissions($permissions['read'] ?? []); - $write = $this->migrateWildcardPermissions($permissions['write'] ?? []); - - /** - * Suffix collection name with a subsequent number to make it unique if possible. - */ - $suffix = 1; - while ($this->dbInternal->findOne('collections', [ - new Query('name', Query::TYPE_EQUAL, [$name]) - ])) { - $name .= ' - ' . $suffix++; - } - - $this->dbInternal->createDocument('collections', new Document([ - '$id' => $id, - '$read' => $read, - '$write' => $write, - 'permission' => 'document', - 'dateCreated' => time(), - 'dateUpdated' => time(), - 'name' => $name, - 'search' => implode(' ', [$id, $name]), - ])); - } else { - Console::warning('Skipped Collection ' . $newCollection->getId() . ' from ' . $newCollection->getCollection()); - } - - /** - * Migrate collection rules to attributes - */ - $attributes = $this->getCollectionAttributes($oldCollection); - - foreach ($attributes as $attribute) { - try { - $this->dbExternal->createAttribute( - collection: $attribute['$collection'], - id: $attribute['$id'], - type: $attribute['type'], - size: $attribute['size'], - required: $attribute['required'], - default: $attribute['default'], - signed: $attribute['signed'], - array: $attribute['array'], - format: null, - filters: $attribute['filters'] - ); - - $this->dbInternal->createDocument('attributes', new Document([ - '$id' => $attribute['$collection'] . '_' . $attribute['$id'], - 'key' => $attribute['$id'], - 'collectionId' => $attribute['$collection'], - 'type' => $attribute['type'], - 'status' => 'available', - 'size' => $attribute['size'], - 'required' => $attribute['required'], - 'signed' => $attribute['signed'], - 'default' => $attribute['default'], - 'array' => $attribute['array'], - 'format' => null, - 'filters' => $attribute['filters'] - ])); - - Console::log('Created "' . $attribute['$id'] . '" attribute in collection: ' . $name); - } catch (\Throwable $th) { - Console::log($th->getMessage() . ' - (' . $attribute['$id'] . '" attribute in collection ' . $name . ')'); - } - } - - /** - * Migrate all external documents - */ - $sumDocs = $this->limit; - $offsetDocs = 0; - while ($sumDocs >= $this->limit) { - $allDocs = $this->oldProjectDB->getCollection([ - 'limit' => $this->limit, - 'offset' => $offsetDocs, - 'orderType' => 'DESC', - 'filters' => [ - '$collection=' . $id - ] - ]); - - $sumDocs = \count($allDocs); - foreach ($allDocs as $document) { - if (!$this->dbExternal->getDocument($id, $document->getId())->isEmpty()) { - continue; - } - foreach ($document as $key => $attr) { - /** - * Convert nested Document to JSON strings. - */ - if ($document->getAttribute($key) instanceof OldDocument) { - $document[$key] = json_encode($this->fixDocument($attr)->getArrayCopy()); - } - /** - * Convert numeric Attributes to float. - */ - if (is_numeric($attr)) { - $document[$key] = floatval($attr); - } - if (\is_array($attr)) { - foreach ($attr as $index => $child) { - /** - * Convert array of nested Document to array JSON strings. - */ - if ($document->getAttribute($key)[$index] instanceof OldDocument) { - $document[$key][$index] = json_encode($this->fixDocument($child)->getArrayCopy()); - } - /** - * Convert array of numeric Attributes to array float. - */ - if (is_numeric($attr)) { - $document[$key][$index] = floatval($child); // Convert any numeric to float - } - } - } - } - $document = new Document($document->getArrayCopy()); - $document = $this->migratePermissions($document); - $this->dbExternal->createDocument($id, $document); - } - $offsetDocs += $this->limit; - } - } - $offset += $this->limit; + if ($this->options['migrateCollections']) { + $this->migrateExternalCollections(); } } else { Console::log('Skipped console project migration.'); @@ -315,7 +165,7 @@ class V11 extends Migration $sum = $this->limit; $offset = 0; - Authorization::disable(); + /** * Migrate internal documents */ @@ -373,6 +223,192 @@ class V11 extends Migration Console::log('Migrated ' . $sum . ' Documents.'); } + /** + * Migrate external collections for Project + * + * @return void + * @throws Exception + * @throws Throwable + * @throws Limit + * @throws ExceptionAuthorization + * @throws Structure + */ + protected function migrateExternalCollections(): void + { + $sum = $this->limit; + $offset = 0; + + while ($sum >= $this->limit) { + $databaseCollections = $this->oldProjectDB->getCollection([ + 'limit' => $this->limit, + 'offset' => $offset, + 'orderType' => 'DESC', + 'filters' => [ + '$collection=' . OldDatabase::SYSTEM_COLLECTION_COLLECTIONS, + ] + ]); + + $sum = \count($databaseCollections); + Console::log('Migrating Collections: ' . $offset . ' / ' . $this->oldProjectDB->getSum()); + + foreach ($databaseCollections as $oldCollection) { + $id = $oldCollection->getId(); + $permissions = $oldCollection->getPermissions(); + $name = $oldCollection->getAttribute('name'); + $newCollection = $this->dbExternal->getCollection($id); + + if ($newCollection->isEmpty()) { + $this->dbExternal->createCollection($id); + /** + * Migrate permissions + */ + $read = $this->migrateWildcardPermissions($permissions['read'] ?? []); + $write = $this->migrateWildcardPermissions($permissions['write'] ?? []); + + /** + * Suffix collection name with a subsequent number to make it unique if possible. + */ + $suffix = 1; + while ($this->dbInternal->findOne('collections', [ + new Query('name', Query::TYPE_EQUAL, [$name]) + ])) { + $name .= ' - ' . $suffix++; + } + + $this->dbInternal->createDocument('collections', new Document([ + '$id' => $id, + '$read' => $read, + '$write' => $write, + 'permission' => 'document', + 'dateCreated' => time(), + 'dateUpdated' => time(), + 'name' => $name, + 'search' => implode(' ', [$id, $name]), + ])); + } else { + Console::warning('Skipped Collection ' . $newCollection->getId() . ' from ' . $newCollection->getCollection()); + } + + /** + * Migrate collection rules to attributes + */ + $attributes = $this->getCollectionAttributes($oldCollection); + + foreach ($attributes as $attribute) { + try { + $this->dbExternal->createAttribute( + collection: $attribute['$collection'], + id: $attribute['$id'], + type: $attribute['type'], + size: $attribute['size'], + required: $attribute['required'], + default: $attribute['default'], + signed: $attribute['signed'], + array: $attribute['array'], + format: null, + filters: $attribute['filters'] + ); + + $this->dbInternal->createDocument('attributes', new Document([ + '$id' => $attribute['$collection'] . '_' . $attribute['$id'], + 'key' => $attribute['$id'], + 'collectionId' => $attribute['$collection'], + 'type' => $attribute['type'], + 'status' => 'available', + 'size' => $attribute['size'], + 'required' => $attribute['required'], + 'signed' => $attribute['signed'], + 'default' => $attribute['default'], + 'array' => $attribute['array'], + 'format' => null, + 'filters' => $attribute['filters'] + ])); + + Console::log('Created "' . $attribute['$id'] . '" attribute in collection: ' . $name); + } catch (\Throwable $th) { + Console::log($th->getMessage() . ' - (' . $attribute['$id'] . '" attribute in collection ' . $name . ')'); + } + } + if ($this->options['migrateDocuments']) { + $this->migrateExternalDocuments(collection: $id); + } + } + $offset += $this->limit; + } + } + /** + * Migrate all external documents + * + * @return void + * @throws Exception + * @throws Throwable + * @throws ExceptionAuthorization + * @throws Structure + */ + protected function migrateExternalDocuments(string $collection): void + { + $sum = $this->limit; + $offset = 0; + while ($sum >= $this->limit) { + $allDocs = $this->oldProjectDB->getCollection([ + 'limit' => $this->limit, + 'offset' => $offset, + 'orderType' => 'DESC', + 'filters' => [ + '$collection=' . $collection + ] + ]); + + $sum = \count($allDocs); + foreach ($allDocs as $document) { + if (!$this->dbExternal->getDocument($collection, $document->getId())->isEmpty()) { + continue; + } + foreach ($document as $key => $attr) { + /** + * Convert nested Document to JSON strings. + */ + if ($document->getAttribute($key) instanceof OldDocument) { + $document[$key] = json_encode($this->fixDocument($attr)->getArrayCopy()); + } + /** + * Convert numeric Attributes to float. + */ + if (is_numeric($attr)) { + $document[$key] = floatval($attr); + } + if (\is_array($attr)) { + foreach ($attr as $index => $child) { + /** + * Convert array of nested Document to array JSON strings. + */ + if ($document->getAttribute($key)[$index] instanceof OldDocument) { + $document[$key][$index] = json_encode($this->fixDocument($child)->getArrayCopy()); + } + /** + * Convert array of numeric Attributes to array float. + */ + if (is_numeric($attr)) { + $document[$key][$index] = floatval($child); // Convert any numeric to float + } + } + } + } + $document = new Document($document->getArrayCopy()); + $document = $this->migratePermissions($document); + $this->dbExternal->createDocument($collection, $document); + } + $offset += $this->limit; + } + } + + /** + * Migrates single docuemnt. + * + * @param OldDocument $oldDocument + * @return Document + * @throws Exception + */ protected function fixDocument(OldDocument $oldDocument): Document { $document = new Document($oldDocument->getArrayCopy()); @@ -620,7 +656,6 @@ class V11 extends Migration } /** - * * @param Document $document * @return string|null * @throws Exception