From 34e80acb7fda4837bd52ed3734d9ce9a0b26ae86 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 8 Dec 2021 18:49:44 +0100 Subject: [PATCH 01/13] feat(migration): version 0.12.x --- app/config/collections.old.php | 4 ++-- app/config/collections.php | 20 ++++++++++---------- app/http.php | 4 ---- app/init.php | 6 +++--- app/tasks/migrate.php | 28 ++++++++++++++++++++++------ 5 files changed, 37 insertions(+), 25 deletions(-) diff --git a/app/config/collections.old.php b/app/config/collections.old.php index e72967f0d8..93faacad66 100644 --- a/app/config/collections.old.php +++ b/app/config/collections.old.php @@ -1617,8 +1617,8 @@ foreach ($providers as $index => $provider) { foreach ($auth as $index => $method) { $collections[Database::SYSTEM_COLLECTION_PROJECTS]['rules'][] = [ '$collection' => Database::SYSTEM_COLLECTION_RULES, - 'label' => $method['name'] || '', - 'key' => $method['key'] || '', + 'label' => $method['name'] ?? '', + 'key' => $method['key'] ?? '', 'type' => Database::SYSTEM_VAR_TYPE_BOOLEAN, 'default' => true, 'required' => false, diff --git a/app/config/collections.php b/app/config/collections.php index 4122cd9f06..db3bc22e28 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -476,7 +476,7 @@ $collections = [ 'size' => 16384, 'signed' => true, 'required' => false, - 'default' => null, + 'default' => [], 'array' => false, 'filters' => ['json'], ], @@ -487,7 +487,7 @@ $collections = [ 'size' => 16384, 'signed' => true, 'required' => false, - 'default' => null, + 'default' => [], 'array' => false, 'filters' => ['json'], ], @@ -498,7 +498,7 @@ $collections = [ 'size' => 16384, 'signed' => true, 'required' => false, - 'default' => null, + 'default' => [], 'array' => false, 'filters' => ['json'], ], @@ -873,7 +873,7 @@ $collections = [ 'format' => '', 'size' => Database::LENGTH_KEY, 'signed' => true, - 'required' => true, + 'required' => false, 'default' => null, 'array' => false, 'filters' => [], @@ -884,7 +884,7 @@ $collections = [ 'format' => '', 'size' => Database::LENGTH_KEY, 'signed' => true, - 'required' => true, + 'required' => false, 'default' => null, 'array' => false, 'filters' => [], @@ -990,7 +990,7 @@ $collections = [ 'size' => 16384, 'signed' => true, 'required' => false, - 'default' => null, + 'default' => [], 'array' => false, 'filters' => ['json'], ], @@ -1034,7 +1034,7 @@ $collections = [ 'size' => 16384, 'signed' => true, 'required' => false, - 'default' => null, + 'default' => [], 'array' => true, 'filters' => ['json'], ], @@ -1045,7 +1045,7 @@ $collections = [ 'size' => 16384, 'signed' => true, 'required' => false, - 'default' => null, + 'default' => [], 'array' => true, 'filters' => ['json'], ], @@ -1056,7 +1056,7 @@ $collections = [ 'size' => 16384, 'signed' => true, 'required' => false, - 'default' => null, + 'default' => [], 'array' => true, 'filters' => ['json'], ], @@ -1805,7 +1805,7 @@ $collections = [ 'size' => 8192, 'signed' => true, 'required' => false, - 'default' => null, + 'default' => [], 'array' => false, 'filters' => ['json', 'encrypt'], ], diff --git a/app/http.php b/app/http.php index 28b070d3a4..3b0ec8a8fe 100644 --- a/app/http.php +++ b/app/http.php @@ -81,10 +81,6 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { return $redis; }); - App::setResource('app', function() use (&$app) { - return $app; - }); - $dbForConsole = $app->getResource('dbForConsole'); /** @var Utopia\Database\Database $dbForConsole */ if(!$dbForConsole->exists()) { diff --git a/app/init.php b/app/init.php index 6992728987..92d05d4650 100644 --- a/app/init.php +++ b/app/init.php @@ -176,7 +176,7 @@ DatabaseOld::addFilter('encrypt', $key = App::getEnv('_APP_OPENSSL_KEY_V1'); $iv = OpenSSL::randomPseudoBytes(OpenSSL::cipherIVLength(OpenSSL::CIPHER_AES_128_GCM)); $tag = null; - + return json_encode([ 'data' => OpenSSL::encrypt($value, OpenSSL::CIPHER_AES_128_GCM, $key, 0, $iv, $tag), 'method' => OpenSSL::CIPHER_AES_128_GCM, @@ -216,7 +216,7 @@ Database::addFilter('enum', return $value; }, function($value, Document $attribute) { - $formatOptions = json_decode($attribute->getAttribute('formatOptions', []), true); + $formatOptions = json_decode($attribute->getAttribute('formatOptions', '[]'), true); if (isset($formatOptions['elements'])) { $attribute->setAttribute('elements', $formatOptions['elements']); } @@ -235,7 +235,7 @@ Database::addFilter('range', return $value; }, function($value, Document $attribute) { - $formatOptions = json_decode($attribute->getAttribute('formatOptions', []), true); + $formatOptions = json_decode($attribute->getAttribute('formatOptions', '[]'), true); if (isset($formatOptions['min']) || isset($formatOptions['max'])) { $attribute ->setAttribute('min', $formatOptions['min']) diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index 22ce22af13..61557a4c19 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -17,31 +17,46 @@ $cli ->task('migrate') ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true) ->action(function ($version) use ($register) { + Authorization::disable(); if (!array_key_exists($version, Migration::$versions)) { Console::error("Version {$version} not found."); Console::exit(1); return; } + if (str_starts_with($version, '0.12.')) { + Console::error('WARNING'); + Console::warning('Migrating to Version 0.12.x introduces a major breaking change within the Database Service!'); + Console::warning('Before migrating, please read about the breaking changes here:'); + Console::info('https://appwrite.io/guide-to-db-migration'); + $confirm = Console::confirm("If you want to proceed, type 'yes':"); + if($confirm != 'yes') { + Console::exit(1); + return; + } + } + + Config::load('collectionsold' , __DIR__.'/../config/collections.old.php'); + Console::success('Starting Data Migration to version '.$version); + $db = $register->get('db', true); $cache = $register->get('cache', true); + $cache->flushAll(); $consoleDB = new Database(); $consoleDB ->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache)) ->setNamespace('app_console') // Main DB - ->setMocks(Config::getParam('collections.old', [])); + ->setMocks(Config::getParam('collectionsold', [])); $projectDB = new Database(); $projectDB ->setAdapter(new RedisAdapter(new MySQLAdapter($db, $cache), $cache)) - ->setMocks(Config::getParam('collections.old', [])); + ->setMocks(Config::getParam('collectionsold', [])); $console = $consoleDB->getDocument('console'); - Authorization::disable(); - $limit = 30; $sum = 30; $offset = 0; @@ -49,13 +64,13 @@ $cli $count = 0; $class = 'Appwrite\\Migration\\Version\\'.Migration::$versions[$version]; - $migration = new $class($register->get('db')); + $migration = new $class($register->get('db'), $register->get('cache')); while ($sum > 0) { foreach ($projects as $project) { try { $migration - ->setProject($project, $projectDB) + ->setProject($project, $projectDB, $consoleDB) ->execute(); } catch (\Throwable $th) { throw $th; @@ -79,6 +94,7 @@ $cli Console::log('Fetched '.$count.'/'.$consoleDB->getSum().' projects...'); } } + $cache->flushAll(); Console::success('Data Migration Completed'); }); From 3f92e6991a28255ea0ad8f4d6d370ec947d1af44 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 8 Dec 2021 18:50:04 +0100 Subject: [PATCH 02/13] leftover commits --- composer.lock | 262 ++++++---- src/Appwrite/Database/Document.php | 22 +- src/Appwrite/Migration/Migration.php | 62 ++- src/Appwrite/Migration/Version/V11.php | 639 +++++++++++++++++++++++++ 4 files changed, 875 insertions(+), 110 deletions(-) create mode 100644 src/Appwrite/Migration/Version/V11.php diff --git a/composer.lock b/composer.lock index 82628d1214..b541ac91de 100644 --- a/composer.lock +++ b/composer.lock @@ -489,16 +489,16 @@ }, { "name": "guzzlehttp/guzzle", - "version": "7.4.0", + "version": "7.4.1", "source": { "type": "git", "url": "https://github.com/guzzle/guzzle.git", - "reference": "868b3571a039f0ebc11ac8f344f4080babe2cb94" + "reference": "ee0a041b1760e6a53d2a39c8c34115adc2af2c79" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/guzzle/zipball/868b3571a039f0ebc11ac8f344f4080babe2cb94", - "reference": "868b3571a039f0ebc11ac8f344f4080babe2cb94", + "url": "https://api.github.com/repos/guzzle/guzzle/zipball/ee0a041b1760e6a53d2a39c8c34115adc2af2c79", + "reference": "ee0a041b1760e6a53d2a39c8c34115adc2af2c79", "shasum": "" }, "require": { @@ -507,7 +507,7 @@ "guzzlehttp/psr7": "^1.8.3 || ^2.1", "php": "^7.2.5 || ^8.0", "psr/http-client": "^1.0", - "symfony/deprecation-contracts": "^2.2" + "symfony/deprecation-contracts": "^2.2 || ^3.0" }, "provide": { "psr/http-client-implementation": "1.0" @@ -593,7 +593,7 @@ ], "support": { "issues": "https://github.com/guzzle/guzzle/issues", - "source": "https://github.com/guzzle/guzzle/tree/7.4.0" + "source": "https://github.com/guzzle/guzzle/tree/7.4.1" }, "funding": [ { @@ -609,7 +609,7 @@ "type": "tidelift" } ], - "time": "2021-10-18T09:52:00+00:00" + "time": "2021-12-06T18:43:05+00:00" }, { "name": "guzzlehttp/promises", @@ -1591,25 +1591,25 @@ }, { "name": "symfony/deprecation-contracts", - "version": "v2.5.0", + "version": "v3.0.0", "source": { "type": "git", "url": "https://github.com/symfony/deprecation-contracts.git", - "reference": "6f981ee24cf69ee7ce9736146d1c57c2780598a8" + "reference": "c726b64c1ccfe2896cb7df2e1331c357ad1c8ced" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/6f981ee24cf69ee7ce9736146d1c57c2780598a8", - "reference": "6f981ee24cf69ee7ce9736146d1c57c2780598a8", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/c726b64c1ccfe2896cb7df2e1331c357ad1c8ced", + "reference": "c726b64c1ccfe2896cb7df2e1331c357ad1c8ced", "shasum": "" }, "require": { - "php": ">=7.1" + "php": ">=8.0.2" }, "type": "library", "extra": { "branch-alias": { - "dev-main": "2.5-dev" + "dev-main": "3.0-dev" }, "thanks": { "name": "symfony/contracts", @@ -1638,7 +1638,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v2.5.0" + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.0.0" }, "funding": [ { @@ -1654,7 +1654,7 @@ "type": "tidelift" } ], - "time": "2021-07-12T14:48:14+00:00" + "time": "2021-11-01T23:48:49+00:00" }, { "name": "symfony/polyfill-ctype", @@ -3061,6 +3061,77 @@ }, "time": "2021-11-12T11:09:38+00:00" }, + { + "name": "composer/pcre", + "version": "1.0.0", + "source": { + "type": "git", + "url": "https://github.com/composer/pcre.git", + "reference": "3d322d715c43a1ac36c7fe215fa59336265500f2" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/composer/pcre/zipball/3d322d715c43a1ac36c7fe215fa59336265500f2", + "reference": "3d322d715c43a1ac36c7fe215fa59336265500f2", + "shasum": "" + }, + "require": { + "php": "^5.3.2 || ^7.0 || ^8.0" + }, + "require-dev": { + "phpstan/phpstan": "^1", + "phpstan/phpstan-strict-rules": "^1.1", + "symfony/phpunit-bridge": "^4.2 || ^5" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "1.x-dev" + } + }, + "autoload": { + "psr-4": { + "Composer\\Pcre\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jordi Boggiano", + "email": "j.boggiano@seld.be", + "homepage": "http://seld.be" + } + ], + "description": "PCRE wrapping library that offers type-safe preg_* replacements.", + "keywords": [ + "PCRE", + "preg", + "regex", + "regular expression" + ], + "support": { + "issues": "https://github.com/composer/pcre/issues", + "source": "https://github.com/composer/pcre/tree/1.0.0" + }, + "funding": [ + { + "url": "https://packagist.com", + "type": "custom" + }, + { + "url": "https://github.com/composer", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/composer/composer", + "type": "tidelift" + } + ], + "time": "2021-12-06T15:17:27+00:00" + }, { "name": "composer/semver", "version": "3.2.6", @@ -3144,25 +3215,27 @@ }, { "name": "composer/xdebug-handler", - "version": "2.0.2", + "version": "2.0.3", "source": { "type": "git", "url": "https://github.com/composer/xdebug-handler.git", - "reference": "84674dd3a7575ba617f5a76d7e9e29a7d3891339" + "reference": "6555461e76962fd0379c444c46fd558a0fcfb65e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/xdebug-handler/zipball/84674dd3a7575ba617f5a76d7e9e29a7d3891339", - "reference": "84674dd3a7575ba617f5a76d7e9e29a7d3891339", + "url": "https://api.github.com/repos/composer/xdebug-handler/zipball/6555461e76962fd0379c444c46fd558a0fcfb65e", + "reference": "6555461e76962fd0379c444c46fd558a0fcfb65e", "shasum": "" }, "require": { + "composer/pcre": "^1", "php": "^5.3.2 || ^7.0 || ^8.0", "psr/log": "^1 || ^2 || ^3" }, "require-dev": { - "phpstan/phpstan": "^0.12.55", - "symfony/phpunit-bridge": "^4.2 || ^5" + "phpstan/phpstan": "^1.0", + "phpstan/phpstan-strict-rules": "^1.1", + "symfony/phpunit-bridge": "^4.2 || ^5.0 || ^6.0" }, "type": "library", "autoload": { @@ -3188,7 +3261,7 @@ "support": { "irc": "irc://irc.freenode.org/composer", "issues": "https://github.com/composer/xdebug-handler/issues", - "source": "https://github.com/composer/xdebug-handler/tree/2.0.2" + "source": "https://github.com/composer/xdebug-handler/tree/2.0.3" }, "funding": [ { @@ -3204,7 +3277,7 @@ "type": "tidelift" } ], - "time": "2021-07-31T17:03:58+00:00" + "time": "2021-12-08T13:07:32+00:00" }, { "name": "dnoegel/php-xdg-base-dir", @@ -3655,16 +3728,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.13.1", + "version": "v4.13.2", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "63a79e8daa781cac14e5195e63ed8ae231dd10fd" + "reference": "210577fe3cf7badcc5814d99455df46564f3c077" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/63a79e8daa781cac14e5195e63ed8ae231dd10fd", - "reference": "63a79e8daa781cac14e5195e63ed8ae231dd10fd", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/210577fe3cf7badcc5814d99455df46564f3c077", + "reference": "210577fe3cf7badcc5814d99455df46564f3c077", "shasum": "" }, "require": { @@ -3705,9 +3778,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.13.1" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.13.2" }, - "time": "2021-11-03T20:52:16+00:00" + "time": "2021-11-30T19:35:32+00:00" }, { "name": "openlss/lib-array2xml", @@ -4035,16 +4108,16 @@ }, { "name": "phpspec/prophecy", - "version": "1.14.0", + "version": "v1.15.0", "source": { "type": "git", "url": "https://github.com/phpspec/prophecy.git", - "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e" + "reference": "bbcd7380b0ebf3961ee21409db7b38bc31d69a13" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e", - "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/bbcd7380b0ebf3961ee21409db7b38bc31d69a13", + "reference": "bbcd7380b0ebf3961ee21409db7b38bc31d69a13", "shasum": "" }, "require": { @@ -4096,22 +4169,22 @@ ], "support": { "issues": "https://github.com/phpspec/prophecy/issues", - "source": "https://github.com/phpspec/prophecy/tree/1.14.0" + "source": "https://github.com/phpspec/prophecy/tree/v1.15.0" }, - "time": "2021-09-10T09:02:12+00:00" + "time": "2021-12-08T12:19:24+00:00" }, { "name": "phpunit/php-code-coverage", - "version": "9.2.9", + "version": "9.2.10", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "f301eb1453c9e7a1bc912ee8b0ea9db22c60223b" + "reference": "d5850aaf931743067f4bfc1ae4cbd06468400687" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/f301eb1453c9e7a1bc912ee8b0ea9db22c60223b", - "reference": "f301eb1453c9e7a1bc912ee8b0ea9db22c60223b", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/d5850aaf931743067f4bfc1ae4cbd06468400687", + "reference": "d5850aaf931743067f4bfc1ae4cbd06468400687", "shasum": "" }, "require": { @@ -4167,7 +4240,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.9" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.10" }, "funding": [ { @@ -4175,20 +4248,20 @@ "type": "github" } ], - "time": "2021-11-19T15:21:02+00:00" + "time": "2021-12-05T09:12:13+00:00" }, { "name": "phpunit/php-file-iterator", - "version": "3.0.5", + "version": "3.0.6", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-file-iterator.git", - "reference": "aa4be8575f26070b100fccb67faabb28f21f66f8" + "reference": "cf1c2e7c203ac650e352f4cc675a7021e7d1b3cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/aa4be8575f26070b100fccb67faabb28f21f66f8", - "reference": "aa4be8575f26070b100fccb67faabb28f21f66f8", + "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/cf1c2e7c203ac650e352f4cc675a7021e7d1b3cf", + "reference": "cf1c2e7c203ac650e352f4cc675a7021e7d1b3cf", "shasum": "" }, "require": { @@ -4227,7 +4300,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-file-iterator/issues", - "source": "https://github.com/sebastianbergmann/php-file-iterator/tree/3.0.5" + "source": "https://github.com/sebastianbergmann/php-file-iterator/tree/3.0.6" }, "funding": [ { @@ -4235,7 +4308,7 @@ "type": "github" } ], - "time": "2020-09-28T05:57:25+00:00" + "time": "2021-12-02T12:48:52+00:00" }, { "name": "phpunit/php-invoker", @@ -4523,22 +4596,27 @@ }, { "name": "psr/container", - "version": "1.1.2", + "version": "2.0.2", "source": { "type": "git", "url": "https://github.com/php-fig/container.git", - "reference": "513e0666f7216c7459170d56df27dfcefe1689ea" + "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-fig/container/zipball/513e0666f7216c7459170d56df27dfcefe1689ea", - "reference": "513e0666f7216c7459170d56df27dfcefe1689ea", + "url": "https://api.github.com/repos/php-fig/container/zipball/c71ecc56dfe541dbd90c5360474fbc405f8d5963", + "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963", "shasum": "" }, "require": { "php": ">=7.4.0" }, "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0.x-dev" + } + }, "autoload": { "psr-4": { "Psr\\Container\\": "src/" @@ -4565,9 +4643,9 @@ ], "support": { "issues": "https://github.com/php-fig/container/issues", - "source": "https://github.com/php-fig/container/tree/1.1.2" + "source": "https://github.com/php-fig/container/tree/2.0.2" }, - "time": "2021-11-05T16:50:12+00:00" + "time": "2021-11-05T16:47:00+00:00" }, { "name": "sebastian/cli-parser", @@ -5587,28 +5665,29 @@ }, { "name": "symfony/console", - "version": "v5.3.11", + "version": "v5.4.0", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "3e7ab8f5905058984899b05a4648096f558bfeba" + "reference": "ec3661faca1d110d6c307e124b44f99ac54179e3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/3e7ab8f5905058984899b05a4648096f558bfeba", - "reference": "3e7ab8f5905058984899b05a4648096f558bfeba", + "url": "https://api.github.com/repos/symfony/console/zipball/ec3661faca1d110d6c307e124b44f99ac54179e3", + "reference": "ec3661faca1d110d6c307e124b44f99ac54179e3", "shasum": "" }, "require": { "php": ">=7.2.5", - "symfony/deprecation-contracts": "^2.1", + "symfony/deprecation-contracts": "^2.1|^3", "symfony/polyfill-mbstring": "~1.0", "symfony/polyfill-php73": "^1.8", "symfony/polyfill-php80": "^1.16", - "symfony/service-contracts": "^1.1|^2", - "symfony/string": "^5.1" + "symfony/service-contracts": "^1.1|^2|^3", + "symfony/string": "^5.1|^6.0" }, "conflict": { + "psr/log": ">=3", "symfony/dependency-injection": "<4.4", "symfony/dotenv": "<5.1", "symfony/event-dispatcher": "<4.4", @@ -5620,12 +5699,12 @@ }, "require-dev": { "psr/log": "^1|^2", - "symfony/config": "^4.4|^5.0", - "symfony/dependency-injection": "^4.4|^5.0", - "symfony/event-dispatcher": "^4.4|^5.0", - "symfony/lock": "^4.4|^5.0", - "symfony/process": "^4.4|^5.0", - "symfony/var-dumper": "^4.4|^5.0" + "symfony/config": "^4.4|^5.0|^6.0", + "symfony/dependency-injection": "^4.4|^5.0|^6.0", + "symfony/event-dispatcher": "^4.4|^5.0|^6.0", + "symfony/lock": "^4.4|^5.0|^6.0", + "symfony/process": "^4.4|^5.0|^6.0", + "symfony/var-dumper": "^4.4|^5.0|^6.0" }, "suggest": { "psr/log": "For using the console logger", @@ -5665,7 +5744,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v5.3.11" + "source": "https://github.com/symfony/console/tree/v5.4.0" }, "funding": [ { @@ -5681,7 +5760,7 @@ "type": "tidelift" } ], - "time": "2021-11-21T19:41:05+00:00" + "time": "2021-11-29T15:30:56+00:00" }, { "name": "symfony/polyfill-intl-grapheme", @@ -6009,22 +6088,21 @@ }, { "name": "symfony/service-contracts", - "version": "v2.5.0", + "version": "v3.0.0", "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "1ab11b933cd6bc5464b08e81e2c5b07dec58b0fc" + "reference": "36715ebf9fb9db73db0cb24263c79077c6fe8603" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/1ab11b933cd6bc5464b08e81e2c5b07dec58b0fc", - "reference": "1ab11b933cd6bc5464b08e81e2c5b07dec58b0fc", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/36715ebf9fb9db73db0cb24263c79077c6fe8603", + "reference": "36715ebf9fb9db73db0cb24263c79077c6fe8603", "shasum": "" }, "require": { - "php": ">=7.2.5", - "psr/container": "^1.1", - "symfony/deprecation-contracts": "^2.1" + "php": ">=8.0.2", + "psr/container": "^2.0" }, "conflict": { "ext-psr": "<1.1|>=2" @@ -6035,7 +6113,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "2.5-dev" + "dev-main": "3.0-dev" }, "thanks": { "name": "symfony/contracts", @@ -6072,7 +6150,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v2.5.0" + "source": "https://github.com/symfony/service-contracts/tree/v3.0.0" }, "funding": [ { @@ -6088,35 +6166,37 @@ "type": "tidelift" } ], - "time": "2021-11-04T16:48:04+00:00" + "time": "2021-11-04T17:53:12+00:00" }, { "name": "symfony/string", - "version": "v5.3.10", + "version": "v6.0.0", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "d70c35bb20bbca71fc4ab7921e3c6bda1a82a60c" + "reference": "ba727797426af0f587f4800566300bdc0cda0777" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/d70c35bb20bbca71fc4ab7921e3c6bda1a82a60c", - "reference": "d70c35bb20bbca71fc4ab7921e3c6bda1a82a60c", + "url": "https://api.github.com/repos/symfony/string/zipball/ba727797426af0f587f4800566300bdc0cda0777", + "reference": "ba727797426af0f587f4800566300bdc0cda0777", "shasum": "" }, "require": { - "php": ">=7.2.5", + "php": ">=8.0.2", "symfony/polyfill-ctype": "~1.8", "symfony/polyfill-intl-grapheme": "~1.0", "symfony/polyfill-intl-normalizer": "~1.0", - "symfony/polyfill-mbstring": "~1.0", - "symfony/polyfill-php80": "~1.15" + "symfony/polyfill-mbstring": "~1.0" + }, + "conflict": { + "symfony/translation-contracts": "<2.0" }, "require-dev": { - "symfony/error-handler": "^4.4|^5.0", - "symfony/http-client": "^4.4|^5.0", - "symfony/translation-contracts": "^1.1|^2", - "symfony/var-exporter": "^4.4|^5.0" + "symfony/error-handler": "^5.4|^6.0", + "symfony/http-client": "^5.4|^6.0", + "symfony/translation-contracts": "^2.0|^3.0", + "symfony/var-exporter": "^5.4|^6.0" }, "type": "library", "autoload": { @@ -6155,7 +6235,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v5.3.10" + "source": "https://github.com/symfony/string/tree/v6.0.0" }, "funding": [ { @@ -6171,7 +6251,7 @@ "type": "tidelift" } ], - "time": "2021-10-27T18:21:46+00:00" + "time": "2021-10-29T07:35:21+00:00" }, { "name": "textalk/websocket", diff --git a/src/Appwrite/Database/Document.php b/src/Appwrite/Database/Document.php index aa90fba106..94c38b7b30 100644 --- a/src/Appwrite/Database/Document.php +++ b/src/Appwrite/Database/Document.php @@ -91,6 +91,26 @@ class Document extends ArrayObject return $temp; } + /** + * Get Document Attributes + * + * @return array + */ + public function getAttributes(): array + { + $attributes = []; + + foreach ($this as $attribute => $value) { + if(array_key_exists($attribute, ['$id' => true, '$permissions' => true, '$collection' => true, '$execute' => []])) { + continue; + } + + $attributes[$attribute] = $value; + } + + return $attributes; + } + /** * Set Attribute. * @@ -215,7 +235,7 @@ class Document extends ArrayObject * * @return array */ - public function getArrayCopy(array $whitelist = [], array $blacklist = []) + public function getArrayCopy(array $whitelist = [], array $blacklist = []): array { $array = parent::getArrayCopy(); diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 2d398d1bf0..0e078dee5b 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -2,9 +2,10 @@ namespace Appwrite\Migration; -use Appwrite\Database\Document; -use Appwrite\Database\Database; +use Appwrite\Database\Document as OldDocument; +use Appwrite\Database\Database as OldDatabase; use PDO; +use Redis; use Swoole\Runtime; use Utopia\CLI\Console; use Utopia\Exception; @@ -14,22 +15,32 @@ abstract class Migration /** * @var PDO */ - protected $db; + protected PDO $db; + + /** + * @var Redis + */ + protected Redis $cache; /** * @var int */ - protected $limit = 50; + protected int $limit = 50; /** - * @var Document + * @var OldDocument */ - protected $project; + protected OldDocument $project; /** - * @var Database + * @var OldDatabase */ - protected $projectDB; + protected OldDatabase $oldProjectDB; + + /** + * @var OldDatabase + */ + protected OldDatabase $oldConsoleDB; /** * @var array @@ -49,7 +60,7 @@ abstract class Migration '0.10.3' => 'V09', '0.10.4' => 'V09', '0.11.0' => 'V10', - '0.12.0' => 'V10', + '0.12.0' => 'V11', ]; /** @@ -57,24 +68,32 @@ abstract class Migration * * @param PDO $pdo */ - public function __construct(PDO $db) + public function __construct(PDO $db, Redis $cache = null) { $this->db = $db; + if(!is_null($cache)) { + $this->cache = $cache; + } } /** * Set project for migration. * - * @param Document $project - * @param Database $projectDB + * @param OldDocument $project + * @param OldDatabase $projectDB + * @param OldDatabase $oldConsoleDB * - * @return Migration + * @return self */ - public function setProject(Document $project, Database $projectDB): Migration + public function setProject(OldDocument $project, OldDatabase $projectDB, OldDatabase $oldConsoleDB): self { $this->project = $project; - $this->projectDB = $projectDB; - $this->projectDB->setNamespace('app_' . $project->getId()); + + $this->oldProjectDB = $projectDB; + $this->oldProjectDB->setNamespace('app_' . $project->getId()); + + $this->oldConsoleDB = $oldConsoleDB; + return $this; } @@ -117,7 +136,7 @@ abstract class Migration } try { - $new = $this->projectDB->overwriteDocument($document->getArrayCopy()); + $new = $this->projectDB->overwriteDocument($new->getArrayCopy()); } catch (\Throwable $th) { Console::error('Failed to update document: ' . $th->getMessage()); return; @@ -134,7 +153,14 @@ abstract class Migration } } - public function check_diff_multi($array1, $array2) + /** + * Checks 2 arrays for differences. + * + * @param array $array1 + * @param array $array2 + * @return array + */ + public function check_diff_multi(array $array1, array $array2) { $result = array(); diff --git a/src/Appwrite/Migration/Version/V11.php b/src/Appwrite/Migration/Version/V11.php new file mode 100644 index 0000000000..f03c0eda90 --- /dev/null +++ b/src/Appwrite/Migration/Version/V11.php @@ -0,0 +1,639 @@ +cache)); + $this->dbInternal = new Database(new MariaDB($this->db), $cacheAdapter); + $this->dbExternal = new Database(new MariaDB($this->db), $cacheAdapter); + $this->dbConsole = new Database(new MariaDB($this->db), $cacheAdapter); + $this->dbConsole->setNamespace('project_console_internal'); + } + + $this->newCollections = Config::getParam('collections', []); + $this->oldCollections = Config::getParam('collectionsold', []); + } + + public function execute(): void + { + Authorization::disable(); + Runtime::enableCoroutine(SWOOLE_HOOK_ALL); + + /** + * Get a project to check if the version works with this migration. + */ + if (!str_starts_with($this->oldConsoleDB->getCollectionFirst([ + 'filters' => [ + '$collection=' . OldDatabase::SYSTEM_COLLECTION_PROJECTS + ] + ])->getAttribute('version'), '0.11.')) { + throw new Exception("Can only migrate from version 0.11.x to 0.12.x"); + } + $oldProject = $this->project; + + $this->dbInternal->setNamespace('project_' . $oldProject->getId() . '_internal'); + $this->dbExternal->setNamespace('project_' . $oldProject->getId() . '_external'); + + /** + * Create internal/external structure for projects and skip the console project. + */ + if ($oldProject->getId() !== 'console') { + $project = $this->dbConsole->getDocument('projects', $oldProject->getId()); + + /** + * Migrate Project Document. + */ + if ($project->isEmpty()) { + $newProject = $this->fixDocument($oldProject); + $newProject->setAttribute('version', '0.12.0'); + $project = $this->dbConsole->createDocument('projects', $newProject); + Console::log('Migrating project: ' . $oldProject->getAttribute('name') . ' (' . $oldProject->getId() . ')'); + } + + /** + * Create internal DB tables + */ + if (!$this->dbInternal->exists()) { + $this->dbInternal->create(); + Console::log('Created internal tables for : ' . $project->getAttribute('name') . ' (' . $project->getId() . ')'); + } + + /** + * Create external DB tables + */ + if (!$this->dbExternal->exists()) { + $this->dbExternal->create(); + Console::log('Created external tables for : ' . $project->getAttribute('name') . ' (' . $project->getId() . ')'); + } + + /** + * Create Audit tables + */ + if ($this->dbInternal->getCollection(Audit::COLLECTION)->isEmpty()) { + $audit = new Audit($this->dbInternal); + $audit->setup(); + Console::log('Created audit tables for : ' . $project->getAttribute('name') . ' (' . $project->getId() . ')'); + } + + /** + * Create Abuse tables + */ + if ($this->dbInternal->getCollection(TimeLimit::COLLECTION)->isEmpty()) { + $adapter = new TimeLimit("", 0, 1, $this->dbInternal); + $adapter->setup(); + Console::log('Created abuse tables for : ' . $project->getAttribute('name') . ' (' . $project->getId() . ')'); + } + + /** + * Create internal collections for Project + */ + foreach ($this->newCollections as $key => $collection) { + if (!$this->dbInternal->getCollection($key)->isEmpty()) continue; // Skip if project collection already exists + + $attributes = []; + $indexes = []; + + foreach ($collection['attributes'] as $attribute) { + $attributes[] = new Document([ + '$id' => $attribute['$id'], + 'type' => $attribute['type'], + 'size' => $attribute['size'], + 'required' => $attribute['required'], + 'signed' => $attribute['signed'], + 'array' => $attribute['array'], + 'filters' => $attribute['filters'], + ]); + } + + foreach ($collection['indexes'] as $index) { + $indexes[] = new Document([ + '$id' => $index['$id'], + 'type' => $index['type'], + 'attributes' => $index['attributes'], + 'lengths' => $index['lengths'], + 'orders' => $index['orders'], + ]); + } + + $this->dbInternal->createCollection($key, $attributes, $indexes); + } + + $sum = $this->limit; + $offset = 0; + + /** + * Migrate external collections for Project + */ + while ($sum >= $this->limit) { + $databaseCollections = $this->oldProjectDB->getCollection([ + 'limit' => $this->limit, + 'offset' => $offset, + 'orderType' => 'DESC', + 'filters' => [ + '$collection=' . OldDatabase::SYSTEM_COLLECTION_COLLECTIONS, + ] + ]); + + $sum = \count($databaseCollections); + Console::log('Migrating Collections: ' . $offset . ' / ' . $this->oldProjectDB->getSum()); + + foreach ($databaseCollections as $oldCollection) { + $id = $oldCollection->getId(); + $permissions = $oldCollection->getPermissions(); + $name = $oldCollection->getAttribute('name'); + $newCollection = $this->dbExternal->getCollection($id); + + if ($newCollection->isEmpty()) { + $this->dbExternal->createCollection($id); + /** + * Migrate permissions + */ + $read = $this->migrateWildcardPermissions($permissions['read'] ?? []); + $write = $this->migrateWildcardPermissions($permissions['write'] ?? []); + + /** + * Suffix collection name with a subsequent number to make it unique if possible. + */ + $suffix = 1; + while ($this->dbInternal->findOne('collections', [ + new Query('name', Query::TYPE_EQUAL, [$name]) + ])) { + $name .= ' - ' . $suffix++; + } + + $this->dbInternal->createDocument('collections', new Document([ + '$id' => $id, + '$read' => $read, + '$write' => $write, + 'permission' => 'document', + 'dateCreated' => time(), + 'dateUpdated' => time(), + 'name' => $name, + 'search' => implode(' ', [$id, $name]), + ])); + } else { + Console::warning('Skipped Collection ' . $newCollection->getId() . ' from ' . $newCollection->getCollection()); + } + + /** + * Migrate collection rules to attributes + */ + $attributes = $this->getCollectionAttributes($oldCollection); + + foreach ($attributes as $attribute) { + try { + $this->dbExternal->createAttribute( + collection: $attribute['$collection'], + id: $attribute['$id'], + type: $attribute['type'], + size: $attribute['size'], + required: $attribute['required'], + default: $attribute['default'], + signed: $attribute['signed'], + array: $attribute['array'], + format: null, + filters: $attribute['filters'] + ); + + $this->dbInternal->createDocument('attributes', new Document([ + '$id' => $attribute['$collection'] . '_' . $attribute['$id'], + 'key' => $attribute['$id'], + 'collectionId' => $attribute['$collection'], + 'type' => $attribute['type'], + 'status' => 'available', + 'size' => $attribute['size'], + 'required' => $attribute['required'], + 'signed' => $attribute['signed'], + 'default' => $attribute['default'], + 'array' => $attribute['array'], + 'format' => null, + 'filters' => $attribute['filters'] + ])); + + Console::log('Created "' . $attribute['$id'] . '" attribute in collection: ' . $name); + } catch (\Throwable $th) { + Console::log($th->getMessage() . ' - (' . $attribute['$id'] . '" attribute in collection ' . $name . ')'); + } + } + + /** + * Migrate all external documents + */ + $sumDocs = $this->limit; + $offsetDocs = 0; + while ($sumDocs >= $this->limit) { + $allDocs = $this->oldProjectDB->getCollection([ + 'limit' => $this->limit, + 'offset' => $offsetDocs, + 'orderType' => 'DESC', + 'filters' => [ + '$collection=' . $id + ] + ]); + + $sumDocs = \count($allDocs); + foreach ($allDocs as $document) { + if (!$this->dbExternal->getDocument($id, $document->getId())->isEmpty()) { + continue; + } + foreach ($document as $key => $attr) { + /** + * Convert nested Document to JSON strings. + */ + if ($document->getAttribute($key) instanceof OldDocument) { + $document[$key] = json_encode($this->fixDocument($attr)->getArrayCopy()); + } + /** + * Convert numeric Attributes to float. + */ + if (is_numeric($attr)) { + $document[$key] = floatval($attr); + } + if (\is_array($attr)) { + foreach ($attr as $index => $child) { + /** + * Convert array of nested Document to array JSON strings. + */ + if ($document->getAttribute($key)[$index] instanceof OldDocument) { + $document[$key][$index] = json_encode($this->fixDocument($child)->getArrayCopy()); + } + /** + * Convert array of numeric Attributes to array float. + */ + if (is_numeric($attr)) { + $document[$key][$index] = floatval($child); // Convert any numeric to float + } + } + } + } + $document = new Document($document->getArrayCopy()); + $document = $this->migratePermissions($document); + $this->dbExternal->createDocument($id, $document); + } + $offsetDocs += $this->limit; + } + } + $offset += $this->limit; + } + } else { + Console::log('Skipped console project migration.'); + } + + $sum = $this->limit; + $offset = 0; + Authorization::disable(); + /** + * Migrate internal documents + */ + while ($sum >= $this->limit) { + $all = $this->oldProjectDB->getCollection([ + 'limit' => $this->limit, + 'offset' => $offset, + 'orderType' => 'DESC', + 'filters' => [ + '$collection!=' . OldDatabase::SYSTEM_COLLECTION_COLLECTIONS, + '$collection!=' . OldDatabase::SYSTEM_COLLECTION_RULES, + '$collection!=' . OldDatabase::SYSTEM_COLLECTION_TASKS, + '$collection!=' . OldDatabase::SYSTEM_COLLECTION_PROJECTS, + '$collection!=' . OldDatabase::SYSTEM_COLLECTION_CONNECTIONS, + ] + ]); + + $sum = \count($all); + + Console::log('Migrating Documents: ' . $offset . ' / ' . $this->oldProjectDB->getSum()); + + foreach ($all as $document) { + if ( + !array_key_exists($document->getCollection(), $this->oldCollections) + ) { + continue; + } + + $old = $document->getArrayCopy(); + $new = $this->fixDocument($document); + + if (empty($new->getId())) { + Console::warning('Skipped Document due to missing ID.'); + continue; + } + + try { + if ($this->dbInternal->getDocument($new->getCollection(), $new->getId())->isEmpty()) { + $this->dbInternal->createDocument($new->getCollection(), $new); + } else { + Console::warning('Skipped Document ' . $new->getId() . ' from ' . $new->getCollection()); + } + } catch (\Throwable $th) { + Console::error('Failed to update document: ' . $th->getMessage()); + continue; + + if ($document && $new->getId() !== $document->getId()) { + throw new Exception('Duplication Error'); + } + } + } + + $offset += $this->limit; + } + Console::log('Migrated ' . $sum . ' Documents.'); + } + + protected function fixDocument(OldDocument $oldDocument): Document + { + $document = new Document($oldDocument->getArrayCopy()); + $document = $this->migratePermissions($document); + + /** + * Check attributes and set their default values. + */ + if (array_key_exists($document->getCollection(), $this->oldCollections)) { + foreach ($this->newCollections[$document->getCollection()]['attributes'] as $attr) { + if ( + (!$attr['array'] || + ($attr['array'] && array_key_exists('filter', $attr) + && in_array('json', $attr['filter']))) + && empty($document->getAttribute($attr['$id'], null)) + ) { + $document->setAttribute($attr['$id'], $attr['default'] ?? null); + } + } + } + + switch ($document->getAttribute('$collection')) { + case OldDatabase::SYSTEM_COLLECTION_PLATFORMS: + $projectId = $this->getProjectIdFromReadPermissions($document); + + /** + * Set Project ID + */ + if ($document->getAttribute('projectId') === null) { + $document->setAttribute('projectId', $projectId); + } + + /** + * Set empty key and store if null + */ + if ($document->getAttribute('key') === null) { + $document->setAttribute('key', ''); + } + if ($document->getAttribute('store') === null) { + $document->setAttribute('store', ''); + } + + /** + * Reset Permissions + */ + $document->setAttribute('$read', ['role:all']); + $document->setAttribute('$write', ['role:all']); + + break; + case OldDatabase::SYSTEM_COLLECTION_DOMAINS: + $projectId = $this->getProjectIdFromReadPermissions($document); + + /** + * Set Project ID + */ + if ($document->getAttribute('projectId') === null) { + $document->setAttribute('projectId', $projectId); + } + + /** + * Set empty verification if null + */ + if ($document->getAttribute('verification') === null) { + $document->setAttribute('verification', false); + } + + /** + * Reset Permissions + */ + $document->setAttribute('$read', ['role:all']); + $document->setAttribute('$write', ['role:all']); + + break; + case OldDatabase::SYSTEM_COLLECTION_KEYS: + case OldDatabase::SYSTEM_COLLECTION_WEBHOOKS: + $projectId = $this->getProjectIdFromReadPermissions($document); + + /** + * Set Project ID + */ + if ($document->getAttribute('projectId') === null) { + $document->setAttribute('projectId', $projectId); + } + + /** + * Reset Permissions + */ + $document->setAttribute('$read', ['role:all']); + $document->setAttribute('$write', ['role:all']); + + break; + case OldDatabase::SYSTEM_COLLECTION_USERS: + /** + * Set deleted attribute to false + */ + if ($document->getAttribute('deleted') === null) { + $document->setAttribute('deleted', false); + } + /** + * Remove deprecated user status 0 and replace with boolean. + */ + if ($document->getAttribute('status') === 2) { + $document->setAttribute('status', false); + } else { + $document->setAttribute('status', true); + } + + /** + * Set default values for arrays if not set. + */ + if (empty($document->getAttribute('prefs', []))) { + $document->setAttribute('prefs', []); + } + if (empty($document->getAttribute('sessions', []))) { + $document->setAttribute('sessions', []); + } + if (empty($document->getAttribute('tokens', []))) { + $document->setAttribute('tokens', []); + } + if (empty($document->getAttribute('memberships', []))) { + $document->setAttribute('memberships', []); + } + + /** + * Replace user:{self} with user:USER_ID + */ + $write = $document->getWrite(); + $document->setAttribute('$write', str_replace('user:{self}', "user:{$document->getId()}", $write)); + + break; + case OldDatabase::SYSTEM_COLLECTION_FILES: + /** + * Migrating breakind changes on Files. + */ + if (!empty($document->getAttribute('fileOpenSSLVersion', null))) { + $document + ->setAttribute('openSSLVersion', $document->getAttribute('fileOpenSSLVersion')) + ->removeAttribute('fileOpenSSLVersion'); + } + if (!empty($document->getAttribute('fileOpenSSLCipher', null))) { + $document + ->setAttribute('openSSLCipher', $document->getAttribute('fileOpenSSLCipher')) + ->removeAttribute('fileOpenSSLCipher'); + } + if (!empty($document->getAttribute('fileOpenSSLTag', null))) { + $document + ->setAttribute('openSSLTag', $document->getAttribute('fileOpenSSLTag')) + ->removeAttribute('fileOpenSSLTag'); + } + if (!empty($document->getAttribute('fileOpenSSLIV', null))) { + $document + ->setAttribute('openSSLIV', $document->getAttribute('fileOpenSSLIV')) + ->removeAttribute('fileOpenSSLIV'); + } + + /** + * Remove deprecated attributes. + */ + $document->removeAttribute('folderId'); + $document->removeAttribute('token'); + break; + } + + return $document; + } + + /** + * Migrates $permissions to independent $read and $write. + * @param Document $document + * @return Document + */ + protected function migratePermissions(Document $document): Document + { + if ($document->isSet('$permissions')) { + $permissions = $document->getAttribute('$permissions', []); + $read = $this->migrateWildcardPermissions($permissions['read'] ?? []); + $write = $this->migrateWildcardPermissions($permissions['write'] ?? []); + $document->setAttribute('$read', $read); + $document->setAttribute('$write', $write); + $document->removeAttribute('$permissions'); + } + + return $document; + } + + /** + * Takes a permissions array and replaces wildcard * with role:all. + * @param array $permissions + * @return array + */ + protected function migrateWildcardPermissions(array $permissions): array + { + return array_map(function ($permission) { + if ($permission === '*') return 'role:all'; + return $permission; + }, $permissions); + } + + /** + * Get new collection attributes from old collection rules. + * @param OldDocument $collection + * @return array + */ + protected function getCollectionAttributes(OldDocument $collection): array + { + $attributes = []; + foreach ($collection->getAttribute('rules', []) as $key => $value) { + $collectionId = $collection->getId(); + $id = $value['key']; + $array = $value['array'] ?? false; + $required = $value['required'] ?? false; + $default = $value['default'] ?? null; + $default = match ($value['type']) { + OldDatabase::SYSTEM_VAR_TYPE_NUMERIC => floatval($default), + default => $default + }; + $type = match ($value['type']) { + OldDatabase::SYSTEM_VAR_TYPE_TEXT => Database::VAR_STRING, + OldDatabase::SYSTEM_VAR_TYPE_EMAIL => Database::VAR_STRING, + OldDatabase::SYSTEM_VAR_TYPE_DOCUMENT => Database::VAR_STRING, + OldDatabase::SYSTEM_VAR_TYPE_IP => Database::VAR_STRING, + OldDatabase::SYSTEM_VAR_TYPE_URL => Database::VAR_STRING, + OldDatabase::SYSTEM_VAR_TYPE_WILDCARD => Database::VAR_STRING, + OldDatabase::SYSTEM_VAR_TYPE_NUMERIC => Database::VAR_FLOAT, + OldDatabase::SYSTEM_VAR_TYPE_BOOLEAN => Database::VAR_BOOLEAN, + default => Database::VAR_STRING + }; + + $size = $type === Database::VAR_STRING ? 65_535 : 0; // Max size of text in MariaDB + + $attributes[$key] = [ + '$collection' => $collectionId, + '$id' => $id, + 'type' => $type, + 'size' => $size, + 'required' => $required, + 'default' => $default, + 'array' => $array, + 'signed' => true, + 'filters' => [] + ]; + } + + return $attributes; + } + + /** + * + * @param Document $document + * @return string|null + * @throws Exception + */ + protected function getProjectIdFromReadPermissions(Document $document): string|null + { + $readPermissions = $document->getRead(); + $teamId = str_replace('team:', '', reset($readPermissions)); + return $this->oldConsoleDB->getCollectionFirst([ + 'filters' => [ + '$collection=' . OldDatabase::SYSTEM_COLLECTION_PROJECTS, + 'teamId=' . $teamId + ] + ])->getId(); + } +} From 9d47280d7847219a02cc0ed72ffabd95f45e34fb Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 9 Dec 2021 11:42:49 +0100 Subject: [PATCH 03/13] feat(migratio): add options --- app/tasks/migrate.php | 52 +++- src/Appwrite/Migration/Migration.php | 8 +- src/Appwrite/Migration/Version/V11.php | 357 ++++++++++++++----------- 3 files changed, 245 insertions(+), 172 deletions(-) diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index 61557a4c19..3506226bbb 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -11,7 +11,7 @@ use Appwrite\Database\Adapter\Redis as RedisAdapter; use Appwrite\Migration\Migration; use Utopia\Validator\Text; -Config::load('collections.old', __DIR__.'/../config/collections.old.php'); +Config::load('collections.old', __DIR__ . '/../config/collections.old.php'); $cli ->task('migrate') @@ -23,22 +23,54 @@ $cli Console::exit(1); return; } - + $options = []; if (str_starts_with($version, '0.12.')) { Console::error('WARNING'); Console::warning('Migrating to Version 0.12.x introduces a major breaking change within the Database Service!'); Console::warning('Before migrating, please read about the breaking changes here:'); Console::info('https://appwrite.io/guide-to-db-migration'); $confirm = Console::confirm("If you want to proceed, type 'yes':"); - if($confirm != 'yes') { + if ($confirm != 'yes') { + Console::exit(1); + return; + } + + Console::log(''); + Console::log('Collections'); + Console::log('--------------------'); + Console::warning('Be aware that following actions will happen during the migration:'); + Console::warning('- Nested Document rules will be migrated to String attributes'); + Console::warning('- Numeric rules will be migrated to float attributes'); + Console::info("Do you want to migrate your Database Collections?"); + $options['migrateCollections'] = Console::confirm("Type 'yes' or 'no':"); + + if ($options['migrateCollections'] === 'yes') { + Console::log(''); + Console::log('Documents'); + Console::log('------------------'); + Console::warning('Be aware that following actions will happen during the migration:'); + Console::warning('- Nested Documents will be stored as JSON values'); + Console::warning('- All Numeric values will be converted to float'); + Console::info("Do you want to migrate your Database Documents?"); + $options['migrateDocuments'] = Console::confirm("Type 'yes' or 'no':"); + } else { + $options['migrateDocuments'] = 'no'; + } + + + if ( + !in_array($options['migrateDocuments'], ['yes', 'no']) + || !in_array($options['migrateCollections'], ['yes', 'no']) + ) { + Console::error("You must reply with 'yes' or 'no'!"); Console::exit(1); return; } } - Config::load('collectionsold' , __DIR__.'/../config/collections.old.php'); + Config::load('collectionsold', __DIR__ . '/../config/collections.old.php'); - Console::success('Starting Data Migration to version '.$version); + Console::success('Starting Data Migration to version ' . $version); $db = $register->get('db', true); $cache = $register->get('cache', true); @@ -63,8 +95,8 @@ $cli $projects = [$console]; $count = 0; - $class = 'Appwrite\\Migration\\Version\\'.Migration::$versions[$version]; - $migration = new $class($register->get('db'), $register->get('cache')); + $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; + $migration = new $class($register->get('db'), $register->get('cache'), $options); while ($sum > 0) { foreach ($projects as $project) { @@ -74,7 +106,7 @@ $cli ->execute(); } catch (\Throwable $th) { throw $th; - Console::error('Failed to update project ("'.$project->getId().'") version with error: '.$th->getMessage()); + Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage()); } } @@ -82,7 +114,7 @@ $cli 'limit' => $limit, 'offset' => $offset, 'filters' => [ - '$collection='.Database::SYSTEM_COLLECTION_PROJECTS, + '$collection=' . Database::SYSTEM_COLLECTION_PROJECTS, ], ]); @@ -91,7 +123,7 @@ $cli $count = $count + $sum; if ($sum > 0) { - Console::log('Fetched '.$count.'/'.$consoleDB->getSum().' projects...'); + Console::log('Fetched ' . $count . '/' . $consoleDB->getSum() . ' projects...'); } } $cache->flushAll(); diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 0e078dee5b..d4b8d5fe01 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -12,6 +12,11 @@ use Utopia\Exception; abstract class Migration { + /** + * @var array + */ + protected array $options; + /** * @var PDO */ @@ -68,8 +73,9 @@ abstract class Migration * * @param PDO $pdo */ - public function __construct(PDO $db, Redis $cache = null) + public function __construct(PDO $db, Redis $cache = null, array $options = []) { + $this->options = $options; $this->db = $db; if(!is_null($cache)) { $this->cache = $cache; diff --git a/src/Appwrite/Migration/Version/V11.php b/src/Appwrite/Migration/Version/V11.php index f03c0eda90..b56b3f5617 100644 --- a/src/Appwrite/Migration/Version/V11.php +++ b/src/Appwrite/Migration/Version/V11.php @@ -9,6 +9,7 @@ use Exception; use PDO; use Redis; use Swoole\Runtime; +use Throwable; use Utopia\Abuse\Adapters\TimeLimit; use Utopia\Audit\Audit; use Utopia\Cache\Cache; @@ -18,6 +19,9 @@ use Utopia\Config\Config; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Exception\Limit; +use Utopia\Database\Exception\Authorization as ExceptionAuthorization; +use Utopia\Database\Exception\Structure; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; @@ -32,9 +36,10 @@ class V11 extends Migration protected array $oldCollections; protected array $newCollections; - public function __construct(PDO $db, Redis $cache = null) + public function __construct(PDO $db, Redis $cache = null, array $options = []) { - parent::__construct($db, $cache); + parent::__construct($db, $cache, $options); + $this->options = array_map(fn($option) => $option === 'yes' ? true : false, $this->options); if (!is_null($cache)) { $cacheAdapter = new Cache(new RedisCache($this->cache)); @@ -151,163 +156,8 @@ class V11 extends Migration $this->dbInternal->createCollection($key, $attributes, $indexes); } - - $sum = $this->limit; - $offset = 0; - - /** - * Migrate external collections for Project - */ - while ($sum >= $this->limit) { - $databaseCollections = $this->oldProjectDB->getCollection([ - 'limit' => $this->limit, - 'offset' => $offset, - 'orderType' => 'DESC', - 'filters' => [ - '$collection=' . OldDatabase::SYSTEM_COLLECTION_COLLECTIONS, - ] - ]); - - $sum = \count($databaseCollections); - Console::log('Migrating Collections: ' . $offset . ' / ' . $this->oldProjectDB->getSum()); - - foreach ($databaseCollections as $oldCollection) { - $id = $oldCollection->getId(); - $permissions = $oldCollection->getPermissions(); - $name = $oldCollection->getAttribute('name'); - $newCollection = $this->dbExternal->getCollection($id); - - if ($newCollection->isEmpty()) { - $this->dbExternal->createCollection($id); - /** - * Migrate permissions - */ - $read = $this->migrateWildcardPermissions($permissions['read'] ?? []); - $write = $this->migrateWildcardPermissions($permissions['write'] ?? []); - - /** - * Suffix collection name with a subsequent number to make it unique if possible. - */ - $suffix = 1; - while ($this->dbInternal->findOne('collections', [ - new Query('name', Query::TYPE_EQUAL, [$name]) - ])) { - $name .= ' - ' . $suffix++; - } - - $this->dbInternal->createDocument('collections', new Document([ - '$id' => $id, - '$read' => $read, - '$write' => $write, - 'permission' => 'document', - 'dateCreated' => time(), - 'dateUpdated' => time(), - 'name' => $name, - 'search' => implode(' ', [$id, $name]), - ])); - } else { - Console::warning('Skipped Collection ' . $newCollection->getId() . ' from ' . $newCollection->getCollection()); - } - - /** - * Migrate collection rules to attributes - */ - $attributes = $this->getCollectionAttributes($oldCollection); - - foreach ($attributes as $attribute) { - try { - $this->dbExternal->createAttribute( - collection: $attribute['$collection'], - id: $attribute['$id'], - type: $attribute['type'], - size: $attribute['size'], - required: $attribute['required'], - default: $attribute['default'], - signed: $attribute['signed'], - array: $attribute['array'], - format: null, - filters: $attribute['filters'] - ); - - $this->dbInternal->createDocument('attributes', new Document([ - '$id' => $attribute['$collection'] . '_' . $attribute['$id'], - 'key' => $attribute['$id'], - 'collectionId' => $attribute['$collection'], - 'type' => $attribute['type'], - 'status' => 'available', - 'size' => $attribute['size'], - 'required' => $attribute['required'], - 'signed' => $attribute['signed'], - 'default' => $attribute['default'], - 'array' => $attribute['array'], - 'format' => null, - 'filters' => $attribute['filters'] - ])); - - Console::log('Created "' . $attribute['$id'] . '" attribute in collection: ' . $name); - } catch (\Throwable $th) { - Console::log($th->getMessage() . ' - (' . $attribute['$id'] . '" attribute in collection ' . $name . ')'); - } - } - - /** - * Migrate all external documents - */ - $sumDocs = $this->limit; - $offsetDocs = 0; - while ($sumDocs >= $this->limit) { - $allDocs = $this->oldProjectDB->getCollection([ - 'limit' => $this->limit, - 'offset' => $offsetDocs, - 'orderType' => 'DESC', - 'filters' => [ - '$collection=' . $id - ] - ]); - - $sumDocs = \count($allDocs); - foreach ($allDocs as $document) { - if (!$this->dbExternal->getDocument($id, $document->getId())->isEmpty()) { - continue; - } - foreach ($document as $key => $attr) { - /** - * Convert nested Document to JSON strings. - */ - if ($document->getAttribute($key) instanceof OldDocument) { - $document[$key] = json_encode($this->fixDocument($attr)->getArrayCopy()); - } - /** - * Convert numeric Attributes to float. - */ - if (is_numeric($attr)) { - $document[$key] = floatval($attr); - } - if (\is_array($attr)) { - foreach ($attr as $index => $child) { - /** - * Convert array of nested Document to array JSON strings. - */ - if ($document->getAttribute($key)[$index] instanceof OldDocument) { - $document[$key][$index] = json_encode($this->fixDocument($child)->getArrayCopy()); - } - /** - * Convert array of numeric Attributes to array float. - */ - if (is_numeric($attr)) { - $document[$key][$index] = floatval($child); // Convert any numeric to float - } - } - } - } - $document = new Document($document->getArrayCopy()); - $document = $this->migratePermissions($document); - $this->dbExternal->createDocument($id, $document); - } - $offsetDocs += $this->limit; - } - } - $offset += $this->limit; + if ($this->options['migrateCollections']) { + $this->migrateExternalCollections(); } } else { Console::log('Skipped console project migration.'); @@ -315,7 +165,7 @@ class V11 extends Migration $sum = $this->limit; $offset = 0; - Authorization::disable(); + /** * Migrate internal documents */ @@ -373,6 +223,192 @@ class V11 extends Migration Console::log('Migrated ' . $sum . ' Documents.'); } + /** + * Migrate external collections for Project + * + * @return void + * @throws Exception + * @throws Throwable + * @throws Limit + * @throws ExceptionAuthorization + * @throws Structure + */ + protected function migrateExternalCollections(): void + { + $sum = $this->limit; + $offset = 0; + + while ($sum >= $this->limit) { + $databaseCollections = $this->oldProjectDB->getCollection([ + 'limit' => $this->limit, + 'offset' => $offset, + 'orderType' => 'DESC', + 'filters' => [ + '$collection=' . OldDatabase::SYSTEM_COLLECTION_COLLECTIONS, + ] + ]); + + $sum = \count($databaseCollections); + Console::log('Migrating Collections: ' . $offset . ' / ' . $this->oldProjectDB->getSum()); + + foreach ($databaseCollections as $oldCollection) { + $id = $oldCollection->getId(); + $permissions = $oldCollection->getPermissions(); + $name = $oldCollection->getAttribute('name'); + $newCollection = $this->dbExternal->getCollection($id); + + if ($newCollection->isEmpty()) { + $this->dbExternal->createCollection($id); + /** + * Migrate permissions + */ + $read = $this->migrateWildcardPermissions($permissions['read'] ?? []); + $write = $this->migrateWildcardPermissions($permissions['write'] ?? []); + + /** + * Suffix collection name with a subsequent number to make it unique if possible. + */ + $suffix = 1; + while ($this->dbInternal->findOne('collections', [ + new Query('name', Query::TYPE_EQUAL, [$name]) + ])) { + $name .= ' - ' . $suffix++; + } + + $this->dbInternal->createDocument('collections', new Document([ + '$id' => $id, + '$read' => $read, + '$write' => $write, + 'permission' => 'document', + 'dateCreated' => time(), + 'dateUpdated' => time(), + 'name' => $name, + 'search' => implode(' ', [$id, $name]), + ])); + } else { + Console::warning('Skipped Collection ' . $newCollection->getId() . ' from ' . $newCollection->getCollection()); + } + + /** + * Migrate collection rules to attributes + */ + $attributes = $this->getCollectionAttributes($oldCollection); + + foreach ($attributes as $attribute) { + try { + $this->dbExternal->createAttribute( + collection: $attribute['$collection'], + id: $attribute['$id'], + type: $attribute['type'], + size: $attribute['size'], + required: $attribute['required'], + default: $attribute['default'], + signed: $attribute['signed'], + array: $attribute['array'], + format: null, + filters: $attribute['filters'] + ); + + $this->dbInternal->createDocument('attributes', new Document([ + '$id' => $attribute['$collection'] . '_' . $attribute['$id'], + 'key' => $attribute['$id'], + 'collectionId' => $attribute['$collection'], + 'type' => $attribute['type'], + 'status' => 'available', + 'size' => $attribute['size'], + 'required' => $attribute['required'], + 'signed' => $attribute['signed'], + 'default' => $attribute['default'], + 'array' => $attribute['array'], + 'format' => null, + 'filters' => $attribute['filters'] + ])); + + Console::log('Created "' . $attribute['$id'] . '" attribute in collection: ' . $name); + } catch (\Throwable $th) { + Console::log($th->getMessage() . ' - (' . $attribute['$id'] . '" attribute in collection ' . $name . ')'); + } + } + if ($this->options['migrateDocuments']) { + $this->migrateExternalDocuments(collection: $id); + } + } + $offset += $this->limit; + } + } + /** + * Migrate all external documents + * + * @return void + * @throws Exception + * @throws Throwable + * @throws ExceptionAuthorization + * @throws Structure + */ + protected function migrateExternalDocuments(string $collection): void + { + $sum = $this->limit; + $offset = 0; + while ($sum >= $this->limit) { + $allDocs = $this->oldProjectDB->getCollection([ + 'limit' => $this->limit, + 'offset' => $offset, + 'orderType' => 'DESC', + 'filters' => [ + '$collection=' . $collection + ] + ]); + + $sum = \count($allDocs); + foreach ($allDocs as $document) { + if (!$this->dbExternal->getDocument($collection, $document->getId())->isEmpty()) { + continue; + } + foreach ($document as $key => $attr) { + /** + * Convert nested Document to JSON strings. + */ + if ($document->getAttribute($key) instanceof OldDocument) { + $document[$key] = json_encode($this->fixDocument($attr)->getArrayCopy()); + } + /** + * Convert numeric Attributes to float. + */ + if (is_numeric($attr)) { + $document[$key] = floatval($attr); + } + if (\is_array($attr)) { + foreach ($attr as $index => $child) { + /** + * Convert array of nested Document to array JSON strings. + */ + if ($document->getAttribute($key)[$index] instanceof OldDocument) { + $document[$key][$index] = json_encode($this->fixDocument($child)->getArrayCopy()); + } + /** + * Convert array of numeric Attributes to array float. + */ + if (is_numeric($attr)) { + $document[$key][$index] = floatval($child); // Convert any numeric to float + } + } + } + } + $document = new Document($document->getArrayCopy()); + $document = $this->migratePermissions($document); + $this->dbExternal->createDocument($collection, $document); + } + $offset += $this->limit; + } + } + + /** + * Migrates single docuemnt. + * + * @param OldDocument $oldDocument + * @return Document + * @throws Exception + */ protected function fixDocument(OldDocument $oldDocument): Document { $document = new Document($oldDocument->getArrayCopy()); @@ -620,7 +656,6 @@ class V11 extends Migration } /** - * * @param Document $document * @return string|null * @throws Exception From 04a3e0633dbbf25048cc2aea215de6b9ff30f386 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 9 Dec 2021 13:51:29 +0100 Subject: [PATCH 04/13] add coroutine --- src/Appwrite/Migration/Migration.php | 2 +- src/Appwrite/Migration/Version/V11.php | 149 +++++++++++++------------ 2 files changed, 79 insertions(+), 72 deletions(-) diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index d4b8d5fe01..36bb9cad33 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -30,7 +30,7 @@ abstract class Migration /** * @var int */ - protected int $limit = 50; + protected int $limit = 500; /** * @var OldDocument diff --git a/src/Appwrite/Migration/Version/V11.php b/src/Appwrite/Migration/Version/V11.php index b56b3f5617..663ad4beea 100644 --- a/src/Appwrite/Migration/Version/V11.php +++ b/src/Appwrite/Migration/Version/V11.php @@ -39,8 +39,8 @@ class V11 extends Migration public function __construct(PDO $db, Redis $cache = null, array $options = []) { parent::__construct($db, $cache, $options); - $this->options = array_map(fn($option) => $option === 'yes' ? true : false, $this->options); - + $this->options = array_map(fn ($option) => $option === 'yes' ? true : false, $this->options); + var_dump($this->options); if (!is_null($cache)) { $cacheAdapter = new Cache(new RedisCache($this->cache)); $this->dbInternal = new Database(new MariaDB($this->db), $cacheAdapter); @@ -58,16 +58,6 @@ class V11 extends Migration Authorization::disable(); Runtime::enableCoroutine(SWOOLE_HOOK_ALL); - /** - * Get a project to check if the version works with this migration. - */ - if (!str_starts_with($this->oldConsoleDB->getCollectionFirst([ - 'filters' => [ - '$collection=' . OldDatabase::SYSTEM_COLLECTION_PROJECTS - ] - ])->getAttribute('version'), '0.11.')) { - throw new Exception("Can only migrate from version 0.11.x to 0.12.x"); - } $oldProject = $this->project; $this->dbInternal->setNamespace('project_' . $oldProject->getId() . '_internal'); @@ -175,11 +165,21 @@ class V11 extends Migration 'offset' => $offset, 'orderType' => 'DESC', 'filters' => [ - '$collection!=' . OldDatabase::SYSTEM_COLLECTION_COLLECTIONS, - '$collection!=' . OldDatabase::SYSTEM_COLLECTION_RULES, - '$collection!=' . OldDatabase::SYSTEM_COLLECTION_TASKS, - '$collection!=' . OldDatabase::SYSTEM_COLLECTION_PROJECTS, - '$collection!=' . OldDatabase::SYSTEM_COLLECTION_CONNECTIONS, + '$collection=' . OldDatabase::SYSTEM_COLLECTION_DOMAINS, + '$collection=' . OldDatabase::SYSTEM_COLLECTION_EXECUTIONS, + '$collection=' . OldDatabase::SYSTEM_COLLECTION_FILES, + '$collection=' . OldDatabase::SYSTEM_COLLECTION_FUNCTIONS, + '$collection=' . OldDatabase::SYSTEM_COLLECTION_KEYS, + '$collection=' . OldDatabase::SYSTEM_COLLECTION_MEMBERSHIPS, + '$collection=' . OldDatabase::SYSTEM_COLLECTION_PLATFORMS, + '$collection=' . OldDatabase::SYSTEM_COLLECTION_SESSIONS, + '$collection=' . OldDatabase::SYSTEM_COLLECTION_TAGS, + '$collection=' . OldDatabase::SYSTEM_COLLECTION_TEAMS, + '$collection=' . OldDatabase::SYSTEM_COLLECTION_TOKENS, + '$collection=' . OldDatabase::SYSTEM_COLLECTION_USAGES, + '$collection=' . OldDatabase::SYSTEM_COLLECTION_USERS, + '$collection=' . OldDatabase::SYSTEM_COLLECTION_WEBHOOKS, + '$collection=' . OldDatabase::SYSTEM_COLLECTION_CERTIFICATES, ] ]); @@ -187,36 +187,37 @@ class V11 extends Migration Console::log('Migrating Documents: ' . $offset . ' / ' . $this->oldProjectDB->getSum()); - foreach ($all as $document) { - if ( - !array_key_exists($document->getCollection(), $this->oldCollections) - ) { - continue; - } - - $old = $document->getArrayCopy(); - $new = $this->fixDocument($document); - - if (empty($new->getId())) { - Console::warning('Skipped Document due to missing ID.'); - continue; - } - - try { - if ($this->dbInternal->getDocument($new->getCollection(), $new->getId())->isEmpty()) { - $this->dbInternal->createDocument($new->getCollection(), $new); - } else { - Console::warning('Skipped Document ' . $new->getId() . ' from ' . $new->getCollection()); + go(function ($all) { + foreach ($all as $document) { + if ( + !array_key_exists($document->getCollection(), $this->oldCollections) + ) { + return; } - } catch (\Throwable $th) { - Console::error('Failed to update document: ' . $th->getMessage()); - continue; - if ($document && $new->getId() !== $document->getId()) { - throw new Exception('Duplication Error'); + $new = $this->fixDocument($document); + + if (empty($new->getId())) { + Console::warning('Skipped Document due to missing ID.'); + return; + } + + try { + if ($this->dbInternal->getDocument($new->getCollection(), $new->getId())->isEmpty()) { + $this->dbInternal->createDocument($new->getCollection(), $new); + } else { + Console::warning('Skipped Document ' . $new->getId() . ' from ' . $new->getCollection()); + } + } catch (\Throwable $th) { + Console::error('Failed to update document: ' . $th->getMessage()); + return; + + if ($document && $new->getId() !== $document->getId()) { + throw new Exception('Duplication Error'); + } } } - } + }, $all); $offset += $this->limit; } @@ -326,7 +327,7 @@ class V11 extends Migration Console::log('Created "' . $attribute['$id'] . '" attribute in collection: ' . $name); } catch (\Throwable $th) { - Console::log($th->getMessage() . ' - (' . $attribute['$id'] . '" attribute in collection ' . $name . ')'); + Console::log($th->getMessage() . ' - ("' . $attribute['$id'] . '" attribute in collection ' . $name . ')'); } } if ($this->options['migrateDocuments']) { @@ -349,6 +350,7 @@ class V11 extends Migration { $sum = $this->limit; $offset = 0; + while ($sum >= $this->limit) { $allDocs = $this->oldProjectDB->getCollection([ 'limit' => $this->limit, @@ -360,40 +362,45 @@ class V11 extends Migration ]); $sum = \count($allDocs); + + Console::log('Migrating External Documents for Collection ' . $collection . ': ' . $offset . ' / ' . $this->oldProjectDB->getSum()); + foreach ($allDocs as $document) { if (!$this->dbExternal->getDocument($collection, $document->getId())->isEmpty()) { continue; } - foreach ($document as $key => $attr) { - /** - * Convert nested Document to JSON strings. - */ - if ($document->getAttribute($key) instanceof OldDocument) { - $document[$key] = json_encode($this->fixDocument($attr)->getArrayCopy()); - } - /** - * Convert numeric Attributes to float. - */ - if (is_numeric($attr)) { - $document[$key] = floatval($attr); - } - if (\is_array($attr)) { - foreach ($attr as $index => $child) { - /** - * Convert array of nested Document to array JSON strings. - */ - if ($document->getAttribute($key)[$index] instanceof OldDocument) { - $document[$key][$index] = json_encode($this->fixDocument($child)->getArrayCopy()); - } - /** - * Convert array of numeric Attributes to array float. - */ - if (is_numeric($attr)) { - $document[$key][$index] = floatval($child); // Convert any numeric to float + go(function ($document) { + foreach ($document as $key => $attr) { + /** + * Convert nested Document to JSON strings. + */ + if ($document->getAttribute($key) instanceof OldDocument) { + $document[$key] = json_encode($this->fixDocument($attr)->getArrayCopy()); + } + /** + * Convert numeric Attributes to float. + */ + if (is_numeric($attr)) { + $document[$key] = floatval($attr); + } + if (\is_array($attr)) { + foreach ($attr as $index => $child) { + /** + * Convert array of nested Document to array JSON strings. + */ + if ($document->getAttribute($key)[$index] instanceof OldDocument) { + $document[$key][$index] = json_encode($this->fixDocument($child)->getArrayCopy()); + } + /** + * Convert array of numeric Attributes to array float. + */ + if (is_numeric($attr)) { + $document[$key][$index] = floatval($child); // Convert any numeric to float + } } } } - } + }, $document); $document = new Document($document->getArrayCopy()); $document = $this->migratePermissions($document); $this->dbExternal->createDocument($collection, $document); From 3ab5a070e5385a15dd1293769fc9b2567b7a9559 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 9 Dec 2021 15:03:08 +0100 Subject: [PATCH 05/13] fix counting documents --- app/tasks/migrate.php | 1 + src/Appwrite/Migration/Version/V11.php | 75 +++++++++++--------------- 2 files changed, 32 insertions(+), 44 deletions(-) diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index 3506226bbb..6179304136 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -128,5 +128,6 @@ $cli } $cache->flushAll(); + sleep(1); // Wait for Coroutines to finish Console::success('Data Migration Completed'); }); diff --git a/src/Appwrite/Migration/Version/V11.php b/src/Appwrite/Migration/Version/V11.php index 663ad4beea..0e61bf4bed 100644 --- a/src/Appwrite/Migration/Version/V11.php +++ b/src/Appwrite/Migration/Version/V11.php @@ -155,7 +155,7 @@ class V11 extends Migration $sum = $this->limit; $offset = 0; - + $total = 0; /** * Migrate internal documents */ @@ -165,21 +165,11 @@ class V11 extends Migration 'offset' => $offset, 'orderType' => 'DESC', 'filters' => [ - '$collection=' . OldDatabase::SYSTEM_COLLECTION_DOMAINS, - '$collection=' . OldDatabase::SYSTEM_COLLECTION_EXECUTIONS, - '$collection=' . OldDatabase::SYSTEM_COLLECTION_FILES, - '$collection=' . OldDatabase::SYSTEM_COLLECTION_FUNCTIONS, - '$collection=' . OldDatabase::SYSTEM_COLLECTION_KEYS, - '$collection=' . OldDatabase::SYSTEM_COLLECTION_MEMBERSHIPS, - '$collection=' . OldDatabase::SYSTEM_COLLECTION_PLATFORMS, - '$collection=' . OldDatabase::SYSTEM_COLLECTION_SESSIONS, - '$collection=' . OldDatabase::SYSTEM_COLLECTION_TAGS, - '$collection=' . OldDatabase::SYSTEM_COLLECTION_TEAMS, - '$collection=' . OldDatabase::SYSTEM_COLLECTION_TOKENS, - '$collection=' . OldDatabase::SYSTEM_COLLECTION_USAGES, - '$collection=' . OldDatabase::SYSTEM_COLLECTION_USERS, - '$collection=' . OldDatabase::SYSTEM_COLLECTION_WEBHOOKS, - '$collection=' . OldDatabase::SYSTEM_COLLECTION_CERTIFICATES, + '$collection!=' . OldDatabase::SYSTEM_COLLECTION_COLLECTIONS, + '$collection!=' . OldDatabase::SYSTEM_COLLECTION_RULES, + '$collection!=' . OldDatabase::SYSTEM_COLLECTION_TASKS, + '$collection!=' . OldDatabase::SYSTEM_COLLECTION_PROJECTS, + '$collection!=' . OldDatabase::SYSTEM_COLLECTION_CONNECTIONS, ] ]); @@ -187,41 +177,38 @@ class V11 extends Migration Console::log('Migrating Documents: ' . $offset . ' / ' . $this->oldProjectDB->getSum()); - go(function ($all) { - foreach ($all as $document) { - if ( - !array_key_exists($document->getCollection(), $this->oldCollections) - ) { - return; + foreach ($all as $document) { + if ( + !array_key_exists($document->getCollection(), $this->oldCollections) + ) { + continue; + } + + $new = $this->fixDocument($document); + + if (empty($new->getId())) { + Console::warning('Skipped Document due to missing ID.'); + continue; + } + + try { + if ($this->dbInternal->getDocument($new->getCollection(), $new->getId())->isEmpty()) { + $this->dbInternal->createDocument($new->getCollection(), $new); } + } catch (\Throwable $th) { + Console::error('Failed to update document: ' . $th->getMessage()); + continue; - $new = $this->fixDocument($document); - - if (empty($new->getId())) { - Console::warning('Skipped Document due to missing ID.'); - return; - } - - try { - if ($this->dbInternal->getDocument($new->getCollection(), $new->getId())->isEmpty()) { - $this->dbInternal->createDocument($new->getCollection(), $new); - } else { - Console::warning('Skipped Document ' . $new->getId() . ' from ' . $new->getCollection()); - } - } catch (\Throwable $th) { - Console::error('Failed to update document: ' . $th->getMessage()); - return; - - if ($document && $new->getId() !== $document->getId()) { - throw new Exception('Duplication Error'); - } + if ($document && $new->getId() !== $document->getId()) { + throw new Exception('Duplication Error'); } } - }, $all); + } $offset += $this->limit; + $total += $sum; } - Console::log('Migrated ' . $sum . ' Documents.'); + Console::log('Migrated ' . $total . ' Documents.'); } /** From 337083e9f46be734f3f66b639d1808242e02ed85 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 9 Dec 2021 15:45:56 +0100 Subject: [PATCH 06/13] add more verbose logging --- src/Appwrite/Migration/Version/V11.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Migration/Version/V11.php b/src/Appwrite/Migration/Version/V11.php index 0e61bf4bed..6642e919b5 100644 --- a/src/Appwrite/Migration/Version/V11.php +++ b/src/Appwrite/Migration/Version/V11.php @@ -40,7 +40,7 @@ class V11 extends Migration { parent::__construct($db, $cache, $options); $this->options = array_map(fn ($option) => $option === 'yes' ? true : false, $this->options); - var_dump($this->options); + if (!is_null($cache)) { $cacheAdapter = new Cache(new RedisCache($this->cache)); $this->dbInternal = new Database(new MariaDB($this->db), $cacheAdapter); @@ -63,6 +63,11 @@ class V11 extends Migration $this->dbInternal->setNamespace('project_' . $oldProject->getId() . '_internal'); $this->dbExternal->setNamespace('project_' . $oldProject->getId() . '_external'); + Console::info(''); + Console::info('------------------------------------'); + Console::info('Migrating project ' . $oldProject->getAttribute('name')); + Console::info('------------------------------------'); + /** * Create internal/external structure for projects and skip the console project. */ @@ -76,7 +81,7 @@ class V11 extends Migration $newProject = $this->fixDocument($oldProject); $newProject->setAttribute('version', '0.12.0'); $project = $this->dbConsole->createDocument('projects', $newProject); - Console::log('Migrating project: ' . $oldProject->getAttribute('name') . ' (' . $oldProject->getId() . ')'); + Console::log('Created project document: ' . $oldProject->getAttribute('name') . ' (' . $oldProject->getId() . ')'); } /** @@ -156,6 +161,7 @@ class V11 extends Migration $sum = $this->limit; $offset = 0; $total = 0; + /** * Migrate internal documents */ From 83e1241db93b9075668270711c9fc48d9bc0f690 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 9 Dec 2021 15:46:07 +0100 Subject: [PATCH 07/13] wait for event loop to finish before exiting cli --- app/tasks/migrate.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index 6179304136..e9271e793e 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -128,6 +128,6 @@ $cli } $cache->flushAll(); - sleep(1); // Wait for Coroutines to finish + Swoole\Event::wait(); // Wait for Coroutines to finish Console::success('Data Migration Completed'); }); From b2ac3fb5f39a01e913a67f074adb1c7d91772aec Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 9 Dec 2021 16:37:41 +0100 Subject: [PATCH 08/13] add better logging --- src/Appwrite/Migration/Version/V11.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Migration/Version/V11.php b/src/Appwrite/Migration/Version/V11.php index 6642e919b5..e7d1285072 100644 --- a/src/Appwrite/Migration/Version/V11.php +++ b/src/Appwrite/Migration/Version/V11.php @@ -181,7 +181,7 @@ class V11 extends Migration $sum = \count($all); - Console::log('Migrating Documents: ' . $offset . ' / ' . $this->oldProjectDB->getSum()); + Console::log('Migrating Internal Documents: ' . $offset . ' / ' . $this->oldProjectDB->getSum()); foreach ($all as $document) { if ( @@ -214,7 +214,7 @@ class V11 extends Migration $offset += $this->limit; $total += $sum; } - Console::log('Migrated ' . $total . ' Documents.'); + Console::log('Migrated ' . $total . ' Internal Documents.'); } /** From ea7d9f617c769a4d902084818a49dadcbd815819 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 10 Dec 2021 11:51:50 +0100 Subject: [PATCH 09/13] handle float min/max --- src/Appwrite/Migration/Version/V11.php | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Migration/Version/V11.php b/src/Appwrite/Migration/Version/V11.php index e7d1285072..98997439c4 100644 --- a/src/Appwrite/Migration/Version/V11.php +++ b/src/Appwrite/Migration/Version/V11.php @@ -299,7 +299,8 @@ class V11 extends Migration default: $attribute['default'], signed: $attribute['signed'], array: $attribute['array'], - format: null, + format: $attribute['format'] ?? null, + formatOptions: $attribute['formatOptions'] ?? [], filters: $attribute['filters'] ); @@ -314,7 +315,8 @@ class V11 extends Migration 'signed' => $attribute['signed'], 'default' => $attribute['default'], 'array' => $attribute['array'], - 'format' => null, + 'format' => $attribute['format'] ?? null, + 'formatOptions' => $attribute['formatOptions'] ?? null, 'filters' => $attribute['filters'] ])); @@ -650,6 +652,13 @@ class V11 extends Migration 'signed' => true, 'filters' => [] ]; + + if ($type === Database::VAR_FLOAT) { + $attributes[$key]['format'] = APP_DATABASE_ATTRIBUTE_FLOAT_RANGE; + $attributes[$key]['formatOptions'] = []; + $attributes[$key]['formatOptions']['min'] = -PHP_FLOAT_MAX; + $attributes[$key]['formatOptions']['max'] = PHP_FLOAT_MAX; + } } return $attributes; From 784183c64611fd12bf96513a59eb616edf3f7366 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 13 Dec 2021 10:44:07 +0100 Subject: [PATCH 10/13] Update src/Appwrite/Migration/Version/V11.php Co-authored-by: kodumbeats --- src/Appwrite/Migration/Version/V11.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Migration/Version/V11.php b/src/Appwrite/Migration/Version/V11.php index 98997439c4..ec91a53a4a 100644 --- a/src/Appwrite/Migration/Version/V11.php +++ b/src/Appwrite/Migration/Version/V11.php @@ -43,8 +43,8 @@ class V11 extends Migration if (!is_null($cache)) { $cacheAdapter = new Cache(new RedisCache($this->cache)); - $this->dbInternal = new Database(new MariaDB($this->db), $cacheAdapter); - $this->dbExternal = new Database(new MariaDB($this->db), $cacheAdapter); + $this->dbInternal = new Database(new MariaDB($this->db), $cacheAdapter); // namespace is set on execution + $this->dbExternal = new Database(new MariaDB($this->db), $cacheAdapter); // namespace is set on execution $this->dbConsole = new Database(new MariaDB($this->db), $cacheAdapter); $this->dbConsole->setNamespace('project_console_internal'); } From 361006d42c74cf5886324be524c60de29914eea0 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Mon, 13 Dec 2021 11:59:13 +0100 Subject: [PATCH 11/13] adapt to review --- app/tasks/migrate.php | 4 ++++ src/Appwrite/Migration/Migration.php | 9 ++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index e9271e793e..f6b47814dd 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -25,7 +25,9 @@ $cli } $options = []; if (str_starts_with($version, '0.12.')) { + Console::error('--------------------'); Console::error('WARNING'); + Console::error('--------------------'); Console::warning('Migrating to Version 0.12.x introduces a major breaking change within the Database Service!'); Console::warning('Before migrating, please read about the breaking changes here:'); Console::info('https://appwrite.io/guide-to-db-migration'); @@ -41,6 +43,7 @@ $cli Console::warning('Be aware that following actions will happen during the migration:'); Console::warning('- Nested Document rules will be migrated to String attributes'); Console::warning('- Numeric rules will be migrated to float attributes'); + Console::warning('- Wildcard and Markdown rules will be converted to string attributes'); Console::info("Do you want to migrate your Database Collections?"); $options['migrateCollections'] = Console::confirm("Type 'yes' or 'no':"); @@ -51,6 +54,7 @@ $cli Console::warning('Be aware that following actions will happen during the migration:'); Console::warning('- Nested Documents will be stored as JSON values'); Console::warning('- All Numeric values will be converted to float'); + Console::warning('- All Wildcard and Markdown values will be converted to string'); Console::info("Do you want to migrate your Database Documents?"); $options['migrateDocuments'] = Console::confirm("Type 'yes' or 'no':"); } else { diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 36bb9cad33..08effc05db 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -71,13 +71,16 @@ abstract class Migration /** * Migration constructor. * - * @param PDO $pdo + * @param PDO $db + * @param Redis|null $cache + * @param array $options + * @return void */ public function __construct(PDO $db, Redis $cache = null, array $options = []) { $this->options = $options; $this->db = $db; - if(!is_null($cache)) { + if (!is_null($cache)) { $this->cache = $cache; } } @@ -166,7 +169,7 @@ abstract class Migration * @param array $array2 * @return array */ - public function check_diff_multi(array $array1, array $array2) + public function check_diff_multi(array $array1, array $array2): array { $result = array(); From ae37c630bc9982e77efab84ecfd7240b9f803c39 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 17 Dec 2021 11:58:11 +0100 Subject: [PATCH 12/13] fix: document level permissions migration --- src/Appwrite/Migration/Version/V11.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Migration/Version/V11.php b/src/Appwrite/Migration/Version/V11.php index ec91a53a4a..ece9ba04e4 100644 --- a/src/Appwrite/Migration/Version/V11.php +++ b/src/Appwrite/Migration/Version/V11.php @@ -271,8 +271,8 @@ class V11 extends Migration $this->dbInternal->createDocument('collections', new Document([ '$id' => $id, - '$read' => $read, - '$write' => $write, + '$read' => [], + '$write' => [], 'permission' => 'document', 'dateCreated' => time(), 'dateUpdated' => time(), From 468bfcb88ff74cebeabe8f7703dad26cbf768359 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 17 Dec 2021 12:18:28 +0100 Subject: [PATCH 13/13] chore: update sdk docs --- .../examples/functions/list-runtimes.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 docs/examples/0.12.x/console-web/examples/functions/list-runtimes.md diff --git a/docs/examples/0.12.x/console-web/examples/functions/list-runtimes.md b/docs/examples/0.12.x/console-web/examples/functions/list-runtimes.md new file mode 100644 index 0000000000..a6d0099e1b --- /dev/null +++ b/docs/examples/0.12.x/console-web/examples/functions/list-runtimes.md @@ -0,0 +1,14 @@ +const sdk = new Appwrite(); + +sdk + .setEndpoint('https://[HOSTNAME_OR_IP]/v1') // Your API Endpoint + .setProject('5df5acd0d48c2') // Your project ID +; + +let promise = sdk.functions.listRuntimes(); + +promise.then(function (response) { + console.log(response); // Success +}, function (error) { + console.log(error); // Failure +}); \ No newline at end of file