From eff09b665d28d778218082fb1a26d4637e98a8c4 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 17 May 2024 15:27:06 +0900 Subject: [PATCH] Refactor migrations to allow for more concurrency --- app/config/collections.php | 175 +++++++++++++----- app/controllers/api/migrations.php | 126 ++++++++++--- app/init.php | 66 +++++++ composer.lock | 71 ++++--- docker-compose.yml | 7 +- src/Appwrite/Event/Migration.php | 5 + .../Platform/Tasks/GetMigrationStats.php | 2 - src/Appwrite/Platform/Workers/Migrations.php | 114 ++++++++---- .../Database/Validator/Queries/Migrations.php | 1 - .../Utopia/Response/Model/Migration.php | 6 - 10 files changed, 419 insertions(+), 154 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index cd0d1b2ef5..de1844fad6 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -4018,17 +4018,6 @@ $projectCollections = array_merge([ 'array' => false, 'filters' => [], ], - [ - '$id' => ID::custom('stage'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => false, - 'filters' => [], - ], [ '$id' => ID::custom('source'), 'type' => Database::VAR_STRING, @@ -4051,6 +4040,36 @@ $projectCollections = array_merge([ 'array' => false, 'filters' => ['json', 'encrypt'], ], + [ + '$id' => ID::custom('statusCounters'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 3000, + 'signed' => true, + 'required' => false, + 'array' => false, + 'filters' => ['subQueryStatusCounters'], + ], + [ + '$id' => ID::custom('resourceData'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 131070, + 'signed' => true, + 'required' => false, + 'array' => false, + 'filters' => ['subQueryResourceData'], + ], + [ + '$id' => ID::custom('errors'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 65535, + 'signed' => true, + 'required' => false, + 'array' => true, + 'filters' => ['subQueryMigrationErrors'], + ], [ '$id' => ID::custom('resources'), 'type' => Database::VAR_STRING, @@ -4062,6 +4081,91 @@ $projectCollections = array_merge([ 'array' => true, 'filters' => [], ], + [ + '$id' => ID::custom('search'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ] + ], + 'indexes' => [ + [ + '$id' => '_key_status', + 'type' => Database::INDEX_KEY, + 'attributes' => ['status'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_source', + 'type' => Database::INDEX_KEY, + 'attributes' => ['source'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => ID::custom('_fulltext_search'), + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['search'], + 'lengths' => [], + 'orders' => [], + ] + ], + ], + 'groupMigrations' => [ + '$collection' => ID::custom(Database::METADATA), + '$id' => ID::custom('groupMigrations'), + 'name' => 'groupMigrations', + 'attributes' => [ + [ + '$id' => ID::custom('migrationId'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [] + ], + [ + '$id' => ID::custom('status'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [] + ], + [ + '$id' => ID::custom('resources'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => true, + 'filters' => [], + ], + [ + '$id' => ID::custom('group'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [] + ], [ '$id' => ID::custom('statusCounters'), 'type' => Database::VAR_STRING, @@ -4095,48 +4199,23 @@ $projectCollections = array_merge([ 'array' => true, 'filters' => [], ], - [ - '$id' => ID::custom('search'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 16384, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ] ], 'indexes' => [ [ - '$id' => '_key_status', - 'type' => Database::INDEX_KEY, - 'attributes' => ['status'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => '_key_stage', - 'type' => Database::INDEX_KEY, - 'attributes' => ['stage'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => '_key_source', - 'type' => Database::INDEX_KEY, - 'attributes' => ['source'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_fulltext_search'), + '$id' => '_key_migrationId', 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['search'], - 'lengths' => [], - 'orders' => [], + 'attributes' => ['migrationId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_group', + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['group'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], ] - ], + ] ], ], $commonCollections); diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index 3899b26ad4..e00bb082b8 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -21,6 +21,7 @@ use Utopia\Migration\Sources\Appwrite; use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; +use Utopia\Migration\Transfer; use Utopia\System\System; use Utopia\Validator\ArrayList; use Utopia\Validator\Host; @@ -31,6 +32,84 @@ use Utopia\Validator\WhiteList; include_once __DIR__ . '/../shared/api.php'; +$triggerMigration = function (string $migrationId, array $resources, Migration $queueForMigrations) { + $groupDocument = new Document([ + '$id' => ID::unique(), + 'status' => 'pending', + 'migrationId' => $migrationId, + 'group' => '', + 'resources' => [], + 'statusCounters' => '', + 'resourceData' => '', + 'errors' => [] + ]); + + if (!empty(array_intersect( + Transfer::GROUP_AUTH_RESOURCES, + $resources + ))) { + $groupDocument->setAttribute('$id', ID::unique()); + $groupDocument->setAttribute('group', Migration::TYPE_AUTH); + $groupDocument->setAttribute('resources', array_intersect( + Transfer::GROUP_AUTH_RESOURCES, + $resources + )); + + $queueForMigrations + ->setType(Migration::TYPE_AUTH) + ->setMigration($groupDocument) + ->trigger(); + } + + if (!empty(array_intersect( + Transfer::GROUP_STORAGE_RESOURCES, + $resources + ))) { + $groupDocument->setAttribute('$id', ID::unique()); + $groupDocument->setAttribute('group', Migration::TYPE_STORAGE); + $groupDocument->setAttribute('resources', array_intersect( + Transfer::GROUP_STORAGE_RESOURCES, + $resources + )); + + $queueForMigrations + ->setMigration($groupDocument) + ->trigger(); + } + + if (!empty(array_intersect( + Transfer::GROUP_DATABASES_RESOURCES, + $resources + ))) { + $groupDocument->setAttribute('$id', ID::unique()); + $groupDocument->setAttribute('group', Migration::TYPE_DATABASES); + $groupDocument->setAttribute('resources', array_intersect( + Transfer::GROUP_DATABASES_RESOURCES, + $resources + )); + + $queueForMigrations + ->setMigration($groupDocument) + ->trigger(); + } + + if (!empty(array_intersect( + Transfer::GROUP_FUNCTIONS_RESOURCES, + $resources + ))) { + $groupDocument->setAttribute('$id', ID::unique()); + $groupDocument->setAttribute('group', Migration::TYPE_FUNCTIONS); + $groupDocument->setAttribute('resources', array_intersect( + Transfer::GROUP_FUNCTIONS_RESOURCES, + $resources + )); + + $queueForMigrations + ->setMigration($groupDocument) + ->trigger(); + } +}; + App::post('/v1/migrations/appwrite') ->groups(['api', 'migrations']) ->desc('Migrate Appwrite Data') @@ -54,18 +133,16 @@ App::post('/v1/migrations/appwrite') ->inject('user') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (array $resources, string $endpoint, string $projectId, string $apiKey, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) { + ->action(function (array $resources, string $endpoint, string $projectId, string $apiKey, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) use ($triggerMigration) { $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), 'status' => 'pending', - 'stage' => 'init', 'source' => Appwrite::getName(), 'credentials' => [ 'endpoint' => $endpoint, 'projectId' => $projectId, 'apiKey' => $apiKey, ], - 'resources' => $resources, 'statusCounters' => '{}', 'resourceData' => '{}', 'errors' => [], @@ -73,12 +150,11 @@ App::post('/v1/migrations/appwrite') $queueForEvents->setParam('migrationId', $migration->getId()); - // Trigger Transfer $queueForMigrations - ->setMigration($migration) ->setProject($project) - ->setUser($user) - ->trigger(); + ->setUser($user); + + $triggerMigration($migration->getId(), $resources, $queueForMigrations); $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) @@ -108,7 +184,7 @@ App::post('/v1/migrations/firebase/oauth') ->inject('queueForEvents') ->inject('queueForMigrations') ->inject('request') - ->action(function (array $resources, string $projectId, Response $response, Database $dbForProject, Database $dbForConsole, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations, Request $request) { + ->action(function (array $resources, string $projectId, Response $response, Database $dbForProject, Database $dbForConsole, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations, Request $request) use ($triggerMigration) { $firebase = new OAuth2Firebase( System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''), System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''), @@ -162,7 +238,6 @@ App::post('/v1/migrations/firebase/oauth') $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), 'status' => 'pending', - 'stage' => 'init', 'source' => Firebase::getName(), 'credentials' => [ 'serviceAccount' => json_encode($serviceAccount), @@ -177,10 +252,10 @@ App::post('/v1/migrations/firebase/oauth') // Trigger Transfer $queueForMigrations - ->setMigration($migration) ->setProject($project) - ->setUser($user) - ->trigger(); + ->setUser($user); + + $triggerMigration($migration->getId(), $resources, $queueForMigrations); $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) @@ -208,7 +283,7 @@ App::post('/v1/migrations/firebase') ->inject('user') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (array $resources, string $serviceAccount, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) { + ->action(function (array $resources, string $serviceAccount, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) use ($triggerMigration) { $serviceAccountData = json_decode($serviceAccount, true); if (empty($serviceAccountData)) { @@ -222,7 +297,6 @@ App::post('/v1/migrations/firebase') $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), 'status' => 'pending', - 'stage' => 'init', 'source' => Firebase::getName(), 'credentials' => [ 'serviceAccount' => $serviceAccount, @@ -237,10 +311,10 @@ App::post('/v1/migrations/firebase') // Trigger Transfer $queueForMigrations - ->setMigration($migration) ->setProject($project) - ->setUser($user) - ->trigger(); + ->setUser($user); + + $triggerMigration($migration->getId(), $resources, $queueForMigrations); $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) @@ -273,11 +347,10 @@ App::post('/v1/migrations/supabase') ->inject('user') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (array $resources, string $endpoint, string $apiKey, string $databaseHost, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) { + ->action(function (array $resources, string $endpoint, string $apiKey, string $databaseHost, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) use ($triggerMigration) { $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), 'status' => 'pending', - 'stage' => 'init', 'source' => Supabase::getName(), 'credentials' => [ 'endpoint' => $endpoint, @@ -297,10 +370,10 @@ App::post('/v1/migrations/supabase') // Trigger Transfer $queueForMigrations - ->setMigration($migration) ->setProject($project) - ->setUser($user) - ->trigger(); + ->setUser($user); + + $triggerMigration($migration->getId(), $resources, $queueForMigrations); $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) @@ -334,11 +407,10 @@ App::post('/v1/migrations/nhost') ->inject('user') ->inject('queueForEvents') ->inject('queueForMigrations') - ->action(function (array $resources, string $subdomain, string $region, string $adminSecret, string $database, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) { + ->action(function (array $resources, string $subdomain, string $region, string $adminSecret, string $database, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) use ($triggerMigration) { $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), 'status' => 'pending', - 'stage' => 'init', 'source' => NHost::getName(), 'credentials' => [ 'subdomain' => $subdomain, @@ -359,10 +431,10 @@ App::post('/v1/migrations/nhost') // Trigger Transfer $queueForMigrations - ->setMigration($migration) ->setProject($project) - ->setUser($user) - ->trigger(); + ->setUser($user); + + $triggerMigration($migration->getId(), $resources, $queueForMigrations); $response ->setStatusCode(Response::STATUS_CODE_ACCEPTED) diff --git a/app/init.php b/app/init.php index f69bd323c6..c197c33e80 100644 --- a/app/init.php +++ b/app/init.php @@ -680,6 +680,72 @@ Database::addFilter( } ); +Database::addFilter( + 'subQueryStatusCounters', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + $groups = $database->find('groupMigrations', [ + Query::equal('migrationId', [$document->getId()]), + ]); + + $statusCounter = []; + + foreach ($groups as $document) { + $data = $document->getAttribute('statusCounters'); + + $statusCounter = array_merge($statusCounter, $data); + } + + return $statusCounter; + } +); + +Database::addFilter( + 'subQueryResourceData', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + $groups = $database->find('groupMigrations', [ + Query::equal('migrationId', [$document->getId()]), + ]); + + $resourceData = []; + + foreach ($groups as $document) { + $data = $document->getAttribute('resourceData'); + + $resourceData = array_merge($resourceData, $data); + } + + return $resourceData; + } +); + +Database::addFilter( + 'subQueryMigrationErrors', + function (mixed $value) { + return; + }, + function (mixed $value, Document $document, Database $database) { + $groups = $database->find('groupMigrations', [ + Query::equal('migrationId', [$document->getId()]), + ]); + + $errors = []; + + foreach ($groups as $document) { + $data = $document->getAttribute('errors'); + + $errors = array_merge($errors, $data); + } + + return $errors; + } +); + /** * DB Formats */ diff --git a/composer.lock b/composer.lock index 34a3cc143c..6d36867fcf 100644 --- a/composer.lock +++ b/composer.lock @@ -822,16 +822,16 @@ }, { "name": "paragonie/constant_time_encoding", - "version": "v2.6.3", + "version": "v2.7.0", "source": { "type": "git", "url": "https://github.com/paragonie/constant_time_encoding.git", - "reference": "58c3f47f650c94ec05a151692652a868995d2938" + "reference": "52a0d99e69f56b9ec27ace92ba56897fe6993105" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/58c3f47f650c94ec05a151692652a868995d2938", - "reference": "58c3f47f650c94ec05a151692652a868995d2938", + "url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/52a0d99e69f56b9ec27ace92ba56897fe6993105", + "reference": "52a0d99e69f56b9ec27ace92ba56897fe6993105", "shasum": "" }, "require": { @@ -885,7 +885,7 @@ "issues": "https://github.com/paragonie/constant_time_encoding/issues", "source": "https://github.com/paragonie/constant_time_encoding" }, - "time": "2022-06-14T06:56:20+00:00" + "time": "2024-05-08T12:18:48+00:00" }, { "name": "phpmailer/phpmailer", @@ -1672,16 +1672,16 @@ }, { "name": "utopia-php/dsn", - "version": "0.2.0", + "version": "0.2.1", "source": { "type": "git", "url": "https://github.com/utopia-php/dsn.git", - "reference": "c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7" + "reference": "42ee37a3d1785100b2f69091c9d4affadb6846eb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/dsn/zipball/c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7", - "reference": "c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7", + "url": "https://api.github.com/repos/utopia-php/dsn/zipball/42ee37a3d1785100b2f69091c9d4affadb6846eb", + "reference": "42ee37a3d1785100b2f69091c9d4affadb6846eb", "shasum": "" }, "require": { @@ -1713,9 +1713,9 @@ ], "support": { "issues": "https://github.com/utopia-php/dsn/issues", - "source": "https://github.com/utopia-php/dsn/tree/0.2.0" + "source": "https://github.com/utopia-php/dsn/tree/0.2.1" }, - "time": "2023-11-02T12:01:43+00:00" + "time": "2024-05-07T02:01:25+00:00" }, { "name": "utopia-php/framework", @@ -1966,22 +1966,21 @@ }, { "name": "utopia-php/migration", - "version": "0.4.0", + "version": "0.4.4", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "a72f27bd3dde68752fb185d306c4820e1b8d9657" + "reference": "a8a5d392bebf082faf289f4dfe09d9fd76844c33" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/a72f27bd3dde68752fb185d306c4820e1b8d9657", - "reference": "a72f27bd3dde68752fb185d306c4820e1b8d9657", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/a8a5d392bebf082faf289f4dfe09d9fd76844c33", + "reference": "a8a5d392bebf082faf289f4dfe09d9fd76844c33", "shasum": "" }, "require": { "appwrite/appwrite": "10.1.0", - "php": "8.*", - "utopia-php/cli": "0.*" + "php": "8.*" }, "require-dev": { "laravel/pint": "1.*", @@ -2008,9 +2007,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.4.0" + "source": "https://github.com/utopia-php/migration/tree/0.4.4" }, - "time": "2024-02-25T12:35:21+00:00" + "time": "2024-05-17T05:25:31+00:00" }, { "name": "utopia-php/mongo", @@ -2731,16 +2730,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.38.2", + "version": "0.38.5", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "51284668529e2b10ed933412a42b603c76cded23" + "reference": "830a46cc8e34ee096a76d4af6f00adf008a7cbf8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/51284668529e2b10ed933412a42b603c76cded23", - "reference": "51284668529e2b10ed933412a42b603c76cded23", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/830a46cc8e34ee096a76d4af6f00adf008a7cbf8", + "reference": "830a46cc8e34ee096a76d4af6f00adf008a7cbf8", "shasum": "" }, "require": { @@ -2776,9 +2775,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.38.2" + "source": "https://github.com/appwrite/sdk-generator/tree/0.38.5" }, - "time": "2024-04-25T07:49:29+00:00" + "time": "2024-05-17T00:59:59+00:00" }, { "name": "doctrine/deprecations", @@ -2899,16 +2898,16 @@ }, { "name": "laravel/pint", - "version": "v1.15.2", + "version": "v1.15.3", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "2c9f8004899815f3f0ee3cb28ef7281e2b589134" + "reference": "3600b5d17aff52f6100ea4921849deacbbeb8656" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/2c9f8004899815f3f0ee3cb28ef7281e2b589134", - "reference": "2c9f8004899815f3f0ee3cb28ef7281e2b589134", + "url": "https://api.github.com/repos/laravel/pint/zipball/3600b5d17aff52f6100ea4921849deacbbeb8656", + "reference": "3600b5d17aff52f6100ea4921849deacbbeb8656", "shasum": "" }, "require": { @@ -2961,7 +2960,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-04-23T15:42:34+00:00" + "time": "2024-04-30T15:02:26+00:00" }, { "name": "matthiasmullie/minify", @@ -3568,16 +3567,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "1.28.0", + "version": "1.29.0", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "cd06d6b1a1b3c75b0b83f97577869fd85a3cd4fb" + "reference": "536889f2b340489d328f5ffb7b02bb6b183ddedc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/cd06d6b1a1b3c75b0b83f97577869fd85a3cd4fb", - "reference": "cd06d6b1a1b3c75b0b83f97577869fd85a3cd4fb", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/536889f2b340489d328f5ffb7b02bb6b183ddedc", + "reference": "536889f2b340489d328f5ffb7b02bb6b183ddedc", "shasum": "" }, "require": { @@ -3609,9 +3608,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.28.0" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.29.0" }, - "time": "2024-04-03T18:51:33+00:00" + "time": "2024-05-06T12:04:23+00:00" }, { "name": "phpunit/php-code-coverage", diff --git a/docker-compose.yml b/docker-compose.yml index b98152c288..29d956d512 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -595,7 +595,7 @@ services: appwrite-worker-migrations: entrypoint: worker-migrations <<: *x-logging - container_name: appwrite-worker-migrations + # container_name: appwrite-worker-migrations restart: unless-stopped image: appwrite-dev networks: @@ -624,8 +624,9 @@ services: - _APP_DB_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - - _APP_MIGRATIONS_FIREBASE_CLIENT_ID - - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET + deploy: + mode: replicated + replicas: 4 appwrite-task-maintenance: entrypoint: maintenance diff --git a/src/Appwrite/Event/Migration.php b/src/Appwrite/Event/Migration.php index 478291829b..3d330c5ccf 100644 --- a/src/Appwrite/Event/Migration.php +++ b/src/Appwrite/Event/Migration.php @@ -11,6 +11,11 @@ class Migration extends Event protected string $type = ''; protected ?Document $migration = null; + const TYPE_AUTH = 'Auth'; + const TYPE_DATABASES = 'Databases'; + const TYPE_STORAGE = 'Storage'; + const TYPE_FUNCTIONS = 'Functions'; + public function __construct(protected Connection $connection) { parent::__construct($connection); diff --git a/src/Appwrite/Platform/Tasks/GetMigrationStats.php b/src/Appwrite/Platform/Tasks/GetMigrationStats.php index 30a9ebb8e9..ba062b4bca 100644 --- a/src/Appwrite/Platform/Tasks/GetMigrationStats.php +++ b/src/Appwrite/Platform/Tasks/GetMigrationStats.php @@ -25,7 +25,6 @@ class GetMigrationStats extends Action '$id', '$createdAt', 'status', - 'stage', 'source' ]; @@ -119,7 +118,6 @@ class GetMigrationStats extends Action $migration->getAttribute('$id'), $migration->getAttribute('$createdAt'), $migration->getAttribute('status'), - $migration->getAttribute('stage'), $migration->getAttribute('source'), ]; }, $migrations); diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 7bbfcfbce2..69675d800b 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -15,7 +15,9 @@ use Utopia\Database\Exception\Conflict; use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; use Utopia\Database\Helpers\ID; +use Utopia\Database\Query; use Utopia\Logger\Log; +use Utopia\Logger\Log\Breadcrumb; use Utopia\Migration\Destinations\Appwrite as DestinationsAppwrite; use Utopia\Migration\Exception as MigrationException; use Utopia\Migration\Source; @@ -161,7 +163,7 @@ class Migrations extends Action roles: $target['roles'], ); - return $this->dbForProject->updateDocument('migrations', $migration->getId(), $migration); + return $this->dbForProject->updateDocument($migration->getCollection(), $migration->getId(), $migration); } /** @@ -241,26 +243,28 @@ class Migrations extends Action * @throws Structure * @throws \Utopia\Database\Exception */ - protected function processMigration(Document $project, Document $migration, Log $log): void + protected function processMigration(Document $project, Document $group, Log $log): void { /** - * @var Document $migrationDocument + * @var Document $migration * @var Transfer $transfer */ - $migrationDocument = null; + $groupDocument = null; $transfer = null; $projectDocument = $this->dbForConsole->getDocument('projects', $project->getId()); $tempAPIKey = $this->generateAPIKey($projectDocument); try { - $migrationDocument = $this->dbForProject->getDocument('migrations', $migration->getId()); - $migrationDocument->setAttribute('stage', 'processing'); - $migrationDocument->setAttribute('status', 'processing'); - $this->updateMigrationDocument($migrationDocument, $projectDocument); + $group = $this->dbForProject->getDocument('groupMigrations', $group->getId()); - $log->addTag('type', $migrationDocument->getAttribute('source')); + $migration = $this->dbForProject->getDocument('migrations', $group->getAttribute('migrationId', '')); + $migration->setAttribute('status', 'processing'); - $source = $this->processSource($migrationDocument->getAttribute('source'), $migrationDocument->getAttribute('credentials')); + $this->updateMigrationDocument($migration, $projectDocument); + + $log->addTag('type', $migration->getAttribute('source')); + + $source = $this->processSource($migration->getAttribute('source'), $migration->getAttribute('credentials')); $source->report(); @@ -276,21 +280,43 @@ class Migrations extends Action ); /** Start Transfer */ - $migrationDocument->setAttribute('stage', 'migrating'); - $this->updateMigrationDocument($migrationDocument, $projectDocument); - $transfer->run($migrationDocument->getAttribute('resources'), function () use ($migrationDocument, $transfer, $projectDocument) { - $migrationDocument->setAttribute('resourceData', json_encode($transfer->getCache())); - $migrationDocument->setAttribute('statusCounters', json_encode($transfer->getStatusCounters())); + $this->updateMigrationDocument($migration, $projectDocument); - $this->updateMigrationDocument($migrationDocument, $projectDocument); + // Calculate group resources + $resources = $group->getAttribute('resources'); + + switch ($group->getAttribute('group')) { + case Transfer::GROUP_AUTH: + $resources = array_intersect(Transfer::GROUP_AUTH_RESOURCES, $resources); + break; + case Transfer::GROUP_STORAGE: + $resources = array_intersect(Transfer::GROUP_STORAGE_RESOURCES, $resources); + break; + case Transfer::GROUP_DATABASES: + $resources = array_intersect(Transfer::GROUP_DATABASES_RESOURCES, $resources); + break; + case Transfer::GROUP_FUNCTIONS: + $resources = array_intersect(Transfer::GROUP_FUNCTIONS_RESOURCES, $resources); + break; + default: + throw new Exception('Migration worker was initialized with unknown group'); + } + + $log->addTag('migrationGroup', $group->getAttribute('group')); + $log->addExtra('migrationResources', json_encode($resources)); + + $transfer->run($resources, function () use ($group, $transfer, $projectDocument) { + $group->setAttribute('resourceData', json_encode($transfer->getCache())); + $group->setAttribute('statusCounters', json_encode($transfer->getStatusCounters())); + + $this->updateMigrationDocument($group, $projectDocument); }); $sourceErrors = $source->getErrors(); $destinationErrors = $destination->getErrors(); if (!empty($sourceErrors) || !empty($destinationErrors)) { - $migrationDocument->setAttribute('status', 'failed'); - $migrationDocument->setAttribute('stage', 'finished'); + $migration->setAttribute('status', 'failed'); $errorMessages = []; foreach ($sourceErrors as $error) { @@ -302,23 +328,48 @@ class Migrations extends Action $errorMessages[] = "Error occurred while pushing '{$error->getResourceType()}:{$error->getResourceId()}' to destination with message: '{$error->getMessage()}'"; } - $migrationDocument->setAttribute('errors', $errorMessages); - $this->updateMigrationDocument($migrationDocument, $projectDocument); + $group->setAttribute('errors', $errorMessages); + $this->updateMigrationDocument($group, $projectDocument); + $this->updateMigrationDocument($migration, $projectDocument); return; } - $migrationDocument->setAttribute('status', 'completed'); - $migrationDocument->setAttribute('stage', 'finished'); + $group->setAttribute('status', 'completed'); + $this->updateMigrationDocument($group, $project); + + // Check if all other groups are finished, if so set parent document to completed aswell. + $groupDocuments = $this->dbForProject->find('groupMigrations', [Query::equal('migrationId', [$migration->getId()])]); + + $result = 'completed'; + foreach ($groupDocuments as $document) { + if ($document->getId() == $group->getId()) { + continue; + } + + $status = $document->getAttribute('status', 'processing'); + + if ($status == 'processing' || $status == 'pending') { + $result = 'processing'; + break; + } + + // Only fail parent if all have stopped processing. + if ($status == 'failed') { + break; + } + } + + $migration->setAttribute('status', $result); + $this->updateMigrationDocument($migration, $project); } catch (\Throwable $th) { Console::error($th->getMessage()); - if ($migrationDocument) { + if ($migration && $groupDocument) { Console::error($th->getMessage()); Console::error($th->getTraceAsString()); - $migrationDocument->setAttribute('status', 'failed'); - $migrationDocument->setAttribute('stage', 'finished'); - $migrationDocument->setAttribute('errors', [$th->getMessage()]); + $migration->setAttribute('status', 'failed'); + $groupDocument->setAttribute('errors', [$th->getMessage()]); return; } @@ -337,17 +388,18 @@ class Migrations extends Action $errorMessages[] = "Error occurred while pushing '{$error->getResourceType()}:{$error->getResourceId()}' to destination with message '{$error->getMessage()}'"; } - $migrationDocument->setAttribute('errors', $errorMessages); + $group->setAttribute('errors', $errorMessages); } } finally { if ($tempAPIKey) { $this->removeAPIKey($tempAPIKey); } - if ($migrationDocument) { - $this->updateMigrationDocument($migrationDocument, $projectDocument); + if ($migration) { + $this->updateMigrationDocument($migration, $projectDocument); + $this->updateMigrationDocument($group, $projectDocument); - if ($migrationDocument->getAttribute('status', '') == 'failed') { - throw new Exception(implode("\n", $migrationDocument->getAttribute('errors', []))); + if ($migration->getAttribute('status', '') == 'failed') { + throw new Exception(implode("\n", $migration->getAttribute('errors', []))); } } } diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Migrations.php b/src/Appwrite/Utopia/Database/Validator/Queries/Migrations.php index 6b9e9e6d32..2d44b96f7c 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Migrations.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Migrations.php @@ -6,7 +6,6 @@ class Migrations extends Base { public const ALLOWED_ATTRIBUTES = [ 'status', - 'stage', 'source', 'resources', 'statusCounters', diff --git a/src/Appwrite/Utopia/Response/Model/Migration.php b/src/Appwrite/Utopia/Response/Model/Migration.php index 5a54eef3ad..d2f9725ae4 100644 --- a/src/Appwrite/Utopia/Response/Model/Migration.php +++ b/src/Appwrite/Utopia/Response/Model/Migration.php @@ -34,12 +34,6 @@ class Migration extends Model 'default' => '', 'example' => 'pending', ]) - ->addRule('stage', [ - 'type' => self::TYPE_STRING, - 'description' => 'Migration stage ( init, processing, source-check, destination-check, migrating, finished )', - 'default' => '', - 'example' => 'init', - ]) ->addRule('source', [ 'type' => self::TYPE_STRING, 'description' => 'A string containing the type of source of the migration.',