diff --git a/app/config/collections.php b/app/config/collections.php index 96b09f210d..f24a5e0269 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -4018,6 +4018,17 @@ $projectCollections = array_merge([ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('stage'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => ID::custom('source'), 'type' => Database::VAR_STRING, @@ -4101,6 +4112,13 @@ $projectCollections = array_merge([ 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => '_key_stage', + 'type' => Database::INDEX_KEY, + 'attributes' => ['stage'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], [ '$id' => '_key_source', 'type' => Database::INDEX_KEY, @@ -4117,10 +4135,10 @@ $projectCollections = array_merge([ ] ], ], - 'groupMigrations' => [ + 'migrationsGroup' => [ '$collection' => ID::custom(Database::METADATA), - '$id' => ID::custom('groupMigrations'), - 'name' => 'groupMigrations', + '$id' => ID::custom('migrationsGroup'), + 'name' => 'migrationsGroup', 'attributes' => [ [ '$id' => ID::custom('migrationId'), diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php index bd5ee38e5c..46cd44b2ba 100644 --- a/app/controllers/api/migrations.php +++ b/app/controllers/api/migrations.php @@ -41,25 +41,24 @@ $triggerMigration = function (string $migrationId, array $resources, Migration $ ]; foreach ($groups as $group => $groupResources) { - if (!empty(array_intersect( + $filteredResources = array_intersect( $groupResources, $resources - ))) { + ); + + if (!empty($filteredResources)) { $groupDocument = new Document([ '$id' => ID::unique(), 'status' => 'pending', 'migrationId' => $migrationId, 'group' => $group, - 'resources' => array_intersect( - $groupResources, - $resources - ), + 'resources' => $filteredResources, 'statusCounters' => '', 'resourceData' => '', 'errors' => [] ]); - $dbForProject->createDocument('groupMigrations', $groupDocument); + $dbForProject->createDocument('migrationsGroup', $groupDocument); $queueForMigrations ->setType($group) @@ -96,6 +95,7 @@ App::post('/v1/migrations/appwrite') $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), 'status' => 'pending', + 'stage' => 'init', 'source' => Appwrite::getName(), 'credentials' => [ 'endpoint' => $endpoint, @@ -197,6 +197,7 @@ App::post('/v1/migrations/firebase/oauth') $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), 'status' => 'pending', + 'stage' => 'init', 'source' => Firebase::getName(), 'credentials' => [ 'serviceAccount' => json_encode($serviceAccount), @@ -256,6 +257,7 @@ App::post('/v1/migrations/firebase') $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), 'status' => 'pending', + 'stage' => 'init', 'source' => Firebase::getName(), 'credentials' => [ 'serviceAccount' => $serviceAccount, @@ -310,6 +312,7 @@ App::post('/v1/migrations/supabase') $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), 'status' => 'pending', + 'stage' => 'init', 'source' => Supabase::getName(), 'credentials' => [ 'endpoint' => $endpoint, @@ -370,6 +373,7 @@ App::post('/v1/migrations/nhost') $migration = $dbForProject->createDocument('migrations', new Document([ '$id' => ID::unique(), 'status' => 'pending', + 'stage' => 'init', 'source' => NHost::getName(), 'credentials' => [ 'subdomain' => $subdomain, diff --git a/app/init.php b/app/init.php index 414cbe0670..d6866f24a3 100644 --- a/app/init.php +++ b/app/init.php @@ -690,7 +690,7 @@ Database::addFilter( return; }, function (mixed $value, Document $document, Database $database) { - $groups = $database->find('groupMigrations', [ + $groups = $database->find('migrationsGroup', [ Query::equal('migrationId', [$document->getId()]), ]); @@ -712,7 +712,7 @@ Database::addFilter( return; }, function (mixed $value, Document $document, Database $database) { - $groups = $database->find('groupMigrations', [ + $groups = $database->find('migrationsGroup', [ Query::equal('migrationId', [$document->getId()]), ]); @@ -734,7 +734,7 @@ Database::addFilter( return; }, function (mixed $value, Document $document, Database $database) { - $groups = $database->find('groupMigrations', [ + $groups = $database->find('migrationsGroup', [ Query::equal('migrationId', [$document->getId()]), ]); diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 9043ab6de6..a6d4517a50 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -157,7 +157,7 @@ class Migrations extends Action roles: $target['roles'], ); - return $this->dbForProject->updateDocument($migration->getCollection(), $migration->getId(), $migration); + return $this->dbForProject->updateDocument('migrations', $migration->getId(), $migration); } /** @@ -249,17 +249,16 @@ class Migrations extends Action $tempAPIKey = $this->generateAPIKey($projectDocument); try { - $group = $this->dbForProject->getDocument('groupMigrations', $group->getId()); + $group = $this->dbForProject->getDocument('migrationsGroup', $group->getId()); $migration = $this->dbForProject->getDocument('migrations', $group->getAttribute('migrationId', '')); $migration->setAttribute('status', 'processing'); $log->addBreadcrumb(new Breadcrumb("debug", "migration", "Migration hit stage 'processing'", \microtime(true))); + $log->addTag('type', $migration->getAttribute('source')); $this->updateMigrationDocument($migration, $projectDocument); - $log->addTag('type', $migration->getAttribute('source')); - $source = $this->processSource($migration->getAttribute('source'), $migration->getAttribute('credentials')); $source->report(); @@ -275,26 +274,8 @@ class Migrations extends Action $destination ); - // Calculate group resources $resources = $group->getAttribute('resources'); - switch ($group->getAttribute('group')) { - case Transfer::GROUP_AUTH: - $resources = array_intersect(Transfer::GROUP_AUTH_RESOURCES, $resources); - break; - case Transfer::GROUP_STORAGE: - $resources = array_intersect(Transfer::GROUP_STORAGE_RESOURCES, $resources); - break; - case Transfer::GROUP_DATABASES: - $resources = array_intersect(Transfer::GROUP_DATABASES_RESOURCES, $resources); - break; - case Transfer::GROUP_FUNCTIONS: - $resources = array_intersect(Transfer::GROUP_FUNCTIONS_RESOURCES, $resources); - break; - default: - throw new Exception('Migration worker was initialized with unknown group'); - } - $log->addTag('migrationGroup', $group->getAttribute('group')); $log->addExtra('migrationResources', json_encode($resources)); $log->addBreadcrumb(new Breadcrumb("debug", "migration", "Migration hit stage 'migrating'", \microtime(true))); @@ -310,7 +291,6 @@ class Migrations extends Action $destinationErrors = $destination->getErrors(); if (!empty($sourceErrors) || !empty($destinationErrors)) { - $log->addBreadcrumb(new Breadcrumb("debug", "migration", "Migration hit stage 'finished' and failed", \microtime(true))); $migration->setAttribute('status', 'failed'); $errorMessages = []; @@ -326,6 +306,7 @@ class Migrations extends Action $group->setAttribute('errors', $errorMessages); $this->updateMigrationDocument($group, $projectDocument); $this->updateMigrationDocument($migration, $projectDocument); + $log->addBreadcrumb(new Breadcrumb("debug", "migration", "Migration hit stage 'finished' and failed", \microtime(true))); $log->addExtra('migrationErrors', json_encode($errorMessages)); return; @@ -336,7 +317,7 @@ class Migrations extends Action $this->updateMigrationDocument($group, $project); // Check if all other groups are finished, if so set parent document to completed aswell. - $groupDocuments = $this->dbForProject->find('groupMigrations', [Query::equal('migrationId', [$migration->getId()])]); + $groupDocuments = $this->dbForProject->find('migrationsGroup', [Query::equal('migrationId', [$migration->getId()])]); $result = 'completed'; foreach ($groupDocuments as $document) { @@ -346,15 +327,16 @@ class Migrations extends Action $status = $document->getAttribute('status', 'processing'); - if ($status == 'processing' || $status == 'pending') { + if ($status === 'processing' || $status === 'pending') { $result = 'processing'; break; } // Only fail parent if all have stopped processing. - if ($status == 'failed') { + if ($status === 'failed') { + $result = 'failed'; break; - } + } } $migration->setAttribute('status', $result);