Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor migrations to allow for more concurrency

Browse Source
This commit is contained in:
Bradley Schofield
2024-05-17 15:27:06 +09:00
parent 20bf989dca
commit eff09b665d
10 changed files with 419 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/config/collections.php
+127 -48
View File
@@ -4018,17 +4018,6 @@ $projectCollections = array_merge([
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('stage'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('source'),
'type' => Database::VAR_STRING,
@@ -4051,6 +4040,36 @@ $projectCollections = array_merge([
'array' => false,
'filters' => ['json', 'encrypt'],
],
[
'$id' => ID::custom('statusCounters'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 3000,
'signed' => true,
'required' => false,
'array' => false,
'filters' => ['subQueryStatusCounters'],
],
[
'$id' => ID::custom('resourceData'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 131070,
'signed' => true,
'required' => false,
'array' => false,
'filters' => ['subQueryResourceData'],
],
[
'$id' => ID::custom('errors'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 65535,
'signed' => true,
'required' => false,
'array' => true,
'filters' => ['subQueryMigrationErrors'],
],
[
'$id' => ID::custom('resources'),
'type' => Database::VAR_STRING,
@@ -4062,6 +4081,91 @@ $projectCollections = array_merge([
'array' => true,
'filters' => [],
],
[
'$id' => ID::custom('search'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 16384,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
]
],
'indexes' => [
[
'$id' => '_key_status',
'type' => Database::INDEX_KEY,
'attributes' => ['status'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_source',
'type' => Database::INDEX_KEY,
'attributes' => ['source'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_fulltext_search'),
'type' => Database::INDEX_FULLTEXT,
'attributes' => ['search'],
'lengths' => [],
'orders' => [],
]
],
],
'groupMigrations' => [
'$collection' => ID::custom(Database::METADATA),
'$id' => ID::custom('groupMigrations'),
'name' => 'groupMigrations',
'attributes' => [
[
'$id' => ID::custom('migrationId'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => []
],
[
'$id' => ID::custom('status'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => []
],
[
'$id' => ID::custom('resources'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => true,
'default' => null,
'array' => true,
'filters' => [],
],
[
'$id' => ID::custom('group'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => true,
'default' => null,
'array' => false,
'filters' => []
],
[
'$id' => ID::custom('statusCounters'),
'type' => Database::VAR_STRING,
@@ -4095,48 +4199,23 @@ $projectCollections = array_merge([
'array' => true,
'filters' => [],
],
[
'$id' => ID::custom('search'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => 16384,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
]
],
'indexes' => [
[
'$id' => '_key_status',
'type' => Database::INDEX_KEY,
'attributes' => ['status'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_stage',
'type' => Database::INDEX_KEY,
'attributes' => ['stage'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_source',
'type' => Database::INDEX_KEY,
'attributes' => ['source'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => ID::custom('_fulltext_search'),
'$id' => '_key_migrationId',
'type' => Database::INDEX_FULLTEXT,
'attributes' => ['search'],
'lengths' => [],
'orders' => [],
'attributes' => ['migrationId'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],
[
'$id' => '_key_group',
'type' => Database::INDEX_FULLTEXT,
'attributes' => ['group'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
]
],
]
],
], $commonCollections);
app/controllers/api/migrations.php
+99 -27
View File
@@ -21,6 +21,7 @@ use Utopia\Migration\Sources\Appwrite;
use Utopia\Migration\Sources\Firebase;
use Utopia\Migration\Sources\NHost;
use Utopia\Migration\Sources\Supabase;
use Utopia\Migration\Transfer;
use Utopia\System\System;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Host;
@@ -31,6 +32,84 @@ use Utopia\Validator\WhiteList;
include_once __DIR__ . '/../shared/api.php';
$triggerMigration = function (string $migrationId, array $resources, Migration $queueForMigrations) {
$groupDocument = new Document([
'$id' => ID::unique(),
'status' => 'pending',
'migrationId' => $migrationId,
'group' => '',
'resources' => [],
'statusCounters' => '',
'resourceData' => '',
'errors' => []
]);
if (!empty(array_intersect(
Transfer::GROUP_AUTH_RESOURCES,
$resources
))) {
$groupDocument->setAttribute('$id', ID::unique());
$groupDocument->setAttribute('group', Migration::TYPE_AUTH);
$groupDocument->setAttribute('resources', array_intersect(
Transfer::GROUP_AUTH_RESOURCES,
$resources
));
$queueForMigrations
->setType(Migration::TYPE_AUTH)
->setMigration($groupDocument)
->trigger();
}
if (!empty(array_intersect(
Transfer::GROUP_STORAGE_RESOURCES,
$resources
))) {
$groupDocument->setAttribute('$id', ID::unique());
$groupDocument->setAttribute('group', Migration::TYPE_STORAGE);
$groupDocument->setAttribute('resources', array_intersect(
Transfer::GROUP_STORAGE_RESOURCES,
$resources
));
$queueForMigrations
->setMigration($groupDocument)
->trigger();
}
if (!empty(array_intersect(
Transfer::GROUP_DATABASES_RESOURCES,
$resources
))) {
$groupDocument->setAttribute('$id', ID::unique());
$groupDocument->setAttribute('group', Migration::TYPE_DATABASES);
$groupDocument->setAttribute('resources', array_intersect(
Transfer::GROUP_DATABASES_RESOURCES,
$resources
));
$queueForMigrations
->setMigration($groupDocument)
->trigger();
}
if (!empty(array_intersect(
Transfer::GROUP_FUNCTIONS_RESOURCES,
$resources
))) {
$groupDocument->setAttribute('$id', ID::unique());
$groupDocument->setAttribute('group', Migration::TYPE_FUNCTIONS);
$groupDocument->setAttribute('resources', array_intersect(
Transfer::GROUP_FUNCTIONS_RESOURCES,
$resources
));
$queueForMigrations
->setMigration($groupDocument)
->trigger();
}
};
App::post('/v1/migrations/appwrite')
->groups(['api', 'migrations'])
->desc('Migrate Appwrite Data')
@@ -54,18 +133,16 @@ App::post('/v1/migrations/appwrite')
->inject('user')
->inject('queueForEvents')
->inject('queueForMigrations')
->action(function (array $resources, string $endpoint, string $projectId, string $apiKey, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) {
->action(function (array $resources, string $endpoint, string $projectId, string $apiKey, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) use ($triggerMigration) {
$migration = $dbForProject->createDocument('migrations', new Document([
'$id' => ID::unique(),
'status' => 'pending',
'stage' => 'init',
'source' => Appwrite::getName(),
'credentials' => [
'endpoint' => $endpoint,
'projectId' => $projectId,
'apiKey' => $apiKey,
],
'resources' => $resources,
'statusCounters' => '{}',
'resourceData' => '{}',
'errors' => [],
@@ -73,12 +150,11 @@ App::post('/v1/migrations/appwrite')
$queueForEvents->setParam('migrationId', $migration->getId());
// Trigger Transfer
$queueForMigrations
->setMigration($migration)
->setProject($project)
->setUser($user)
->trigger();
->setUser($user);
$triggerMigration($migration->getId(), $resources, $queueForMigrations);
$response
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
@@ -108,7 +184,7 @@ App::post('/v1/migrations/firebase/oauth')
->inject('queueForEvents')
->inject('queueForMigrations')
->inject('request')
->action(function (array $resources, string $projectId, Response $response, Database $dbForProject, Database $dbForConsole, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations, Request $request) {
->action(function (array $resources, string $projectId, Response $response, Database $dbForProject, Database $dbForConsole, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations, Request $request) use ($triggerMigration) {
$firebase = new OAuth2Firebase(
System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''),
System::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''),
@@ -162,7 +238,6 @@ App::post('/v1/migrations/firebase/oauth')
$migration = $dbForProject->createDocument('migrations', new Document([
'$id' => ID::unique(),
'status' => 'pending',
'stage' => 'init',
'source' => Firebase::getName(),
'credentials' => [
'serviceAccount' => json_encode($serviceAccount),
@@ -177,10 +252,10 @@ App::post('/v1/migrations/firebase/oauth')
// Trigger Transfer
$queueForMigrations
->setMigration($migration)
->setProject($project)
->setUser($user)
->trigger();
->setUser($user);
$triggerMigration($migration->getId(), $resources, $queueForMigrations);
$response
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
@@ -208,7 +283,7 @@ App::post('/v1/migrations/firebase')
->inject('user')
->inject('queueForEvents')
->inject('queueForMigrations')
->action(function (array $resources, string $serviceAccount, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) {
->action(function (array $resources, string $serviceAccount, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) use ($triggerMigration) {
$serviceAccountData = json_decode($serviceAccount, true);
if (empty($serviceAccountData)) {
@@ -222,7 +297,6 @@ App::post('/v1/migrations/firebase')
$migration = $dbForProject->createDocument('migrations', new Document([
'$id' => ID::unique(),
'status' => 'pending',
'stage' => 'init',
'source' => Firebase::getName(),
'credentials' => [
'serviceAccount' => $serviceAccount,
@@ -237,10 +311,10 @@ App::post('/v1/migrations/firebase')
// Trigger Transfer
$queueForMigrations
->setMigration($migration)
->setProject($project)
->setUser($user)
->trigger();
->setUser($user);
$triggerMigration($migration->getId(), $resources, $queueForMigrations);
$response
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
@@ -273,11 +347,10 @@ App::post('/v1/migrations/supabase')
->inject('user')
->inject('queueForEvents')
->inject('queueForMigrations')
->action(function (array $resources, string $endpoint, string $apiKey, string $databaseHost, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) {
->action(function (array $resources, string $endpoint, string $apiKey, string $databaseHost, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) use ($triggerMigration) {
$migration = $dbForProject->createDocument('migrations', new Document([
'$id' => ID::unique(),
'status' => 'pending',
'stage' => 'init',
'source' => Supabase::getName(),
'credentials' => [
'endpoint' => $endpoint,
@@ -297,10 +370,10 @@ App::post('/v1/migrations/supabase')
// Trigger Transfer
$queueForMigrations
->setMigration($migration)
->setProject($project)
->setUser($user)
->trigger();
->setUser($user);
$triggerMigration($migration->getId(), $resources, $queueForMigrations);
$response
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
@@ -334,11 +407,10 @@ App::post('/v1/migrations/nhost')
->inject('user')
->inject('queueForEvents')
->inject('queueForMigrations')
->action(function (array $resources, string $subdomain, string $region, string $adminSecret, string $database, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) {
->action(function (array $resources, string $subdomain, string $region, string $adminSecret, string $database, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Migration $queueForMigrations) use ($triggerMigration) {
$migration = $dbForProject->createDocument('migrations', new Document([
'$id' => ID::unique(),
'status' => 'pending',
'stage' => 'init',
'source' => NHost::getName(),
'credentials' => [
'subdomain' => $subdomain,
@@ -359,10 +431,10 @@ App::post('/v1/migrations/nhost')
// Trigger Transfer
$queueForMigrations
->setMigration($migration)
->setProject($project)
->setUser($user)
->trigger();
->setUser($user);
$triggerMigration($migration->getId(), $resources, $queueForMigrations);
$response
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
app/init.php
+66
View File
@@ -680,6 +680,72 @@ Database::addFilter(
}
);
Database::addFilter(
'subQueryStatusCounters',
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
$groups = $database->find('groupMigrations', [
Query::equal('migrationId', [$document->getId()]),
]);
$statusCounter = [];
foreach ($groups as $document) {
$data = $document->getAttribute('statusCounters');
$statusCounter = array_merge($statusCounter, $data);
}
return $statusCounter;
}
);
Database::addFilter(
'subQueryResourceData',
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
$groups = $database->find('groupMigrations', [
Query::equal('migrationId', [$document->getId()]),
]);
$resourceData = [];
foreach ($groups as $document) {
$data = $document->getAttribute('resourceData');
$resourceData = array_merge($resourceData, $data);
}
return $resourceData;
}
);
Database::addFilter(
'subQueryMigrationErrors',
function (mixed $value) {
return;
},
function (mixed $value, Document $document, Database $database) {
$groups = $database->find('groupMigrations', [
Query::equal('migrationId', [$document->getId()]),
]);
$errors = [];
foreach ($groups as $document) {
$data = $document->getAttribute('errors');
$errors = array_merge($errors, $data);
}
return $errors;
}
);
/**
* DB Formats
*/
composer.lock
Generated
+35 -36
View File
@@ -822,16 +822,16 @@
},
{
"name": "paragonie/constant_time_encoding",
"version": "v2.6.3",
"version": "v2.7.0",
"source": {
"type": "git",
"url": "https://github.com/paragonie/constant_time_encoding.git",
"reference": "58c3f47f650c94ec05a151692652a868995d2938"
"reference": "52a0d99e69f56b9ec27ace92ba56897fe6993105"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/58c3f47f650c94ec05a151692652a868995d2938",
"reference": "58c3f47f650c94ec05a151692652a868995d2938",
"url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/52a0d99e69f56b9ec27ace92ba56897fe6993105",
"reference": "52a0d99e69f56b9ec27ace92ba56897fe6993105",
"shasum": ""
},
"require": {
@@ -885,7 +885,7 @@
"issues": "https://github.com/paragonie/constant_time_encoding/issues",
"source": "https://github.com/paragonie/constant_time_encoding"
},
"time": "2022-06-14T06:56:20+00:00"
"time": "2024-05-08T12:18:48+00:00"
},
{
"name": "phpmailer/phpmailer",
@@ -1672,16 +1672,16 @@
},
{
"name": "utopia-php/dsn",
"version": "0.2.0",
"version": "0.2.1",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/dsn.git",
"reference": "c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7"
"reference": "42ee37a3d1785100b2f69091c9d4affadb6846eb"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/dsn/zipball/c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7",
"reference": "c11f37a12c3f6aaf9fea97ca7cb363dcc93668d7",
"url": "https://api.github.com/repos/utopia-php/dsn/zipball/42ee37a3d1785100b2f69091c9d4affadb6846eb",
"reference": "42ee37a3d1785100b2f69091c9d4affadb6846eb",
"shasum": ""
},
"require": {
@@ -1713,9 +1713,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/dsn/issues",
"source": "https://github.com/utopia-php/dsn/tree/0.2.0"
"source": "https://github.com/utopia-php/dsn/tree/0.2.1"
},
"time": "2023-11-02T12:01:43+00:00"
"time": "2024-05-07T02:01:25+00:00"
},
{
"name": "utopia-php/framework",
@@ -1966,22 +1966,21 @@
},
{
"name": "utopia-php/migration",
"version": "0.4.0",
"version": "0.4.4",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/migration.git",
"reference": "a72f27bd3dde68752fb185d306c4820e1b8d9657"
"reference": "a8a5d392bebf082faf289f4dfe09d9fd76844c33"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/a72f27bd3dde68752fb185d306c4820e1b8d9657",
"reference": "a72f27bd3dde68752fb185d306c4820e1b8d9657",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/a8a5d392bebf082faf289f4dfe09d9fd76844c33",
"reference": "a8a5d392bebf082faf289f4dfe09d9fd76844c33",
"shasum": ""
},
"require": {
"appwrite/appwrite": "10.1.0",
"php": "8.*",
"utopia-php/cli": "0.*"
"php": "8.*"
},
"require-dev": {
"laravel/pint": "1.*",
@@ -2008,9 +2007,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/migration/issues",
"source": "https://github.com/utopia-php/migration/tree/0.4.0"
"source": "https://github.com/utopia-php/migration/tree/0.4.4"
},
"time": "2024-02-25T12:35:21+00:00"
"time": "2024-05-17T05:25:31+00:00"
},
{
"name": "utopia-php/mongo",
@@ -2731,16 +2730,16 @@
"packages-dev": [
{
"name": "appwrite/sdk-generator",
"version": "0.38.2",
"version": "0.38.5",
"source": {
"type": "git",
"url": "https://github.com/appwrite/sdk-generator.git",
"reference": "51284668529e2b10ed933412a42b603c76cded23"
"reference": "830a46cc8e34ee096a76d4af6f00adf008a7cbf8"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/51284668529e2b10ed933412a42b603c76cded23",
"reference": "51284668529e2b10ed933412a42b603c76cded23",
"url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/830a46cc8e34ee096a76d4af6f00adf008a7cbf8",
"reference": "830a46cc8e34ee096a76d4af6f00adf008a7cbf8",
"shasum": ""
},
"require": {
@@ -2776,9 +2775,9 @@
"description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms",
"support": {
"issues": "https://github.com/appwrite/sdk-generator/issues",
"source": "https://github.com/appwrite/sdk-generator/tree/0.38.2"
"source": "https://github.com/appwrite/sdk-generator/tree/0.38.5"
},
"time": "2024-04-25T07:49:29+00:00"
"time": "2024-05-17T00:59:59+00:00"
},
{
"name": "doctrine/deprecations",
@@ -2899,16 +2898,16 @@
},
{
"name": "laravel/pint",
"version": "v1.15.2",
"version": "v1.15.3",
"source": {
"type": "git",
"url": "https://github.com/laravel/pint.git",
"reference": "2c9f8004899815f3f0ee3cb28ef7281e2b589134"
"reference": "3600b5d17aff52f6100ea4921849deacbbeb8656"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/laravel/pint/zipball/2c9f8004899815f3f0ee3cb28ef7281e2b589134",
"reference": "2c9f8004899815f3f0ee3cb28ef7281e2b589134",
"url": "https://api.github.com/repos/laravel/pint/zipball/3600b5d17aff52f6100ea4921849deacbbeb8656",
"reference": "3600b5d17aff52f6100ea4921849deacbbeb8656",
"shasum": ""
},
"require": {
@@ -2961,7 +2960,7 @@
"issues": "https://github.com/laravel/pint/issues",
"source": "https://github.com/laravel/pint"
},
"time": "2024-04-23T15:42:34+00:00"
"time": "2024-04-30T15:02:26+00:00"
},
{
"name": "matthiasmullie/minify",
@@ -3568,16 +3567,16 @@
},
{
"name": "phpstan/phpdoc-parser",
"version": "1.28.0",
"version": "1.29.0",
"source": {
"type": "git",
"url": "https://github.com/phpstan/phpdoc-parser.git",
"reference": "cd06d6b1a1b3c75b0b83f97577869fd85a3cd4fb"
"reference": "536889f2b340489d328f5ffb7b02bb6b183ddedc"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/cd06d6b1a1b3c75b0b83f97577869fd85a3cd4fb",
"reference": "cd06d6b1a1b3c75b0b83f97577869fd85a3cd4fb",
"url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/536889f2b340489d328f5ffb7b02bb6b183ddedc",
"reference": "536889f2b340489d328f5ffb7b02bb6b183ddedc",
"shasum": ""
},
"require": {
@@ -3609,9 +3608,9 @@
"description": "PHPDoc parser with support for nullable, intersection and generic types",
"support": {
"issues": "https://github.com/phpstan/phpdoc-parser/issues",
"source": "https://github.com/phpstan/phpdoc-parser/tree/1.28.0"
"source": "https://github.com/phpstan/phpdoc-parser/tree/1.29.0"
},
"time": "2024-04-03T18:51:33+00:00"
"time": "2024-05-06T12:04:23+00:00"
},
{
"name": "phpunit/php-code-coverage",
docker-compose.yml
+4 -3
View File
@@ -595,7 +595,7 @@ services:
appwrite-worker-migrations:
entrypoint: worker-migrations
<<: *x-logging
container_name: appwrite-worker-migrations
# container_name: appwrite-worker-migrations
restart: unless-stopped
image: appwrite-dev
networks:
@@ -624,8 +624,9 @@ services:
- _APP_DB_PASS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_MIGRATIONS_FIREBASE_CLIENT_ID
- _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET
deploy:
mode: replicated
replicas: 4
appwrite-task-maintenance:
entrypoint: maintenance
src/Appwrite/Event/Migration.php
+5
View File
@@ -11,6 +11,11 @@ class Migration extends Event
protected string $type = '';
protected ?Document $migration = null;
const TYPE_AUTH = 'Auth';
const TYPE_DATABASES = 'Databases';
const TYPE_STORAGE = 'Storage';
const TYPE_FUNCTIONS = 'Functions';
public function __construct(protected Connection $connection)
{
parent::__construct($connection);
src/Appwrite/Platform/Tasks/GetMigrationStats.php
-2
View File
@@ -25,7 +25,6 @@ class GetMigrationStats extends Action
'$id',
'$createdAt',
'status',
'stage',
'source'
];
@@ -119,7 +118,6 @@ class GetMigrationStats extends Action
$migration->getAttribute('$id'),
$migration->getAttribute('$createdAt'),
$migration->getAttribute('status'),
$migration->getAttribute('stage'),
$migration->getAttribute('source'),
];
}, $migrations);
src/Appwrite/Platform/Workers/Migrations.php
+83 -31
View File
@@ -15,7 +15,9 @@ use Utopia\Database\Exception\Conflict;
use Utopia\Database\Exception\Restricted;
use Utopia\Database\Exception\Structure;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Query;
use Utopia\Logger\Log;
use Utopia\Logger\Log\Breadcrumb;
use Utopia\Migration\Destinations\Appwrite as DestinationsAppwrite;
use Utopia\Migration\Exception as MigrationException;
use Utopia\Migration\Source;
@@ -161,7 +163,7 @@ class Migrations extends Action
roles: $target['roles'],
);
return $this->dbForProject->updateDocument('migrations', $migration->getId(), $migration);
return $this->dbForProject->updateDocument($migration->getCollection(), $migration->getId(), $migration);
}
/**
@@ -241,26 +243,28 @@ class Migrations extends Action
* @throws Structure
* @throws \Utopia\Database\Exception
*/
protected function processMigration(Document $project, Document $migration, Log $log): void
protected function processMigration(Document $project, Document $group, Log $log): void
{
/**
* @var Document $migrationDocument
* @var Document $migration
* @var Transfer $transfer
*/
$migrationDocument = null;
$groupDocument = null;
$transfer = null;
$projectDocument = $this->dbForConsole->getDocument('projects', $project->getId());
$tempAPIKey = $this->generateAPIKey($projectDocument);
try {
$migrationDocument = $this->dbForProject->getDocument('migrations', $migration->getId());
$migrationDocument->setAttribute('stage', 'processing');
$migrationDocument->setAttribute('status', 'processing');
$this->updateMigrationDocument($migrationDocument, $projectDocument);
$group = $this->dbForProject->getDocument('groupMigrations', $group->getId());
$log->addTag('type', $migrationDocument->getAttribute('source'));
$migration = $this->dbForProject->getDocument('migrations', $group->getAttribute('migrationId', ''));
$migration->setAttribute('status', 'processing');
$source = $this->processSource($migrationDocument->getAttribute('source'), $migrationDocument->getAttribute('credentials'));
$this->updateMigrationDocument($migration, $projectDocument);
$log->addTag('type', $migration->getAttribute('source'));
$source = $this->processSource($migration->getAttribute('source'), $migration->getAttribute('credentials'));
$source->report();
@@ -276,21 +280,43 @@ class Migrations extends Action
);
/** Start Transfer */
$migrationDocument->setAttribute('stage', 'migrating');
$this->updateMigrationDocument($migrationDocument, $projectDocument);
$transfer->run($migrationDocument->getAttribute('resources'), function () use ($migrationDocument, $transfer, $projectDocument) {
$migrationDocument->setAttribute('resourceData', json_encode($transfer->getCache()));
$migrationDocument->setAttribute('statusCounters', json_encode($transfer->getStatusCounters()));
$this->updateMigrationDocument($migration, $projectDocument);
$this->updateMigrationDocument($migrationDocument, $projectDocument);
// Calculate group resources
$resources = $group->getAttribute('resources');
switch ($group->getAttribute('group')) {
case Transfer::GROUP_AUTH:
$resources = array_intersect(Transfer::GROUP_AUTH_RESOURCES, $resources);
break;
case Transfer::GROUP_STORAGE:
$resources = array_intersect(Transfer::GROUP_STORAGE_RESOURCES, $resources);
break;
case Transfer::GROUP_DATABASES:
$resources = array_intersect(Transfer::GROUP_DATABASES_RESOURCES, $resources);
break;
case Transfer::GROUP_FUNCTIONS:
$resources = array_intersect(Transfer::GROUP_FUNCTIONS_RESOURCES, $resources);
break;
default:
throw new Exception('Migration worker was initialized with unknown group');
}
$log->addTag('migrationGroup', $group->getAttribute('group'));
$log->addExtra('migrationResources', json_encode($resources));
$transfer->run($resources, function () use ($group, $transfer, $projectDocument) {
$group->setAttribute('resourceData', json_encode($transfer->getCache()));
$group->setAttribute('statusCounters', json_encode($transfer->getStatusCounters()));
$this->updateMigrationDocument($group, $projectDocument);
});
$sourceErrors = $source->getErrors();
$destinationErrors = $destination->getErrors();
if (!empty($sourceErrors) || !empty($destinationErrors)) {
$migrationDocument->setAttribute('status', 'failed');
$migrationDocument->setAttribute('stage', 'finished');
$migration->setAttribute('status', 'failed');
$errorMessages = [];
foreach ($sourceErrors as $error) {
@@ -302,23 +328,48 @@ class Migrations extends Action
$errorMessages[] = "Error occurred while pushing '{$error->getResourceType()}:{$error->getResourceId()}' to destination with message: '{$error->getMessage()}'";
}
$migrationDocument->setAttribute('errors', $errorMessages);
$this->updateMigrationDocument($migrationDocument, $projectDocument);
$group->setAttribute('errors', $errorMessages);
$this->updateMigrationDocument($group, $projectDocument);
$this->updateMigrationDocument($migration, $projectDocument);
return;
}
$migrationDocument->setAttribute('status', 'completed');
$migrationDocument->setAttribute('stage', 'finished');
$group->setAttribute('status', 'completed');
$this->updateMigrationDocument($group, $project);
// Check if all other groups are finished, if so set parent document to completed aswell.
$groupDocuments = $this->dbForProject->find('groupMigrations', [Query::equal('migrationId', [$migration->getId()])]);
$result = 'completed';
foreach ($groupDocuments as $document) {
if ($document->getId() == $group->getId()) {
continue;
}
$status = $document->getAttribute('status', 'processing');
if ($status == 'processing' || $status == 'pending') {
$result = 'processing';
break;
}
// Only fail parent if all have stopped processing.
if ($status == 'failed') {
break;
}
}
$migration->setAttribute('status', $result);
$this->updateMigrationDocument($migration, $project);
} catch (\Throwable $th) {
Console::error($th->getMessage());
if ($migrationDocument) {
if ($migration && $groupDocument) {
Console::error($th->getMessage());
Console::error($th->getTraceAsString());
$migrationDocument->setAttribute('status', 'failed');
$migrationDocument->setAttribute('stage', 'finished');
$migrationDocument->setAttribute('errors', [$th->getMessage()]);
$migration->setAttribute('status', 'failed');
$groupDocument->setAttribute('errors', [$th->getMessage()]);
return;
}
@@ -337,17 +388,18 @@ class Migrations extends Action
$errorMessages[] = "Error occurred while pushing '{$error->getResourceType()}:{$error->getResourceId()}' to destination with message '{$error->getMessage()}'";
}
$migrationDocument->setAttribute('errors', $errorMessages);
$group->setAttribute('errors', $errorMessages);
}
} finally {
if ($tempAPIKey) {
$this->removeAPIKey($tempAPIKey);
}
if ($migrationDocument) {
$this->updateMigrationDocument($migrationDocument, $projectDocument);
if ($migration) {
$this->updateMigrationDocument($migration, $projectDocument);
$this->updateMigrationDocument($group, $projectDocument);
if ($migrationDocument->getAttribute('status', '') == 'failed') {
throw new Exception(implode("\n", $migrationDocument->getAttribute('errors', [])));
if ($migration->getAttribute('status', '') == 'failed') {
throw new Exception(implode("\n", $migration->getAttribute('errors', [])));
}
}
}
src/Appwrite/Utopia/Database/Validator/Queries/Migrations.php
-1
View File
@@ -6,7 +6,6 @@ class Migrations extends Base
{
public const ALLOWED_ATTRIBUTES = [
'status',
'stage',
'source',
'resources',
'statusCounters',
src/Appwrite/Utopia/Response/Model/Migration.php
-6
View File
@@ -34,12 +34,6 @@ class Migration extends Model
'default' => '',
'example' => 'pending',
])
->addRule('stage', [
'type' => self::TYPE_STRING,
'description' => 'Migration stage ( init, processing, source-check, destination-check, migrating, finished )',
'default' => '',
'example' => 'init',
])
->addRule('source', [
'type' => self::TYPE_STRING,
'description' => 'A string containing the type of source of the migration.',