diff --git a/.env b/.env index defc91f4a7..88685e5e33 100644 --- a/.env +++ b/.env @@ -88,4 +88,6 @@ _APP_VCS_GITHUB_APP_ID= _APP_VCS_GITHUB_CLIENT_ID= _APP_VCS_GITHUB_CLIENT_SECRET= _APP_VCS_GITHUB_WEBHOOK_SECRET= +_APP_MIGRATIONS_FIREBASE_CLIENT_ID= +_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET= OPENAI_API_KEY= diff --git a/Dockerfile b/Dockerfile index 1b6578a029..7548ec70f4 100755 --- a/Dockerfile +++ b/Dockerfile @@ -180,6 +180,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-mails && \ chmod +x /usr/local/bin/worker-messaging && \ chmod +x /usr/local/bin/worker-webhooks && \ + chmod +x /usr/local/bin/worker-migrations && \ chmod +x /usr/local/bin/worker-usage # Letsencrypt Permissions diff --git a/app/config/collections.php b/app/config/collections.php index 2c2f2724b6..1d2d4e8741 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -2900,99 +2900,6 @@ $projectCollections = array_merge([ ], ], - 'variables' => [ - '$collection' => Database::METADATA, - '$id' => 'variables', - 'name' => 'variables', - 'attributes' => [ - [ - '$id' => 'functionInternalId', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'functionId', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'key', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'value', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 8192, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => false, - 'filters' => [ 'encrypt' ] - ], - [ - '$id' => ID::custom('search'), - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 16384, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - ], - 'indexes' => [ - [ - '$id' => '_key_function', - 'type' => Database::INDEX_KEY, - 'attributes' => ['functionInternalId'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => '_key_uniqueKey', - 'type' => Database::INDEX_UNIQUE, - 'attributes' => ['functionInternalId', 'key'], - 'lengths' => [Database::LENGTH_KEY, Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], - ], - [ - '$id' => '_key_key', - 'type' => Database::INDEX_KEY, - 'attributes' => ['key'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - [ - '$id' => ID::custom('_fulltext_search'), - 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['search'], - 'lengths' => [], - 'orders' => [], - ], - ], - ], - 'cache' => [ '$collection' => Database::METADATA, '$id' => 'cache', @@ -3167,6 +3074,143 @@ $projectCollections = array_merge([ ], ], ], + + 'migrations' => [ + '$collection' => ID::custom(Database::METADATA), + '$id' => ID::custom('migrations'), + 'name' => 'Migrations', + 'attributes' => [ + [ + '$id' => ID::custom('status'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('stage'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('source'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 8192, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('credentials'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 65536, + 'signed' => true, + 'required' => false, + 'default' => [], + 'array' => false, + 'filters' => ['json', 'encrypt'], + ], + [ + '$id' => ID::custom('resources'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => [], + 'array' => true, + 'filters' => [], + ], + [ + '$id' => ID::custom('statusCounters'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 3000, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => ['json'], + ], + [ + '$id' => ID::custom('resourceData'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 131070, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => ['json'], + ], + [ + '$id' => ID::custom('errors'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 65535, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => true, + 'filters' => [], + ], + [ + '$id' => ID::custom('search'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ] + ], + 'indexes' => [ + [ + '$id' => '_key_status', + 'type' => Database::INDEX_KEY, + 'attributes' => ['status'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_stage', + 'type' => Database::INDEX_KEY, + 'attributes' => ['stage'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_source', + 'type' => Database::INDEX_KEY, + 'attributes' => ['source'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => ID::custom('_fulltext_search'), + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['search'], + 'lengths' => [], + 'orders' => [], + ] + ], + ], ], $commonCollections); $consoleCollections = array_merge([ @@ -4979,7 +5023,7 @@ $dbCollections = [ 'orders' => [Database::ORDER_ASC], ], ], - ], + ] ]; diff --git a/app/config/errors.php b/app/config/errors.php index d5e76397ba..7133817d03 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -686,4 +686,21 @@ return [ 'description' => 'Too many queries.', 'code' => 400, ], + + /** Migrations */ + Exception::MIGRATION_NOT_FOUND => [ + 'name' => Exception::MIGRATION_NOT_FOUND, + 'description' => 'Migration with the requested ID could not be found.', + 'code' => 404, + ], + Exception::MIGRATION_ALREADY_EXISTS => [ + 'name' => Exception::MIGRATION_ALREADY_EXISTS, + 'description' => 'Migration with the requested ID already exists.', + 'code' => 409, + ], + Exception::MIGRATION_IN_PROGRESS => [ + 'name' => Exception::MIGRATION_IN_PROGRESS, + 'description' => 'Migration is already in progress.', + 'code' => 409, + ], ]; diff --git a/app/config/roles.php b/app/config/roles.php index fa1ce5806c..256766d3a4 100644 --- a/app/config/roles.php +++ b/app/config/roles.php @@ -52,7 +52,9 @@ $admins = [ 'execution.read', 'execution.write', 'rules.read', - 'rules.write' + 'rules.write', + 'migrations.read', + 'migrations.write', ]; return [ diff --git a/app/config/scopes.php b/app/config/scopes.php index a17a352122..f7b5a6a58e 100644 --- a/app/config/scopes.php +++ b/app/config/scopes.php @@ -82,4 +82,10 @@ return [ // List of publicly visible scopes 'rules.write' => [ 'description' => 'Access to create, update, and delete your project\'s proxy rules', ], + 'migrations.read' => [ + 'description' => 'Access to read your project\'s migrations', + ], + 'migrations.write' => [ + 'description' => 'Access to create, update, and delete your project\'s migrations.', + ] ]; diff --git a/app/config/services.php b/app/config/services.php index 6309baa69c..9709abd1a6 100644 --- a/app/config/services.php +++ b/app/config/services.php @@ -238,4 +238,17 @@ return [ 'optional' => false, 'icon' => '', ], + 'migrations' => [ + 'key' => 'migrations', + 'name' => 'Migrations', + 'subtitle' => 'The Migrations service allows you to migrate third-party data to your Appwrite server.', + 'description' => '/docs/services/migrations.md', + 'controller' => 'api/migrations.php', + 'sdk' => true, + 'docs' => true, + 'docsUrl' => 'https://appwrite.io/docs/migrations', + 'tests' => true, + 'optional' => true, + 'icon' => '/images/services/migrations.png', + ], ]; diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 9e6637ff58..e6d7a087e5 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -900,7 +900,7 @@ App::get('/v1/account/identities') $queries[] = Query::equal('userInternalId', [$user->getInternalId()]); // Get cursor document if there was a cursor query - $cursor = Query::getByType($queries, Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE); + $cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]); $cursor = reset($cursor); if ($cursor) { /** @var Query $cursor */ @@ -940,7 +940,7 @@ App::delete('/v1/account/identities/:identityId') ->label('sdk.description', '/docs/references/account/delete-identity.md') ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT) ->label('sdk.response.model', Response::MODEL_NONE) - ->param('identityId', [], new UID(), '') + ->param('identityId', [], new UID(), 'Identity ID.') ->inject('response') ->inject('dbForProject') ->action(function (string $identityId, Response $response, Database $dbForProject) { diff --git a/app/controllers/api/migrations.php b/app/controllers/api/migrations.php new file mode 100644 index 0000000000..8d961a0450 --- /dev/null +++ b/app/controllers/api/migrations.php @@ -0,0 +1,974 @@ +groups(['api', 'migrations']) + ->desc('Migrate Appwrite Data') + ->label('scope', 'migrations.write') + ->label('event', 'migrations.create') + ->label('audits.event', 'migration.create') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'createAppwriteMigration') + ->label('sdk.description', '/docs/references/migrations/migration-appwrite.md') + ->label('sdk.response.code', Response::STATUS_CODE_ACCEPTED) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_MIGRATION) + ->param('resources', [], new ArrayList(new WhiteList(Appwrite::getSupportedResources())), 'List of resources to migrate') + ->param('endpoint', '', new URL(), "Source's Appwrite Endpoint") + ->param('projectId', '', new UID(), "Source's Project ID") + ->param('apiKey', '', new Text(512), "Source's API Key") + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('user') + ->inject('events') + ->action(function (array $resources, string $endpoint, string $projectId, string $apiKey, Response $response, Database $dbForProject, Document $project, Document $user, Event $events) { + $migration = $dbForProject->createDocument('migrations', new Document([ + '$id' => ID::unique(), + 'status' => 'pending', + 'stage' => 'init', + 'source' => Appwrite::getName(), + 'credentials' => [ + 'endpoint' => $endpoint, + 'projectId' => $projectId, + 'apiKey' => $apiKey, + ], + 'resources' => $resources, + 'statusCounters' => '{}', + 'resourceData' => '{}', + 'errors' => [], + ])); + + $events->setParam('migrationId', $migration->getId()); + + // Trigger Transfer + $event = new Migration(); + $event + ->setMigration($migration) + ->setProject($project) + ->setUser($user) + ->trigger(); + + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($migration, Response::MODEL_MIGRATION); + }); + +App::post('/v1/migrations/firebase/oauth') + ->groups(['api', 'migrations']) + ->desc('Migrate Firebase Data (OAuth)') + ->label('scope', 'migrations.write') + ->label('event', 'migrations.create') + ->label('audits.event', 'migration.create') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'createFirebaseOAuthMigration') + ->label('sdk.description', '/docs/references/migrations/migration-firebase.md') + ->label('sdk.response.code', Response::STATUS_CODE_ACCEPTED) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_MIGRATION) + ->param('resources', [], new ArrayList(new WhiteList(Firebase::getSupportedResources())), 'List of resources to migrate') + ->param('projectId', '', new Text(65536), 'Project ID of the Firebase Project') + ->inject('response') + ->inject('dbForProject') + ->inject('dbForConsole') + ->inject('project') + ->inject('user') + ->inject('events') + ->inject('request') + ->action(function (array $resources, string $projectId, Response $response, Database $dbForProject, Database $dbForConsole, Document $project, Document $user, Event $events, Request $request) { + $firebase = new OAuth2Firebase( + App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''), + App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''), + $request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect' + ); + + $identity = $dbForConsole->findOne('identities', [ + Query::equal('provider', ['firebase']), + Query::equal('userInternalId', [$user->getInternalId()]), + ]); + if ($identity === false || $identity->isEmpty()) { + throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); + } + + $accessToken = $identity->getAttribute('providerAccessToken'); + $refreshToken = $identity->getAttribute('providerRefreshToken'); + $accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry'); + + $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now'); + if ($isExpired) { + $firebase->refreshTokens($refreshToken); + + $accessToken = $firebase->getAccessToken(''); + $refreshToken = $firebase->getRefreshToken(''); + + $verificationId = $firebase->getUserID($accessToken); + + if (empty($verificationId)) { + throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.'); + } + + $identity = $identity + ->setAttribute('providerAccessToken', $accessToken) + ->setAttribute('providerRefreshToken', $refreshToken) + ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry(''))); + + $dbForConsole->updateDocument('identities', $identity->getId(), $identity); + } + + if ($identity->getAttribute('secret')) { + $serviceAccount = $identity->getAttribute('secret'); + } else { + $serviceAccount = $firebase->createServiceAccount($accessToken, $projectId); + $identity = $identity + ->setAttribute('secret', $serviceAccount); + + $dbForConsole->updateDocument('identities', $identity->getId(), $identity); + } + + $migration = $dbForProject->createDocument('migrations', new Document([ + '$id' => ID::unique(), + 'status' => 'pending', + 'stage' => 'init', + 'source' => Firebase::getName(), + 'credentials' => [ + 'serviceAccount' => json_encode($serviceAccount), + ], + 'resources' => $resources, + 'statusCounters' => '{}', + 'resourceData' => '{}', + 'errors' => [] + ])); + + $events->setParam('migrationId', $migration->getId()); + + // Trigger Transfer + $event = new Migration(); + $event + ->setMigration($migration) + ->setProject($project) + ->setUser($user) + ->trigger(); + + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($migration, Response::MODEL_MIGRATION); + }); + +App::post('/v1/migrations/firebase') + ->groups(['api', 'migrations']) + ->desc('Migrate Firebase Data (Service Account)') + ->label('scope', 'migrations.write') + ->label('event', 'migrations.create') + ->label('audits.event', 'migration.create') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'createFirebaseMigration') + ->label('sdk.description', '/docs/references/migrations/migration-firebase.md') + ->label('sdk.response.code', Response::STATUS_CODE_ACCEPTED) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_MIGRATION) + ->param('resources', [], new ArrayList(new WhiteList(Firebase::getSupportedResources())), 'List of resources to migrate') + ->param('serviceAccount', '', new Text(65536), 'JSON of the Firebase service account credentials') + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('user') + ->inject('events') + ->action(function (array $resources, string $serviceAccount, Response $response, Database $dbForProject, Document $project, Document $user, Event $events) { + $migration = $dbForProject->createDocument('migrations', new Document([ + '$id' => ID::unique(), + 'status' => 'pending', + 'stage' => 'init', + 'source' => Firebase::getName(), + 'credentials' => [ + 'serviceAccount' => $serviceAccount, + ], + 'resources' => $resources, + 'statusCounters' => '{}', + 'resourceData' => '{}', + 'errors' => [], + ])); + + $events->setParam('migrationId', $migration->getId()); + + // Trigger Transfer + $event = new Migration(); + $event + ->setMigration($migration) + ->setProject($project) + ->setUser($user) + ->trigger(); + + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($migration, Response::MODEL_MIGRATION); + }); + +App::post('/v1/migrations/supabase') + ->groups(['api', 'migrations']) + ->desc('Migrate Supabase Data') + ->label('scope', 'migrations.write') + ->label('event', 'migrations.create') + ->label('audits.event', 'migration.create') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'createSupabaseMigration') + ->label('sdk.description', '/docs/references/migrations/migration-supabase.md') + ->label('sdk.response.code', Response::STATUS_CODE_ACCEPTED) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_MIGRATION) + ->param('resources', [], new ArrayList(new WhiteList(Supabase::getSupportedResources(), true)), 'List of resources to migrate') + ->param('endpoint', '', new URL(), 'Source\'s Supabase Endpoint') + ->param('apiKey', '', new Text(512), 'Source\'s API Key') + ->param('databaseHost', '', new Text(512), 'Source\'s Database Host') + ->param('username', '', new Text(512), 'Source\'s Database Username') + ->param('password', '', new Text(512), 'Source\'s Database Password') + ->param('port', 5432, new Integer(true), 'Source\'s Database Port', true) + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('user') + ->inject('events') + ->action(function (array $resources, string $endpoint, string $apiKey, string $databaseHost, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, Document $user, Event $events) { + $migration = $dbForProject->createDocument('migrations', new Document([ + '$id' => ID::unique(), + 'status' => 'pending', + 'stage' => 'init', + 'source' => Supabase::getName(), + 'credentials' => [ + 'endpoint' => $endpoint, + 'apiKey' => $apiKey, + 'databaseHost' => $databaseHost, + 'username' => $username, + 'password' => $password, + 'port' => $port, + ], + 'resources' => $resources, + 'statusCounters' => '{}', + 'resourceData' => '{}', + 'errors' => [], + ])); + + $events->setParam('migrationId', $migration->getId()); + + // Trigger Transfer + $event = new Migration(); + $event + ->setMigration($migration) + ->setProject($project) + ->setUser($user) + ->trigger(); + + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($migration, Response::MODEL_MIGRATION); + }); + +App::post('/v1/migrations/nhost') + ->groups(['api', 'migrations']) + ->desc('Migrate NHost Data') + ->label('scope', 'migrations.write') + ->label('event', 'migrations.create') + ->label('audits.event', 'migration.create') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'createNHostMigration') + ->label('sdk.description', '/docs/references/migrations/migration-nhost.md') + ->label('sdk.response.code', Response::STATUS_CODE_ACCEPTED) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_MIGRATION) + ->param('resources', [], new ArrayList(new WhiteList(NHost::getSupportedResources())), 'List of resources to migrate') + ->param('subdomain', '', new URL(), 'Source\'s Subdomain') + ->param('region', '', new Text(512), 'Source\'s Region') + ->param('adminSecret', '', new Text(512), 'Source\'s Admin Secret') + ->param('database', '', new Text(512), 'Source\'s Database Name') + ->param('username', '', new Text(512), 'Source\'s Database Username') + ->param('password', '', new Text(512), 'Source\'s Database Password') + ->param('port', 5432, new Integer(true), 'Source\'s Database Port', true) + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('user') + ->inject('events') + ->action(function (array $resources, string $subdomain, string $region, string $adminSecret, string $database, string $username, string $password, int $port, Response $response, Database $dbForProject, Document $project, Document $user, Event $events) { + $migration = $dbForProject->createDocument('migrations', new Document([ + '$id' => ID::unique(), + 'status' => 'pending', + 'stage' => 'init', + 'source' => NHost::getName(), + 'credentials' => [ + 'subdomain' => $subdomain, + 'region' => $region, + 'adminSecret' => $adminSecret, + 'database' => $database, + 'username' => $username, + 'password' => $password, + 'port' => $port, + ], + 'resources' => $resources, + 'statusCounters' => '{}', + 'resourceData' => '{}', + 'errors' => [], + ])); + + $events->setParam('migrationId', $migration->getId()); + + // Trigger Transfer + $event = new Migration(); + $event + ->setMigration($migration) + ->setProject($project) + ->setUser($user) + ->trigger(); + + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($migration, Response::MODEL_MIGRATION); + }); + +App::get('/v1/migrations') + ->groups(['api', 'migrations']) + ->desc('List Migrations') + ->label('scope', 'migrations.read') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'list') + ->label('sdk.description', '/docs/references/migrations/list-migrations.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_MIGRATION_LIST) + ->param('queries', [], new Migrations(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Migrations::ALLOWED_ATTRIBUTES), true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->inject('response') + ->inject('dbForProject') + ->action(function (array $queries, string $search, Response $response, Database $dbForProject) { + $queries = Query::parseQueries($queries); + + if (!empty($search)) { + $queries[] = Query::search('search', $search); + } + + // Get cursor document if there was a cursor query + $cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]); + $cursor = reset($cursor); + if ($cursor) { + /** @var Query $cursor */ + $migrationId = $cursor->getValue(); + $cursorDocument = $dbForProject->getDocument('migrations', $migrationId); + + if ($cursorDocument->isEmpty()) { + throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Migration '{$migrationId}' for the 'cursor' value not found."); + } + + $cursor->setValue($cursorDocument); + } + + $filterQueries = Query::groupByType($queries)['filters']; + + $response->dynamic(new Document([ + 'migrations' => $dbForProject->find('migrations', $queries), + 'total' => $dbForProject->count('migrations', $filterQueries, APP_LIMIT_COUNT), + ]), Response::MODEL_MIGRATION_LIST); + }); + +App::get('/v1/migrations/:migrationId') + ->groups(['api', 'migrations']) + ->desc('Get Migration') + ->label('scope', 'migrations.read') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'get') + ->label('sdk.description', '/docs/references/migrations/get-migration.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_MIGRATION) + ->param('migrationId', '', new UID(), 'Migration unique ID.') + ->inject('response') + ->inject('dbForProject') + ->action(function (string $migrationId, Response $response, Database $dbForProject) { + $migration = $dbForProject->getDocument('migrations', $migrationId); + + if ($migration->isEmpty()) { + throw new Exception(Exception::MIGRATION_NOT_FOUND); + } + + $response->dynamic($migration, Response::MODEL_MIGRATION); + }); + +App::get('/v1/migrations/appwrite/report') + ->groups(['api', 'migrations']) + ->desc('Generate a report on Appwrite Data') + ->label('scope', 'migrations.write') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'getAppwriteReport') + ->label('sdk.description', '/docs/references/migrations/migration-appwrite-report.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_MIGRATION_REPORT) + ->param('resources', [], new ArrayList(new WhiteList(Appwrite::getSupportedResources())), 'List of resources to migrate') + ->param('endpoint', '', new URL(), "Source's Appwrite Endpoint") + ->param('projectID', '', new Text(512), "Source's Project ID") + ->param('key', '', new Text(512), "Source's API Key") + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('user') + ->action(function (array $resources, string $endpoint, string $projectID, string $key, Response $response) { + try { + $appwrite = new Appwrite($projectID, $endpoint, $key); + + $response + ->setStatusCode(Response::STATUS_CODE_OK) + ->dynamic(new Document($appwrite->report($resources)), Response::MODEL_MIGRATION_REPORT); + } catch (\Exception $e) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage()); + } + }); + +App::get('/v1/migrations/firebase/report') + ->groups(['api', 'migrations']) + ->desc('Generate a report on Firebase Data') + ->label('scope', 'migrations.write') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'getFirebaseReport') + ->label('sdk.description', '/docs/references/migrations/migration-firebase-report.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_MIGRATION_REPORT) + ->param('resources', [], new ArrayList(new WhiteList(Firebase::getSupportedResources())), 'List of resources to migrate') + ->param('serviceAccount', '', new Text(65536), 'JSON of the Firebase service account credentials') + ->inject('response') + ->action(function (array $resources, string $serviceAccount, Response $response) { + try { + $firebase = new Firebase(json_decode($serviceAccount, true)); + + $response + ->setStatusCode(Response::STATUS_CODE_OK) + ->dynamic(new Document($firebase->report($resources)), Response::MODEL_MIGRATION_REPORT); + } catch (\Exception $e) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage()); + } + }); + +App::get('/v1/migrations/firebase/report/oauth') + ->groups(['api', 'migrations']) + ->desc('Generate a report on Firebase Data using OAuth') + ->label('scope', 'migrations.write') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'getFirebaseReportOAuth') + ->label('sdk.description', '/docs/references/migrations/migration-firebase-report.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_MIGRATION_REPORT) + ->param('resources', [], new ArrayList(new WhiteList(Firebase::getSupportedResources())), 'List of resources to migrate') + ->param('projectId', '', new Text(65536), 'Project ID') + ->inject('response') + ->inject('request') + ->inject('user') + ->inject('dbForConsole') + ->action(function (array $resources, string $projectId, Response $response, Request $request, Document $user, Database $dbForConsole) { + try { + $firebase = new OAuth2Firebase( + App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''), + App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''), + $request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect' + ); + + $identity = $dbForConsole->findOne('identities', [ + Query::equal('provider', ['firebase']), + Query::equal('userInternalId', [$user->getInternalId()]), + ]); + if ($identity === false || $identity->isEmpty()) { + throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); + } + + $accessToken = $identity->getAttribute('providerAccessToken'); + $refreshToken = $identity->getAttribute('providerRefreshToken'); + $accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry'); + + $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now'); + if ($isExpired) { + $firebase->refreshTokens($refreshToken); + + $accessToken = $firebase->getAccessToken(''); + $refreshToken = $firebase->getRefreshToken(''); + + $verificationId = $firebase->getUserID($accessToken); + + if (empty($verificationId)) { + throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.'); + } + + $identity = $identity + ->setAttribute('providerAccessToken', $accessToken) + ->setAttribute('providerRefreshToken', $refreshToken) + ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry(''))); + + $dbForConsole->updateDocument('identities', $identity->getId(), $identity); + } + + // Get Service Account + if ($identity->getAttribute('secret')) { + $serviceAccount = $identity->getAttribute('secret'); + } else { + $serviceAccount = $firebase->createServiceAccount($accessToken, $projectId); + $identity = $identity + ->setAttribute('secret', $serviceAccount); + + $dbForConsole->updateDocument('identities', $identity->getId(), $identity); + } + + $firebase = new Firebase(array_merge($serviceAccount, ['project_id' => $projectId])); + + $report = $firebase->report($resources); + + $response + ->setStatusCode(Response::STATUS_CODE_OK) + ->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT); + } catch (\Exception $e) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage()); + } + }); + +App::get('/v1/migrations/firebase/connect') + ->desc('Authorize with firebase') + ->groups(['api', 'migrations']) + ->label('scope', 'public') + ->label('origin', '*') + ->label('sdk.auth', [APP_AUTH_TYPE_SESSION]) + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'createFirebaseAuth') + ->label('sdk.description', '') + ->label('sdk.response.code', Response::STATUS_CODE_MOVED_PERMANENTLY) + ->label('sdk.response.type', Response::CONTENT_TYPE_HTML) + ->label('sdk.methodType', 'webAuth') + ->label('sdk.hide', true) + ->param('redirect', '', fn ($clients) => new Host($clients), 'URL to redirect back to your Firebase authorization. Only console hostnames are allowed.', true, ['clients']) + ->param('projectId', '', new UID(), 'Project ID') + ->inject('response') + ->inject('request') + ->inject('user') + ->inject('dbForConsole') + ->action(function (string $redirect, string $projectId, Response $response, Request $request, Document $user, Database $dbForConsole) { + $state = \json_encode([ + 'projectId' => $projectId, + 'redirect' => $redirect, + ]); + + $prefs = $user->getAttribute('prefs', []); + $prefs['migrationState'] = $state; + $user->setAttribute('prefs', $prefs); + $dbForConsole->updateDocument('users', $user->getId(), $user); + + $oauth2 = new OAuth2Firebase( + App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''), + App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''), + $request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect' + ); + $url = $oauth2->getLoginURL(); + + $response + ->addHeader('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') + ->addHeader('Pragma', 'no-cache') + ->redirect($url); + }); + +App::get('/v1/migrations/firebase/redirect') + ->desc('Capture and receive data on Firebase authorization') + ->groups(['api', 'migrations']) + ->label('scope', 'public') + ->label('error', __DIR__ . '/../../views/general/error.phtml') + ->param('code', '', new Text(2048), 'OAuth2 code.', true) + ->inject('user') + ->inject('project') + ->inject('request') + ->inject('response') + ->inject('dbForConsole') + ->action(function (string $code, Document $user, Document $project, Request $request, Response $response, Database $dbForConsole) { + $state = $user['prefs']['migrationState'] ?? '{}'; + $prefs['migrationState'] = ''; + $user->setAttribute('prefs', $prefs); + $dbForConsole->updateDocument('users', $user->getId(), $user); + + if (empty($state)) { + throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Installation requests from organisation members for the Appwrite Google App are currently unsupported.'); + } + + $state = \json_decode($state, true); + $redirect = $state['redirect'] ?? ''; + $projectId = $state['projectId'] ?? ''; + + $project = $dbForConsole->getDocument('projects', $projectId); + + if (empty($redirect)) { + $redirect = $request->getProtocol() . '://' . $request->getHostname() . '/console/project-$projectId/settings/migrations'; + } + + if ($project->isEmpty()) { + $response + ->addHeader('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') + ->addHeader('Pragma', 'no-cache') + ->redirect($redirect); + + return; + } + + // OAuth Authroization + if (!empty($code)) { + $oauth2 = new OAuth2Firebase( + App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''), + App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''), + $request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect' + ); + + $accessToken = $oauth2->getAccessToken($code); + $refreshToken = $oauth2->getRefreshToken($code); + $accessTokenExpiry = $oauth2->getAccessTokenExpiry($code); + $email = $oauth2->getUserEmail($accessToken); + $oauth2ID = $oauth2->getUserID($accessToken); + + if (empty($accessToken)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to get access token.'); + } + + if (empty($refreshToken)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to get refresh token.'); + } + + if (empty($accessTokenExpiry)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to get access token expiry.'); + } + + // Makes sure this email is not already used in another identity + $identity = $dbForConsole->findOne('identities', [ + Query::equal('providerEmail', [$email]), + ]); + + if ($identity !== false && !$identity->isEmpty()) { + if ($identity->getAttribute('userInternalId', '') !== $user->getInternalId()) { + throw new Exception(Exception::USER_EMAIL_ALREADY_EXISTS); + } + } + + if ($identity !== false && !$identity->isEmpty()) { + $identity = $identity + ->setAttribute('providerAccessToken', $accessToken) + ->setAttribute('providerRefreshToken', $refreshToken) + ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$accessTokenExpiry)); + + $dbForConsole->updateDocument('identities', $identity->getId(), $identity); + } else { + $identity = $dbForConsole->createDocument('identities', new Document([ + '$id' => ID::unique(), + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::user($user->getId())), + Permission::delete(Role::user($user->getId())), + ], + 'userInternalId' => $user->getInternalId(), + 'userId' => $user->getId(), + 'provider' => 'firebase', + 'providerUid' => $oauth2ID, + 'providerEmail' => $email, + 'providerAccessToken' => $accessToken, + 'providerRefreshToken' => $refreshToken, + 'providerAccessTokenExpiry' => DateTime::addSeconds(new \DateTime(), (int)$accessTokenExpiry), + ])); + } + } else { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Missing OAuth2 code.'); + } + + $response + ->addHeader('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') + ->addHeader('Pragma', 'no-cache') + ->redirect($redirect); + }); + +App::get('/v1/migrations/firebase/projects') + ->desc('List Firebase Projects') + ->groups(['api', 'migrations']) + ->label('scope', 'public') + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'listFirebaseProjects') + ->label('sdk.description', '') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_MIGRATION_FIREBASE_PROJECT_LIST) + ->inject('user') + ->inject('response') + ->inject('project') + ->inject('dbForConsole') + ->inject('request') + ->action(function (Document $user, Response $response, Document $project, Database $dbForConsole, Request $request) { + $firebase = new OAuth2Firebase( + App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''), + App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''), + $request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect' + ); + + $identity = $dbForConsole->findOne('identities', [ + Query::equal('provider', ['firebase']), + Query::equal('userInternalId', [$user->getInternalId()]), + ]); + if ($identity === false || $identity->isEmpty()) { + throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); + } + + $accessToken = $identity->getAttribute('providerAccessToken'); + $refreshToken = $identity->getAttribute('providerRefreshToken'); + $accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry'); + + if (empty($accessToken) || empty($refreshToken) || empty($accessTokenExpiry)) { + throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, 'Not authenticated with Firebase'); + } + + if (App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', '') === '' || App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', '') === '') { + throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, 'Missing Google OAuth credentials'); + } + + $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now'); + if ($isExpired) { + try { + $firebase->refreshTokens($refreshToken); + } catch (\Exception $e) { + throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, 'Failed to refresh Firebase access token'); + } + + $accessToken = $firebase->getAccessToken(''); + $refreshToken = $firebase->getRefreshToken(''); + + $verificationId = $firebase->getUserID($accessToken); + + if (empty($verificationId)) { + throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.'); + } + + $identity = $identity + ->setAttribute('providerAccessToken', $accessToken) + ->setAttribute('providerRefreshToken', $refreshToken) + ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry(''))); + + $dbForConsole->updateDocument('identities', $identity->getId(), $identity); + } + + $projects = $firebase->getProjects($accessToken); + + $output = []; + foreach ($projects as $project) { + $output[] = [ + 'displayName' => $project['displayName'], + 'projectId' => $project['projectId'], + ]; + } + + $response->dynamic(new Document([ + 'projects' => $output, + 'total' => count($output), + ]), Response::MODEL_MIGRATION_FIREBASE_PROJECT_LIST); + }); + +App::get('/v1/migrations/firebase/deauthorize') + ->desc('Revoke Appwrite\'s authorization to access Firebase Projects') + ->groups(['api', 'migrations']) + ->label('scope', 'public') + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'deleteFirebaseAuth') + ->label('sdk.description', '') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->inject('user') + ->inject('response') + ->inject('dbForConsole') + ->action(function (Document $user, Response $response, Database $dbForConsole) { + $identity = $dbForConsole->findOne('identities', [ + Query::equal('provider', ['firebase']), + Query::equal('userInternalId', [$user->getInternalId()]), + ]); + + if ($identity === false || $identity->isEmpty()) { + throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, 'Not authenticated with Firebase'); //TODO: Replace with USER_IDENTITY_NOT_FOUND + } + + $dbForConsole->deleteDocument('identities', $identity->getId()); + + $response->noContent(); + }); + +App::get('/v1/migrations/supabase/report') + ->groups(['api', 'migrations']) + ->desc('Generate a report on Supabase Data') + ->label('scope', 'migrations.write') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'getSupabaseReport') + ->label('sdk.description', '/docs/references/migrations/migration-supabase-report.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_MIGRATION_REPORT) + ->param('resources', [], new ArrayList(new WhiteList(Supabase::getSupportedResources(), true)), 'List of resources to migrate') + ->param('endpoint', '', new URL(), 'Source\'s Supabase Endpoint') + ->param('apiKey', '', new Text(512), 'Source\'s API Key') + ->param('databaseHost', '', new Text(512), 'Source\'s Database Host') + ->param('username', '', new Text(512), 'Source\'s Database Username') + ->param('password', '', new Text(512), 'Source\'s Database Password') + ->param('port', 5432, new Integer(true), 'Source\'s Database Port', true) + ->inject('response') + ->inject('dbForProject') + ->action(function (array $resources, string $endpoint, string $apiKey, string $databaseHost, string $username, string $password, int $port, Response $response) { + try { + $supabase = new Supabase($endpoint, $apiKey, $databaseHost, 'postgres', $username, $password, $port); + + $response + ->setStatusCode(Response::STATUS_CODE_OK) + ->dynamic(new Document($supabase->report($resources)), Response::MODEL_MIGRATION_REPORT); + } catch (\Exception $e) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage()); + } + }); + +App::get('/v1/migrations/nhost/report') + ->groups(['api', 'migrations']) + ->desc('Generate a report on NHost Data') + ->label('scope', 'migrations.write') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'getNHostReport') + ->label('sdk.description', '/docs/references/migrations/migration-nhost-report.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_MIGRATION_REPORT) + ->param('resources', [], new ArrayList(new WhiteList(NHost::getSupportedResources())), 'List of resources to migrate') + ->param('subdomain', '', new URL(), 'Source\'s Subdomain') + ->param('region', '', new Text(512), 'Source\'s Region') + ->param('adminSecret', '', new Text(512), 'Source\'s Admin Secret') + ->param('database', '', new Text(512), 'Source\'s Database Name') + ->param('username', '', new Text(512), 'Source\'s Database Username') + ->param('password', '', new Text(512), 'Source\'s Database Password') + ->param('port', 5432, new Integer(true), 'Source\'s Database Port', true) + ->inject('response') + ->action(function (array $resources, string $subdomain, string $region, string $adminSecret, string $database, string $username, string $password, int $port, Response $response) { + try { + $nhost = new NHost($subdomain, $region, $adminSecret, $database, $username, $password, $port); + + $response + ->setStatusCode(Response::STATUS_CODE_OK) + ->dynamic(new Document($nhost->report($resources)), Response::MODEL_MIGRATION_REPORT); + } catch (\Exception $e) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage()); + } + }); + +App::patch('/v1/migrations/:migrationId') + ->groups(['api', 'migrations']) + ->desc('Retry Migration') + ->label('scope', 'migrations.write') + ->label('event', 'migrations.[migrationId].retry') + ->label('audits.event', 'migration.retry') + ->label('audits.resource', 'migrations/{request.migrationId}') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'retry') + ->label('sdk.description', '/docs/references/migrations/retry-migration.md') + ->label('sdk.response.code', Response::STATUS_CODE_ACCEPTED) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_MIGRATION) + ->param('migrationId', '', new UID(), 'Migration unique ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('user') + ->inject('events') + ->action(function (string $migrationId, Response $response, Database $dbForProject, Document $project, Document $user, Event $eventInstance) { + $migration = $dbForProject->getDocument('migrations', $migrationId); + + if ($migration->isEmpty()) { + throw new Exception(Exception::MIGRATION_NOT_FOUND); + } + + if ($migration->getAttribute('status') !== 'failed') { + throw new Exception(Exception::MIGRATION_IN_PROGRESS, 'Migration not failed yet'); + } + + $migration + ->setAttribute('status', 'pending') + ->setAttribute('dateUpdated', \time()); + + // Trigger Migration + $event = new Migration(); + $event + ->setMigration($migration) + ->setProject($project) + ->setUser($user) + ->trigger(); + + $response->noContent(); + }); + +App::delete('/v1/migrations/:migrationId') + ->groups(['api', 'migrations']) + ->desc('Delete Migration') + ->label('scope', 'migrations.write') + ->label('event', 'migrations.[migrationId].delete') + ->label('audits.event', 'migrationId.delete') + ->label('audits.resource', 'migrations/{request.migrationId}') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'migrations') + ->label('sdk.method', 'delete') + ->label('sdk.description', '/docs/references/functions/delete-migration.md') + ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT) + ->label('sdk.response.model', Response::MODEL_NONE) + ->param('migrationId', '', new UID(), 'Migration ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('deletes') + ->inject('events') + ->action(function (string $migrationId, Response $response, Database $dbForProject, Delete $deletes, Event $events) { + $migration = $dbForProject->getDocument('migrations', $migrationId); + + if ($migration->isEmpty()) { + throw new Exception(Exception::MIGRATION_NOT_FOUND); + } + + if (!$dbForProject->deleteDocument('migrations', $migration->getId())) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove migration from DB', 500); + } + + $events->setParam('migrationId', $migration->getId()); + + $response->noContent(); + }); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 51dd292290..3d509ab476 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -10,6 +10,7 @@ use Appwrite\Network\Validator\Email; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Database\Validator\Queries\Identities; use Utopia\Database\Validator\Queries; +use Appwrite\Utopia\Database\Validator\Queries\Identities; use Appwrite\Utopia\Database\Validator\Queries\Users; use Utopia\Database\Validator\Query\Limit; use Utopia\Database\Validator\Query\Offset; @@ -656,7 +657,7 @@ App::get('/v1/users/identities') } // Get cursor document if there was a cursor query - $cursor = Query::getByType($queries, Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE); + $cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]); $cursor = reset($cursor); if ($cursor) { /** @var Query $cursor */ diff --git a/app/workers/migrations.php b/app/workers/migrations.php new file mode 100644 index 0000000000..9f324feda0 --- /dev/null +++ b/app/workers/migrations.php @@ -0,0 +1,308 @@ +args['type'] ?? ''; + $events = $this->args['events'] ?? []; + $project = new Document($this->args['project'] ?? []); + $user = new Document($this->args['user'] ?? []); + $payload = json_encode($this->args['payload'] ?? []); + + if ($project->getId() === 'console') { + return; + } + + /** + * Handle Event execution. + */ + if (! empty($events)) { + return; + } + + $this->dbForProject = $this->getProjectDB(new Document($this->args['project'])); + + $this->processMigration(); + } + + /** + * Process Source + * + * @return Source + * + * @throws \Exception + */ + protected function processSource(string $source, array $credentials): Source + { + switch ($source) { + case Firebase::getName(): + return new Firebase( + json_decode($credentials['serviceAccount'], true), + ); + break; + case Supabase::getName(): + return new Supabase( + $credentials['endpoint'], + $credentials['apiKey'], + $credentials['databaseHost'], + 'postgres', + $credentials['username'], + $credentials['password'], + $credentials['port'], + ); + break; + case NHost::getName(): + return new NHost( + $credentials['subdomain'], + $credentials['region'], + $credentials['adminSecret'], + $credentials['database'], + $credentials['username'], + $credentials['password'], + $credentials['port'], + ); + break; + case Appwrite::getName(): + return new Appwrite($credentials['projectId'], str_starts_with($credentials['endpoint'], 'http://localhost/v1') ? 'http://appwrite/v1' : $credentials['endpoint'], $credentials['apiKey']); + break; + default: + throw new \Exception('Invalid source type'); + break; + } + } + + protected function updateMigrationDocument(Document $migration, Document $project): Document + { + /** Trigger Realtime */ + $allEvents = Event::generateEvents('migrations.[migrationId].update', [ + 'migrationId' => $migration->getId(), + ]); + + $target = Realtime::fromPayload( + event: $allEvents[0], + payload: $migration, + project: $project + ); + + Realtime::send( + projectId: 'console', + payload: $migration->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'], + ); + + Realtime::send( + projectId: $project->getId(), + payload: $migration->getArrayCopy(), + events: $allEvents, + channels: $target['channels'], + roles: $target['roles'], + ); + + return $this->dbForProject->updateDocument('migrations', $migration->getId(), $migration); + } + + protected function removeAPIKey(Document $apiKey) + { + $consoleDB = $this->getConsoleDB(); + + $consoleDB->deleteDocument('keys', $apiKey->getId()); + } + + protected function generateAPIKey(Document $project): Document + { + $consoleDB = $this->getConsoleDB(); + $generatedSecret = bin2hex(\random_bytes(128)); + + $key = new Document([ + '$id' => ID::unique(), + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'projectInternalId' => $project->getInternalId(), + 'projectId' => $project->getId(), + 'name' => 'Transfer API Key', + 'scopes' => [ + 'users.read', + 'users.write', + 'teams.read', + 'teams.write', + 'databases.read', + 'databases.write', + 'collections.read', + 'collections.write', + 'documents.read', + 'documents.write', + 'buckets.read', + 'buckets.write', + 'files.read', + 'files.write', + 'functions.read', + 'functions.write', + ], + 'expire' => null, + 'sdks' => [], + 'accessedAt' => null, + 'secret' => $generatedSecret, + ]); + + $consoleDB->createDocument('keys', $key); + $consoleDB->deleteCachedDocument('projects', $project->getId()); + + return $key; + } + + /** + * Process Migration + * + * @return void + */ + protected function processMigration(): void + { + /** + * @var Document $migrationDocument + * @var Transfer $transfer + */ + $migrationDocument = null; + $transfer = null; + $projectDocument = $this->getConsoleDB()->getDocument('projects', $this->args['project']['$id']); + $tempAPIKey = $this->generateAPIKey($projectDocument); + + try { + $migrationDocument = $this->dbForProject->getDocument('migrations', $this->args['migration']['$id']); + $migrationDocument->setAttribute('stage', 'processing'); + $migrationDocument->setAttribute('status', 'processing'); + $this->updateMigrationDocument($migrationDocument, $projectDocument); + + $source = $this->processSource($migrationDocument->getAttribute('source'), $migrationDocument->getAttribute('credentials')); + + $source->report(); + + $destination = new DestinationsAppwrite( + $projectDocument->getId(), + 'http://appwrite/v1', + $tempAPIKey['secret'], + ); + + $transfer = new Transfer( + $source, + $destination + ); + + /** Start Transfer */ + $migrationDocument->setAttribute('stage', 'migrating'); + $this->updateMigrationDocument($migrationDocument, $projectDocument); + $transfer->run($migrationDocument->getAttribute('resources'), function () use ($migrationDocument, $transfer, $projectDocument) { + $migrationDocument->setAttribute('resourceData', json_encode($transfer->getCache())); + $migrationDocument->setAttribute('statusCounters', json_encode($transfer->getStatusCounters())); + + $this->updateMigrationDocument($migrationDocument, $projectDocument); + }); + + $errors = $transfer->getReport(Resource::STATUS_ERROR); + + if (count($errors) > 0) { + $migrationDocument->setAttribute('status', 'failed'); + $migrationDocument->setAttribute('stage', 'finished'); + + $errorMessages = []; + foreach ($errors as $error) { + $errorMessages[] = "Failed to transfer resource '{$error['id']}:{$error['resource']}' with message '{$error['message']}'"; + } + + $migrationDocument->setAttribute('errors', $errorMessages); + $this->updateMigrationDocument($migrationDocument, $projectDocument); + + return; + } + + $migrationDocument->setAttribute('status', 'completed'); + $migrationDocument->setAttribute('stage', 'finished'); + } catch (\Throwable $th) { + Console::error($th->getMessage()); + + if ($migrationDocument) { + Console::error($th->getMessage()); + Console::error($th->getTraceAsString()); + $migrationDocument->setAttribute('status', 'failed'); + $migrationDocument->setAttribute('stage', 'finished'); + $migrationDocument->setAttribute('errors', [$th->getMessage()]); + + return; + } + + if ($transfer) { + $errors = $transfer->getReport(Resource::STATUS_ERROR); + + if (count($errors) > 0) { + $migrationDocument->setAttribute('status', 'failed'); + $migrationDocument->setAttribute('stage', 'finished'); + $migrationDocument->setAttribute('errors', $errors); + } + } + } finally { + if ($migrationDocument) { + $this->updateMigrationDocument($migrationDocument, $projectDocument); + } + if ($tempAPIKey) { + $this->removeAPIKey($tempAPIKey); + } + } + } + + /** + * Process Verification + * + * @return void + */ + protected function processVerification(): void + { + } + + public function shutdown(): void + { + } +} diff --git a/bin/worker-migrations b/bin/worker-migrations new file mode 100644 index 0000000000..54e57001b0 --- /dev/null +++ b/bin/worker-migrations @@ -0,0 +1,10 @@ +#!/bin/sh + +if [ -z "$_APP_REDIS_USER" ] && [ -z "$_APP_REDIS_PASS" ] +then + REDIS_BACKEND="${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +else + REDIS_BACKEND="redis://${_APP_REDIS_USER}:${_APP_REDIS_PASS}@${_APP_REDIS_HOST}:${_APP_REDIS_PORT}" +fi + +INTERVAL=0.1 QUEUE='v1-migrations' APP_INCLUDE='/usr/src/code/app/workers/migrations.php' php /usr/src/code/vendor/bin/resque -dopcache.preload=opcache.preload=/usr/src/code/app/preload.php \ No newline at end of file diff --git a/composer.json b/composer.json index 2d639496f2..f56b1d8ead 100644 --- a/composer.json +++ b/composer.json @@ -75,7 +75,8 @@ "adhocore/jwt": "1.1.2", "webonyx/graphql-php": "14.11.*", "slickdeals/statsd": "3.1.0", - "league/csv": "9.7.1" + "league/csv": "9.7.1", + "utopia-php/migration": "^0.2.0" }, "repositories": [ { diff --git a/docker-compose.yml b/docker-compose.yml index 0839173a6a..fe9a058128 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -183,6 +183,8 @@ services: - _APP_VCS_GITHUB_WEBHOOK_SECRET - _APP_VCS_GITHUB_CLIENT_SECRET - _APP_VCS_GITHUB_CLIENT_ID + - _APP_MIGRATIONS_FIREBASE_CLIENT_ID + - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET appwrite-realtime: entrypoint: realtime @@ -613,6 +615,46 @@ services: - _APP_HAMSTER_TIME - _APP_MIXPANEL_TOKEN + appwrite-worker-migrations: + entrypoint: worker-migrations + <<: *x-logging + container_name: appwrite-worker-migrations + restart: unless-stopped + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + - ./tests:/usr/src/code/tests + - ./vendor:/usr/src/code/tests + depends_on: + - mariadb + environment: + - _APP_ENV + - _APP_OPENSSL_KEY_V1 + - _APP_DOMAIN + - _APP_DOMAIN_TARGET + - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_LOGGING_PROVIDER + - _APP_LOGGING_CONFIG + - _APP_MIGRATIONS_FIREBASE_CLIENT_ID + - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE + - _APP_CONNECTIONS_PUBSUB + appwrite-maintenance: entrypoint: maintenance <<: *x-logging diff --git a/docs/references/account/get-prefs.md b/docs/references/account/get-prefs.md index 8edfbb6883..867684dad7 100644 --- a/docs/references/account/get-prefs.md +++ b/docs/references/account/get-prefs.md @@ -1 +1 @@ -Get currently logged in user preferences as a key-value object. \ No newline at end of file +Get the preferences as a key-value object for the currently logged in user. \ No newline at end of file diff --git a/docs/references/account/get.md b/docs/references/account/get.md index f62b1f52bf..4dcae53117 100644 --- a/docs/references/account/get.md +++ b/docs/references/account/get.md @@ -1 +1 @@ -Get currently logged in user data as JSON object. \ No newline at end of file +Get the currently logged in user. \ No newline at end of file diff --git a/docs/references/account/list-identities.md b/docs/references/account/list-identities.md index fdb8c22b9d..4378bfb67a 100644 --- a/docs/references/account/list-identities.md +++ b/docs/references/account/list-identities.md @@ -1 +1 @@ -Get currently logged in user list of identities. \ No newline at end of file +Get the list of identities for the currently logged in user. \ No newline at end of file diff --git a/docs/references/account/list-logs.md b/docs/references/account/list-logs.md index 5cd2a4b87e..1f3a8d67a4 100644 --- a/docs/references/account/list-logs.md +++ b/docs/references/account/list-logs.md @@ -1 +1 @@ -Get currently logged in user list of latest security activity logs. Each log returns user IP address, location and date and time of log. \ No newline at end of file +Get the list of latest security activity logs for the currently logged in user. Each log returns user IP address, location and date and time of log. \ No newline at end of file diff --git a/docs/references/account/list-sessions.md b/docs/references/account/list-sessions.md index e944fd3675..f4eadf285c 100644 --- a/docs/references/account/list-sessions.md +++ b/docs/references/account/list-sessions.md @@ -1 +1 @@ -Get currently logged in user list of active sessions across different devices. \ No newline at end of file +Get the list of active sessions across different devices for the currently logged in user. \ No newline at end of file diff --git a/public/images/vcs/animation-building.gif b/public/images/vcs/animation-building.gif deleted file mode 100644 index 935d657903..0000000000 Binary files a/public/images/vcs/animation-building.gif and /dev/null differ diff --git a/public/images/vcs/state-failed.png b/public/images/vcs/state-failed.png deleted file mode 100644 index 0c4cb90adb..0000000000 Binary files a/public/images/vcs/state-failed.png and /dev/null differ diff --git a/public/images/vcs/state-success.png b/public/images/vcs/state-success.png deleted file mode 100644 index 2a759b01d1..0000000000 Binary files a/public/images/vcs/state-success.png and /dev/null differ diff --git a/public/images/vcs/state-waiting.png b/public/images/vcs/state-waiting.png deleted file mode 100644 index 505f981ac1..0000000000 Binary files a/public/images/vcs/state-waiting.png and /dev/null differ diff --git a/src/Appwrite/Auth/OAuth2/Exception.php b/src/Appwrite/Auth/OAuth2/Exception.php index 99408b8d64..28d8d652f9 100644 --- a/src/Appwrite/Auth/OAuth2/Exception.php +++ b/src/Appwrite/Auth/OAuth2/Exception.php @@ -16,9 +16,15 @@ class Exception extends AppwriteException $this->message = $response; $decoded = json_decode($response, true); if (\is_array($decoded)) { - $this->error = $decoded['error'] ?? $decoded['message'] ?? "Unknown error"; - $this->errorDescription = $decoded['error_description'] ?? "No description"; - $this->message = $this->error . ': ' . $this->errorDescription; + if (\is_array($decoded['error'] ?? '')) { + $this->error = $decoded['error']['status']; + $this->errorDescription = $decoded['error']['message']; + $this->message = $this->error . ': ' . $this->errorDescription; + } else { + $this->error = $decoded['error'] ?? $decoded['message'] ?? 'Unknown error'; + $this->errorDescription = $decoded['error_description'] ?? 'No description'; + $this->message = $this->error . ': ' . $this->errorDescription; + } } $type = match ($code) { 400 => AppwriteException::USER_OAUTH2_BAD_REQUEST, diff --git a/src/Appwrite/Auth/OAuth2/Firebase.php b/src/Appwrite/Auth/OAuth2/Firebase.php new file mode 100644 index 0000000000..0a813881be --- /dev/null +++ b/src/Appwrite/Auth/OAuth2/Firebase.php @@ -0,0 +1,270 @@ + 'offline', + 'client_id' => $this->appID, + 'redirect_uri' => $this->callback, + 'scope' => \implode(' ', $this->getScopes()), + 'state' => \json_encode($this->state), + 'response_type' => 'code', + 'prompt' => 'consent', + ]); + } + + /** + * @param string $code + * + * @return array + */ + protected function getTokens(string $code): array + { + if (empty($this->tokens)) { + $response = $this->request( + 'POST', + 'https://oauth2.googleapis.com/token', + [], + \http_build_query([ + 'client_id' => $this->appID, + 'redirect_uri' => $this->callback, + 'client_secret' => $this->appSecret, + 'code' => $code, + 'grant_type' => 'authorization_code' + ]) + ); + + $this->tokens = \json_decode($response, true); + } + + return $this->tokens; + } + + /** + * @param string $refreshToken + * + * @return array + */ + public function refreshTokens(string $refreshToken): array + { + $response = $this->request( + 'POST', + 'https://oauth2.googleapis.com/token', + [], + \http_build_query([ + 'client_id' => $this->appID, + 'client_secret' => $this->appSecret, + 'grant_type' => 'refresh_token', + 'refresh_token' => $refreshToken + ]) + ); + + $output = []; + \parse_str($response, $output); + $this->tokens = $output; + + if (empty($this->tokens['refresh_token'])) { + $this->tokens['refresh_token'] = $refreshToken; + } + + return $this->tokens; + } + + + /** + * @param string $accessToken + * + * @return string + */ + public function getUserID(string $accessToken): string + { + $user = $this->getUser($accessToken); + + return $user['id'] ?? ''; + } + + /** + * @param string $accessToken + * + * @return string + */ + public function getUserEmail(string $accessToken): string + { + $user = $this->getUser($accessToken); + + return $user['email'] ?? ''; + } + + + /** + * Check if the OAuth email is verified + * + * @link https://docs.github.com/en/rest/users/emails#list-email-addresses-for-the-authenticated-user + * + * @param string $accessToken + * + * @return bool + */ + public function isEmailVerified(string $accessToken): bool + { + $user = $this->getUser($accessToken); + + if ($user['verified'] ?? false) { + return true; + } + + return false; + } + + /** + * @param string $accessToken + * + * @return string + */ + public function getUserName(string $accessToken): string + { + $user = $this->getUser($accessToken); + + return $user['name'] ?? ''; + } + + /** + * @param string $accessToken + * + * @return array + */ + protected function getUser(string $accessToken) + { + if (empty($this->user)) { + $response = $this->request( + 'GET', + 'https://www.googleapis.com/oauth2/v1/userinfo?access_token=' . \urlencode($accessToken), + [], + ); + + $this->user = \json_decode($response, true); + } + + return $this->user; + } + + public function getProjects(string $accessToken): array + { + $projects = $this->request('GET', 'https://firebase.googleapis.com/v1beta1/projects', ['Authorization: Bearer ' . \urlencode($accessToken)]); + + $projects = \json_decode($projects, true); + + return $projects['results']; + } + + /* + Be careful with the setIAMPolicy method, it will overwrite all existing policies + **/ + public function assignIAMRoles(string $accessToken, string $email, string $projectId) + { + // Get IAM Roles + $iamRoles = $this->request('POST', 'https://cloudresourcemanager.googleapis.com/v1/projects/' . $projectId . ':getIamPolicy', [ + 'Authorization: Bearer ' . \urlencode($accessToken), + 'Content-Type: application/json' + ]); + + $iamRoles = \json_decode($iamRoles, true); + + $iamRoles['bindings'][] = [ + 'role' => 'roles/identitytoolkit.admin', + 'members' => [ + 'serviceAccount:' . $email + ] + ]; + + $iamRoles['bindings'][] = [ + 'role' => 'roles/firebase.admin', + 'members' => [ + 'serviceAccount:' . $email + ] + ]; + + // Set IAM Roles + $this->request('POST', 'https://cloudresourcemanager.googleapis.com/v1/projects/' . $projectId . ':setIamPolicy', [ + 'Authorization: Bearer ' . \urlencode($accessToken), + 'Content-Type: application/json' + ], json_encode([ + 'policy' => $iamRoles + ])); + } + + public function createServiceAccount(string $accessToken, string $projectId): array + { + // Create Service Account + $response = $this->request( + 'POST', + 'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts', + [ + 'Authorization: Bearer ' . \urlencode($accessToken), + 'Content-Type: application/json' + ], + json_encode([ + 'accountId' => 'appwrite-migrations', + 'serviceAccount' => [ + 'displayName' => 'Appwrite Migrations' + ] + ]) + ); + + $response = json_decode($response, true); + + $this->assignIAMRoles($accessToken, $response['email'], $projectId); + + // Create Service Account Key + $responseKey = $this->request( + 'POST', + 'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts/' . $response['email'] . '/keys', + [ + 'Authorization: Bearer ' . \urlencode($accessToken), + 'Content-Type: application/json' + ] + ); + + $responseKey = json_decode($responseKey, true); + + return json_decode(base64_decode($responseKey['privateKeyData']), true); + } +} diff --git a/src/Appwrite/Event/Event.php b/src/Appwrite/Event/Event.php index 0cf90eb3d3..23dde49637 100644 --- a/src/Appwrite/Event/Event.php +++ b/src/Appwrite/Event/Event.php @@ -38,6 +38,9 @@ class Event public const MESSAGING_QUEUE_NAME = 'v1-messaging'; public const MESSAGING_CLASS_NAME = 'MessagingV1'; + public const MIGRATIONS_QUEUE_NAME = 'v1-migrations'; + public const MIGRATIONS_CLASS_NAME = 'MigrationsV1'; + protected string $queue = ''; protected string $class = ''; protected string $event = ''; diff --git a/src/Appwrite/Event/Migration.php b/src/Appwrite/Event/Migration.php new file mode 100644 index 0000000000..4d53f16796 --- /dev/null +++ b/src/Appwrite/Event/Migration.php @@ -0,0 +1,98 @@ +migration = $migration; + + return $this; + } + + /** + * Returns set migration document for the function event. + * + * @return null|Document + */ + public function getMigration(): ?Document + { + return $this->migration; + } + + /** + * Sets migration type for the migration event. + * + * @param string $type + * + * @return self + */ + public function setType(string $type): self + { + $this->type = $type; + + return $this; + } + + /** + * Returns set migration type for the migration event. + * + * @return string + */ + public function getType(): string + { + return $this->type; + } + + /** + * Executes the migration event and sends it to the migrations worker. + * + * @return string|bool + * @throws \InvalidArgumentException + */ + public function trigger(): string|bool + { + return Resque::enqueue($this->queue, $this->class, [ + 'project' => $this->project, + 'user' => $this->user, + 'migration' => $this->migration + ]); + } + + /** + * Schedules the migration event and schedules it in the migrations worker queue. + * + * @param \DateTime|int $at + * @return void + * @throws \Resque_Exception + * @throws \ResqueScheduler_InvalidTimestampException + */ + public function schedule(DateTime|int $at): void + { + ResqueScheduler::enqueueAt($at, $this->queue, $this->class, [ + 'project' => $this->project, + 'user' => $this->user, + 'migration' => $this->migration + ]); + } +} diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index c28e360d36..4f467f0b1d 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -32,6 +32,7 @@ class Exception extends \Exception * - Platform * - Domain * - GraphQL + * - Migrations */ /** General */ @@ -213,6 +214,11 @@ class Exception extends \Exception public const GRAPHQL_NO_QUERY = 'graphql_no_query'; public const GRAPHQL_TOO_MANY_QUERIES = 'graphql_too_many_queries'; + /** Migrations */ + public const MIGRATION_NOT_FOUND = 'migration_not_found'; + public const MIGRATION_ALREADY_EXISTS = 'migration_already_exists'; + public const MIGRATION_IN_PROGRESS = 'migration_in_progress'; + protected $type = ''; protected $errors = []; diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Migrations.php b/src/Appwrite/Utopia/Database/Validator/Queries/Migrations.php new file mode 100644 index 0000000000..6b9e9e6d32 --- /dev/null +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Migrations.php @@ -0,0 +1,25 @@ +setModel(new BaseList('Status List', self::MODEL_HEALTH_STATUS_LIST, 'statuses', self::MODEL_HEALTH_STATUS)) ->setModel(new BaseList('Rule List', self::MODEL_PROXY_RULE_LIST, 'rules', self::MODEL_PROXY_RULE)) ->setModel(new BaseList('Locale codes list', self::MODEL_LOCALE_CODE_LIST, 'localeCodes', self::MODEL_LOCALE_CODE)) + ->setModel(new BaseList('Migrations List', self::MODEL_MIGRATION_LIST, 'migrations', self::MODEL_MIGRATION)) + ->setModel(new BaseList('Migrations Firebase Projects List', self::MODEL_MIGRATION_FIREBASE_PROJECT_LIST, 'projects', self::MODEL_MIGRATION_FIREBASE_PROJECT)) // Entities ->setModel(new Database()) ->setModel(new Collection()) @@ -395,6 +406,9 @@ class Response extends SwooleResponse ->setModel(new TemplateSMS()) ->setModel(new TemplateEmail()) ->setModel(new ConsoleVariables()) + ->setModel(new Migration()) + ->setModel(new MigrationReport()) + ->setModel(new MigrationFirebaseProject()) // Verification // Recovery // Tests (keep last) diff --git a/src/Appwrite/Utopia/Response/Model/Migration.php b/src/Appwrite/Utopia/Response/Model/Migration.php new file mode 100644 index 0000000000..0a8fc94a19 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/Migration.php @@ -0,0 +1,96 @@ +addRule('$id', [ + 'type' => self::TYPE_STRING, + 'description' => 'Migration ID.', + 'default' => '', + 'example' => '5e5ea5c16897e', + ]) + ->addRule('$createdAt', [ + 'type' => self::TYPE_DATETIME, + 'description' => 'Variable creation date in ISO 8601 format.', + 'default' => '', + 'example' => self::TYPE_DATETIME_EXAMPLE, + ]) + ->addRule('$updatedAt', [ + 'type' => self::TYPE_DATETIME, + 'description' => 'Variable creation date in ISO 8601 format.', + 'default' => '', + 'example' => self::TYPE_DATETIME_EXAMPLE, + ]) + ->addRule('status', [ + 'type' => self::TYPE_STRING, + 'description' => 'Migration status ( pending, processing, failed. completed ) ', + 'default' => '', + 'example' => 'pending', + ]) + ->addRule('stage', [ + 'type' => self::TYPE_STRING, + 'description' => 'Migration stage ( init, processing, source-check, destination-check, migrating, finished )', + 'default' => '', + 'example' => 'init', + ]) + ->addRule('source', [ + 'type' => self::TYPE_STRING, + 'description' => 'A string containing the type of source of the migration.', + 'default' => '', + 'example' => 'Appwrite', + ]) + ->addRule('resources', [ + 'type' => self::TYPE_STRING, + 'description' => 'Resources to migration.', + 'default' => [], + 'example' => ['user'], + 'array' => true + ]) + ->addRule('statusCounters', [ + 'type' => self::TYPE_JSON, + 'description' => 'A group of counters that represent the total progress of the migration.', + 'default' => [], + 'example' => '{"Database": {"PENDING": 0, "SUCCESS": 1, "ERROR": 0, "SKIP": 0, "PROCESSING": 0, "WARNING": 0}}', + ]) + ->addRule('resourceData', [ + 'type' => self::TYPE_JSON, + 'description' => 'An array of objects containing the report data of the resources that were migrated.', + 'default' => [], + 'example' => '[{"resource":"Database","id":"public","status":"SUCCESS","message":""}]', + ]) + ->addRule('errors', [ + 'type' => self::TYPE_STRING, + 'description' => 'All errors that occurred during the migration process.', + 'default' => [], + 'example' => [], + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'Migration'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_MIGRATION; + } +} diff --git a/src/Appwrite/Utopia/Response/Model/MigrationFirebaseProject.php b/src/Appwrite/Utopia/Response/Model/MigrationFirebaseProject.php new file mode 100644 index 0000000000..cf13c81483 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/MigrationFirebaseProject.php @@ -0,0 +1,46 @@ +addRule('projectId', [ + 'type' => self::TYPE_STRING, + 'description' => 'Project ID.', + 'default' => '', + 'example' => 'my-project', + ]) + ->addRule('displayName', [ + 'type' => self::TYPE_STRING, + 'description' => 'Project display name.', + 'default' => '', + 'example' => 'My Project', + ]); + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'MigrationFirebaseProject'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_MIGRATION_FIREBASE_PROJECT; + } +} diff --git a/src/Appwrite/Utopia/Response/Model/MigrationReport.php b/src/Appwrite/Utopia/Response/Model/MigrationReport.php new file mode 100644 index 0000000000..15164d56fc --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/MigrationReport.php @@ -0,0 +1,89 @@ +addRule(Resource::TYPE_USER, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of users to be migrated.', + 'default' => 0, + 'example' => 20, + ]) + ->addRule(Resource::TYPE_TEAM, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of teams to be migrated.', + 'default' => 0, + 'example' => 20, + ]) + ->addRule(Resource::TYPE_DATABASE, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of databases to be migrated.', + 'default' => 0, + 'example' => 20, + ]) + ->addRule(Resource::TYPE_DOCUMENT, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of documents to be migrated.', + 'default' => 0, + 'example' => 20, + ]) + ->addRule(Resource::TYPE_FILE, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of files to be migrated.', + 'default' => 0, + 'example' => 20, + ]) + ->addRule(Resource::TYPE_BUCKET, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of buckets to be migrated.', + 'default' => 0, + 'example' => 20, + ]) + ->addRule(Resource::TYPE_FUNCTION, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of functions to be migrated.', + 'default' => 0, + 'example' => 20, + ]) + ->addRule('size', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Size of files to be migrated in mb.', + 'default' => 0, + 'example' => 30000, + ]) + ->addRule('version', [ + 'type' => self::TYPE_STRING, + 'description' => 'Version of the Appwrite instance to be migrated.', + 'default' => '', + 'example' => '1.4.0', + ]); + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'Migration Report'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_MIGRATION_REPORT; + } +}