Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix Migrations Stability

Browse Source
This commit is contained in:
Bradley Schofield
2023-08-16 16:01:56 +01:00
parent 3a6f10fa62
commit a0cd65b94b
5 changed files with 164 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/controllers/api/migrations.php
+111 -100
View File
@@ -34,7 +34,7 @@ App::post('/v1/migrations/appwrite')
->groups(['api', 'migrations'])
->desc('Migrate Appwrite Data')
->label('scope', 'migrations.write')
->label('event', 'migrations.create')
->label('event', 'migrations.[migrationId].create')
->label('audits.event', 'migration.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'migrations')
@@ -88,7 +88,7 @@ App::post('/v1/migrations/firebase/oauth')
->groups(['api', 'migrations'])
->desc('Migrate Firebase Data (OAuth)')
->label('scope', 'migrations.write')
->label('event', 'migrations.create')
->label('event', 'migrations.[migrationId].create')
->label('audits.event', 'migration.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'migrations')
@@ -146,12 +146,13 @@ App::post('/v1/migrations/firebase/oauth')
$dbForConsole->updateDocument('identities', $identity->getId(), $identity);
}
if ($identity->getAttribute('secret')) {
$serviceAccount = $identity->getAttribute('secret');
if ($identity->getAttribute('secrets')) {
$serviceAccount = $identity->getAttribute('secrets');
} else {
$firebase->cleanupServiceAccounts($accessToken, $projectId);
$serviceAccount = $firebase->createServiceAccount($accessToken, $projectId);
$identity = $identity
->setAttribute('secret', $serviceAccount);
->setAttribute('secrets', json_encode($serviceAccount));
$dbForConsole->updateDocument('identities', $identity->getId(), $identity);
}
@@ -189,7 +190,7 @@ App::post('/v1/migrations/firebase')
->groups(['api', 'migrations'])
->desc('Migrate Firebase Data (Service Account)')
->label('scope', 'migrations.write')
->label('event', 'migrations.create')
->label('event', 'migrations.[migrationId].create')
->label('audits.event', 'migration.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'migrations')
@@ -239,7 +240,7 @@ App::post('/v1/migrations/supabase')
->groups(['api', 'migrations'])
->desc('Migrate Supabase Data')
->label('scope', 'migrations.write')
->label('event', 'migrations.create')
->label('event', 'migrations.[migrationId].create')
->label('audits.event', 'migration.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'migrations')
@@ -299,7 +300,7 @@ App::post('/v1/migrations/nhost')
->groups(['api', 'migrations'])
->desc('Migrate NHost Data')
->label('scope', 'migrations.write')
->label('event', 'migrations.create')
->label('event', 'migrations.[migrationId].create')
->label('audits.event', 'migration.create')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'migrations')
@@ -309,7 +310,7 @@ App::post('/v1/migrations/nhost')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_MIGRATION)
->param('resources', [], new ArrayList(new WhiteList(NHost::getSupportedResources())), 'List of resources to migrate')
->param('subdomain', '', new URL(), 'Source\'s Subdomain')
->param('subdomain', '', new Text(512), 'Source\'s Subdomain')
->param('region', '', new Text(512), 'Source\'s Region')
->param('adminSecret', '', new Text(512), 'Source\'s Admin Secret')
->param('database', '', new Text(512), 'Source\'s Database Name')
@@ -501,67 +502,73 @@ App::get('/v1/migrations/firebase/report/oauth')
->inject('user')
->inject('dbForConsole')
->action(function (array $resources, string $projectId, Response $response, Request $request, Document $user, Database $dbForConsole) {
try {
$firebase = new OAuth2Firebase(
App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''),
App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''),
$request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect'
);
$firebase = new OAuth2Firebase(
App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''),
App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''),
$request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect'
);
$identity = $dbForConsole->findOne('identities', [
Query::equal('provider', ['firebase']),
Query::equal('userInternalId', [$user->getInternalId()]),
]);
if ($identity === false || $identity->isEmpty()) {
throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
}
$identity = $dbForConsole->findOne('identities', [
Query::equal('provider', ['firebase']),
Query::equal('userInternalId', [$user->getInternalId()]),
]);
$accessToken = $identity->getAttribute('providerAccessToken');
$refreshToken = $identity->getAttribute('providerRefreshToken');
$accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry');
$isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now');
if ($isExpired) {
$firebase->refreshTokens($refreshToken);
$accessToken = $firebase->getAccessToken('');
$refreshToken = $firebase->getRefreshToken('');
$verificationId = $firebase->getUserID($accessToken);
if (empty($verificationId)) {
throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.');
}
$identity = $identity
->setAttribute('providerAccessToken', $accessToken)
->setAttribute('providerRefreshToken', $refreshToken)
->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry('')));
$dbForConsole->updateDocument('identities', $identity->getId(), $identity);
}
// Get Service Account
if ($identity->getAttribute('secret')) {
$serviceAccount = $identity->getAttribute('secret');
} else {
$serviceAccount = $firebase->createServiceAccount($accessToken, $projectId);
$identity = $identity
->setAttribute('secret', $serviceAccount);
$dbForConsole->updateDocument('identities', $identity->getId(), $identity);
}
$firebase = new Firebase(array_merge($serviceAccount, ['project_id' => $projectId]));
$report = $firebase->report($resources);
$response
->setStatusCode(Response::STATUS_CODE_OK)
->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT);
} catch (\Exception $e) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage());
if ($identity === false || $identity->isEmpty()) {
throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
}
$accessToken = $identity->getAttribute('providerAccessToken');
$refreshToken = $identity->getAttribute('providerRefreshToken');
$accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry');
if (empty($accessToken) || empty($refreshToken) || empty($accessTokenExpiry)) {
throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
}
if (App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', '') === '' || App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', '') === '') {
throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
}
$isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now');
if ($isExpired) {
$firebase->refreshTokens($refreshToken);
$accessToken = $firebase->getAccessToken('');
$refreshToken = $firebase->getRefreshToken('');
$verificationId = $firebase->getUserID($accessToken);
if (empty($verificationId)) {
throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.');
}
$identity = $identity
->setAttribute('providerAccessToken', $accessToken)
->setAttribute('providerRefreshToken', $refreshToken)
->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry('')));
$dbForConsole->updateDocument('identities', $identity->getId(), $identity);
}
// Get Service Account
if ($identity->getAttribute('secrets')) {
$serviceAccount = $identity->getAttribute('secrets');
} else {
$firebase->cleanupServiceAccounts($accessToken, $projectId);
$serviceAccount = $firebase->createServiceAccount($accessToken, $projectId);
$identity = $identity
->setAttribute('secrets', json_encode($serviceAccount));
$dbForConsole->updateDocument('identities', $identity->getId(), $identity);
}
$firebase = new Firebase(array_merge($serviceAccount, ['project_id' => $projectId]));
$report = $firebase->report($resources);
$response
->setStatusCode(Response::STATUS_CODE_OK)
->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT);
});
App::get('/v1/migrations/firebase/connect')
@@ -745,6 +752,7 @@ App::get('/v1/migrations/firebase/projects')
Query::equal('provider', ['firebase']),
Query::equal('userInternalId', [$user->getInternalId()]),
]);
if ($identity === false || $identity->isEmpty()) {
throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
}
@@ -754,46 +762,50 @@ App::get('/v1/migrations/firebase/projects')
$accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry');
if (empty($accessToken) || empty($refreshToken) || empty($accessTokenExpiry)) {
throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, 'Not authenticated with Firebase');
throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
}
if (App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', '') === '' || App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', '') === '') {
throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, 'Missing Google OAuth credentials');
throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
}
$isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now');
if ($isExpired) {
try {
$firebase->refreshTokens($refreshToken);
} catch (\Exception $e) {
throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, 'Failed to refresh Firebase access token');
try {
$isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now');
if ($isExpired) {
try {
$firebase->refreshTokens($refreshToken);
} catch (\Exception $e) {
throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
}
$accessToken = $firebase->getAccessToken('');
$refreshToken = $firebase->getRefreshToken('');
$verificationId = $firebase->getUserID($accessToken);
if (empty($verificationId)) {
throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.');
}
$identity = $identity
->setAttribute('providerAccessToken', $accessToken)
->setAttribute('providerRefreshToken', $refreshToken)
->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry('')));
$dbForConsole->updateDocument('identities', $identity->getId(), $identity);
}
$accessToken = $firebase->getAccessToken('');
$refreshToken = $firebase->getRefreshToken('');
$projects = $firebase->getProjects($accessToken);
$verificationId = $firebase->getUserID($accessToken);
if (empty($verificationId)) {
throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.');
$output = [];
foreach ($projects as $project) {
$output[] = [
'displayName' => $project['displayName'],
'projectId' => $project['projectId'],
];
}
$identity = $identity
->setAttribute('providerAccessToken', $accessToken)
->setAttribute('providerRefreshToken', $refreshToken)
->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry('')));
$dbForConsole->updateDocument('identities', $identity->getId(), $identity);
}
$projects = $firebase->getProjects($accessToken);
$output = [];
foreach ($projects as $project) {
$output[] = [
'displayName' => $project['displayName'],
'projectId' => $project['projectId'],
];
} catch (\Exception $e) {
throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
}
$response->dynamic(new Document([
@@ -955,9 +967,8 @@ App::delete('/v1/migrations/:migrationId')
->param('migrationId', '', new UID(), 'Migration ID.')
->inject('response')
->inject('dbForProject')
->inject('deletes')
->inject('events')
->action(function (string $migrationId, Response $response, Database $dbForProject, Delete $deletes, Event $events) {
->action(function (string $migrationId, Response $response, Database $dbForProject, Event $events) {
$migration = $dbForProject->getDocument('migrations', $migrationId);
if ($migration->isEmpty()) {
app/controllers/general.php
+2 -2
View File
@@ -191,7 +191,7 @@ App::init()
$host = $request->getHostname() ?? '';
$mainDomain = App::getEnv('_APP_DOMAIN', '');
// Only run Router when external domain
if ($host !== $mainDomain && $host !== 'localhost' && $host !== 'appwrite') {
if ($host !== $mainDomain && $host !== 'localhost' && $host !== APP_INTERNAL_LOOPBACK_HOST) {
if (router($utopia, $dbForConsole, $swooleRequest, $request, $response)) {
return;
}
@@ -489,7 +489,7 @@ App::options()
$host = $request->getHostname() ?? '';
$mainDomain = App::getEnv('_APP_DOMAIN', '');
// Only run Router when external domain
if ($host !== $mainDomain && $host !== 'localhost' && $host !== 'appwrite') {
if ($host !== $mainDomain && $host !== 'localhost' && $host !== APP_INTERNAL_LOOPBACK_HOST) {
if (router($utopia, $dbForConsole, $swooleRequest, $request, $response)) {
return;
}
app/controllers/web/console.php
+1 -1
View File
@@ -34,7 +34,7 @@ App::get('/console/*')
// Serve static files (console) only for main domain
$host = $request->getHostname() ?? '';
$mainDomain = App::getEnv('_APP_DOMAIN', '');
if ($host !== $mainDomain && $host !== 'localhost' && $host !== 'appwrite') {
if ($host !== $mainDomain && $host !== 'localhost' && $host !== APP_INTERNAL_LOOPBACK_HOST) {
throw new Exception(Exception::GENERAL_ROUTE_NOT_FOUND);
}
app/init.php
+1
View File
@@ -135,6 +135,7 @@ const APP_SOCIAL_DISCORD_CHANNEL = '564160730845151244';
const APP_SOCIAL_DEV = 'https://dev.to/appwrite';
const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite';
const APP_SOCIAL_YOUTUBE = 'https://www.youtube.com/c/appwrite?sub_confirmation=1';
const APP_INTERNAL_LOOPBACK_HOST = 'appwrite';
// Database Reconnect
const DATABASE_RECONNECT_SLEEP = 2;
const DATABASE_RECONNECT_MAX_ATTEMPTS = 10;
src/Appwrite/Auth/OAuth2/Firebase.php
+49 -2
View File
@@ -231,9 +231,22 @@ class Firebase extends OAuth2
]));
}
function generateRandomString($length = 10)
{
$characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
$charactersLength = strlen($characters);
$randomString = '';
for ($i = 0; $i < $length; $i++) {
$randomString .= $characters[random_int(0, $charactersLength - 1)];
}
return $randomString;
}
public function createServiceAccount(string $accessToken, string $projectId): array
{
// Create Service Account
$uid = $this->generateRandomString();
$response = $this->request(
'POST',
'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts',
@@ -242,9 +255,9 @@ class Firebase extends OAuth2
'Content-Type: application/json'
],
json_encode([
'accountId' => 'appwrite-migrations',
'accountId' => 'appwrite-' . $uid,
'serviceAccount' => [
'displayName' => 'Appwrite Migrations'
'displayName' => 'Appwrite Migrations ' . $uid
]
])
);
@@ -267,4 +280,38 @@ class Firebase extends OAuth2
return json_decode(base64_decode($responseKey['privateKeyData']), true);
}
public function cleanupServiceAccounts(string $accessToken, string $projectId)
{
// List Service Accounts
$response = $this->request(
'GET',
'https://iam.googleapis.com/v1/projects/'.$projectId.'/serviceAccounts',
[
'Authorization: Bearer ' . \urlencode($accessToken),
'Content-Type: application/json'
]
);
$response = json_decode($response, true);
if (empty($response['accounts'])) {
return false;
}
foreach ($response['accounts'] as $account) {
if (strpos($account['email'], 'appwrite-') !== false) {
$this->request(
'DELETE',
'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts/' . $account['email'],
[
'Authorization: Bearer ' . \urlencode($accessToken),
'Content-Type: application/json'
]
);
}
}
return true;
}
}