From 4e79259c4e81708634163b3cd4b216d02f1edd40 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 30 Oct 2025 12:27:52 +0530 Subject: [PATCH] Refactor database handling to streamline DSN retrieval for documents and tables databases --- app/config/collections/platform.php | 11 +++ app/controllers/api/projects.php | 92 ++++++++++++------- .../Platform/Modules/Databases/Constants.php | 3 + .../Databases/Http/Databases/Create.php | 58 +----------- .../Http/Databases/Transactions/Action.php | 18 +++- .../Http/Databases/Transactions/Update.php | 29 +++--- .../Http/DocumentsDB/Transactions/Update.php | 1 + .../Http/TablesDB/Transactions/Update.php | 1 + src/Appwrite/Platform/Workers/Migrations.php | 57 +----------- 9 files changed, 113 insertions(+), 157 deletions(-) diff --git a/app/config/collections/platform.php b/app/config/collections/platform.php index b839e51622..09ad724e1f 100644 --- a/app/config/collections/platform.php +++ b/app/config/collections/platform.php @@ -78,6 +78,17 @@ return [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('documentsDatabase'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => ID::custom('logo'), 'type' => Database::VAR_STRING, diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index d63ac72f0a..d60a367c4a 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -51,6 +51,65 @@ use Utopia\Validator\Text; use Utopia\Validator\URL; use Utopia\Validator\WhiteList; +function getDatabaseDSN(string $databasetype, $region): string +{ + $databases = []; + $databaseKeys = []; + /** + * @var string|null $databaseOverride + */ + $databaseOverride = ''; + $dbScheme = ''; + switch ($databasetype) { + case 'documentsDatabase': + $databases = Config::getParam('pools-documentsdb', []); + $databaseKeys = System::getEnv('_APP_DATABASE_DOCUMENTSDB_KEYS', ''); + $databaseOverride = System::getEnv('_APP_DATABASE_DOCUMENTSDB_OVERRIDE'); + $dbScheme = System::getEnv('_APP_DB_HOST_DOCUMENTSDB', 'mongodb'); + break; + default: + // legacy/tablesdb + $databases = Config::getParam('pools-database', []); + $databaseKeys = System::getEnv('_APP_DATABASE_KEYS', ''); + $databaseOverride = System::getEnv('_APP_DATABASE_OVERRIDE'); + $dbScheme = System::getEnv('_APP_DB_HOST', 'mysql'); + break; + } + + if ($region !== 'default') { + $keys = explode(',', $databaseKeys); + $databases = array_filter($keys, function ($value) use ($region) { + return str_contains($value, $region); + }); + } + + $index = \array_search($databaseOverride, $databases); + if ($index !== false) { + $dsn = $databases[$index]; + } else { + $dsn = $databases[array_rand($databases)]; + } + + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + if (\in_array($dsn, $sharedTables)) { + $schema = 'appwrite'; + $database = 'appwrite'; + $namespace = System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', ''); + $dsn = $schema . '://' . $dsn . '?database=' . $database; + + if (!empty($namespace)) { + $dsn .= '&namespace=' . $namespace; + } + } + try { + // for validation + new DSN($dsn); + } catch (\InvalidArgumentException) { + $dsn = $dbScheme.'://' . $dsn; + } + return $dsn; +} + App::init() ->groups(['projects']) ->inject('project') @@ -138,37 +197,7 @@ App::post('/v1/projects') throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project."); } - $databases = Config::getParam('pools-database', []); - - if ($region !== 'default') { - $databaseKeys = System::getEnv('_APP_DATABASE_KEYS', ''); - $keys = explode(',', $databaseKeys); - $databases = array_filter($keys, function ($value) use ($region) { - return str_contains($value, $region); - }); - } - - $databaseOverride = System::getEnv('_APP_DATABASE_OVERRIDE'); - $index = \array_search($databaseOverride, $databases); - if ($index !== false) { - $dsn = $databases[$index]; - } else { - $dsn = $databases[array_rand($databases)]; - } - - // TODO: Temporary until all projects are using shared tables. - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - if (\in_array($dsn, $sharedTables)) { - $schema = 'appwrite'; - $database = 'appwrite'; - $namespace = System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', ''); - $dsn = $schema . '://' . $dsn . '?database=' . $database; - - if (!empty($namespace)) { - $dsn .= '&namespace=' . $namespace; - } - } + $dsn = getDatabaseDSN('databases', $region); try { $project = $dbForPlatform->createDocument('projects', new Document([ @@ -203,6 +232,7 @@ App::post('/v1/projects') 'accessedAt' => DateTime::now(), 'search' => implode(' ', [$projectId, $name]), 'database' => $dsn, + 'documentsDatabase' => getDatabaseDSN('documentsDatabase', $region) ])); } catch (Duplicate) { throw new Exception(Exception::PROJECT_ALREADY_EXISTS); diff --git a/src/Appwrite/Platform/Modules/Databases/Constants.php b/src/Appwrite/Platform/Modules/Databases/Constants.php index cfc297c3f4..c8fe6e79bb 100644 --- a/src/Appwrite/Platform/Modules/Databases/Constants.php +++ b/src/Appwrite/Platform/Modules/Databases/Constants.php @@ -22,3 +22,6 @@ const INDEX = 'index'; const DOCUMENTS = 'document'; const ATTRIBUTES = 'attribute'; const COLLECTIONS = 'collection'; + +const TABLESDB = 'tablesdb'; +const DOCUMENTSDB = 'documentsdb'; diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php index cd45d7763f..6078e2d7d8 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php @@ -19,9 +19,7 @@ use Utopia\Database\Exception\Index as IndexException; use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Exception\Structure as StructureException; use Utopia\Database\Helpers\ID; -use Utopia\DSN\DSN; use Utopia\Swoole\Response as SwooleResponse; -use Utopia\System\System; use Utopia\Validator\Boolean; use Utopia\Validator\Text; @@ -34,58 +32,10 @@ class Create extends Action protected function getDatabaseDSN(Document $project): string { - $region = $project->getAttribute('region'); - $databases = []; - $databaseKeys = []; - /** - * @var string|null $databaseOverride - */ - $databaseOverride = ''; - $dbScheme = ''; - switch ($this->getDatabaseType()) { - case 'documentsdb': - $databases = Config::getParam('pools-documentsdb', []); - $databaseKeys = System::getEnv('_APP_DATABASE_DOCUMENTSDB_KEYS', ''); - $databaseOverride = System::getEnv('_APP_DATABASE_DOCUMENTSDB_OVERRIDE'); - $dbScheme = System::getEnv('_APP_DB_HOST_DOCUMENTSDB', 'mongodb'); - break; - default: - // legacy/tablesdb case where projects having the location of the database - return $project->getAttribute('database'); - } - - if ($region !== 'default') { - $keys = explode(',', $databaseKeys); - $databases = array_filter($keys, function ($value) use ($region) { - return str_contains($value, $region); - }); - } - - $index = \array_search($databaseOverride, $databases); - if ($index !== false) { - $dsn = $databases[$index]; - } else { - $dsn = $databases[array_rand($databases)]; - } - - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - if (\in_array($dsn, $sharedTables)) { - $schema = 'appwrite'; - $database = 'appwrite'; - $namespace = System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', ''); - $dsn = $schema . '://' . $dsn . '?database=' . $database; - - if (!empty($namespace)) { - $dsn .= '&namespace=' . $namespace; - } - } - try { - // for validation - new DSN($dsn); - } catch (\InvalidArgumentException) { - $dsn = $dbScheme.'://' . $dsn; - } - return $dsn; + return match ($this->getDatabaseType()) { + DOCUMENTSDB => $project->getAttribute('documentsDatabase'), + default => $project->getAttribute('database'), + }; } public function __construct() diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php index 8915ae6141..9c5f1f5f26 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Action.php @@ -10,11 +10,25 @@ abstract class Action extends UtopiaAction * The current API context (either 'table' or 'collection'). */ private ?string $context = COLLECTIONS; + private ?string $databaseType = DOCUMENTSDB; + + protected function getDatabaseType(): string + { + return $this->databaseType; + } public function setHttpPath(string $path): UtopiaAction { - if (\str_contains($path, '/tablesdb')) { - $this->context = TABLES; + switch (true) { + case str_contains($path, '/tablesdb'): + $this->context = TABLES; + $this->databaseType = TABLESDB; + break; + + case str_contains($path, '/documentsdb'): + $this->context = COLLECTIONS; + $this->databaseType = DOCUMENTSDB; + break; } return parent::setHttpPath($path); } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php index c7c994c062..b0bae11daa 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php @@ -66,6 +66,7 @@ class Update extends Action ->param('transactionId', '', new UID(), 'Transaction ID.') ->param('commit', false, new Boolean(), 'Commit transaction?', true) ->param('rollback', false, new Boolean(), 'Rollback transaction?', true) + ->inject('project') ->inject('response') ->inject('dbForProject') ->inject('getDatabasesDB') @@ -102,7 +103,7 @@ class Update extends Action * @throws \Utopia\Database\Exception * @throws StructureException */ - public function action(string $transactionId, bool $commit, bool $rollback, UtopiaResponse $response, Database $dbForProject, callable $getDatabasesDB, Document $user, TransactionState $transactionState, Delete $queueForDeletes, Event $queueForEvents, StatsUsage $queueForStatsUsage, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks): void + public function action(string $transactionId, bool $commit, bool $rollback, Document $project, UtopiaResponse $response, Database $dbForProject, callable $getDatabasesDB, Document $user, TransactionState $transactionState, Delete $queueForDeletes, Event $queueForEvents, StatsUsage $queueForStatsUsage, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks): void { if (!$commit && !$rollback) { throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Either commit or rollback must be true'); @@ -137,22 +138,7 @@ class Update extends Action $databaseOperations = []; try { - // transactions scoped to db wise - // taking a sample record to connect and start transactions - $operation = Authorization::skip(fn () => $dbForProject->findOne('transactionLogs', [ - Query::equal('transactionInternalId', [$transaction->getSequence()]) - ])); - - if ($operation->isEmpty()) { - // for transaction logs processing - $dbForDatabases = $dbForProject; - } else { - $databaseInternalId = $operation['databaseInternalId']; - $databaseDoc = Authorization::skip(fn () => $dbForProject->findOne('databases', [ - Query::equal('$sequence', [$databaseInternalId]) - ])); - $dbForDatabases = $getDatabasesDB($databaseDoc); - } + $dbForDatabases = $getDatabasesDB(new Document(['database' => $this->getDatabaseDSN($project)])); $dbForDatabases->withTransaction(function () use ($dbForProject, $transactionState, $queueForDeletes, $transactionId, &$transaction, &$operations, &$totalOperations, &$databaseOperations, $dbForDatabases) { Authorization::skip(fn () => $dbForProject->updateDocument('transactions', $transactionId, new Document([ 'status' => 'committing', @@ -420,6 +406,15 @@ class Update extends Action ->dynamic($transaction, UtopiaResponse::MODEL_TRANSACTION); } + private function getDatabaseDSN(Document $project) + { + var_dump($this->getDatabaseType()); + return match ($this->getDatabaseType()) { + DOCUMENTSDB => $project->getAttribute('documentsDatabase'), + default => $project->getAttribute('database'), + }; + } + /** * Handle create operation * diff --git a/src/Appwrite/Platform/Modules/Databases/Http/DocumentsDB/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/DocumentsDB/Transactions/Update.php index 4e7319ec88..f03f808276 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/DocumentsDB/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/DocumentsDB/Transactions/Update.php @@ -50,6 +50,7 @@ class Update extends TransactionsUpdate ->param('transactionId', '', new UID(), 'Transaction ID.') ->param('commit', false, new Boolean(), 'Commit transaction?', true) ->param('rollback', false, new Boolean(), 'Rollback transaction?', true) + ->inject('project') ->inject('response') ->inject('dbForProject') ->inject('getDatabasesDB') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php index 57ad219d41..237dc9cdb4 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php @@ -50,6 +50,7 @@ class Update extends TransactionsUpdate ->param('transactionId', '', new UID(), 'Transaction ID.') ->param('commit', false, new Boolean(), 'Commit transaction?', true) ->param('rollback', false, new Boolean(), 'Rollback transaction?', true) + ->inject('project') ->inject('response') ->inject('dbForProject') ->inject('getDatabasesDB') diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 765c1bc45e..c402092cf4 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -17,7 +17,6 @@ use Utopia\Database\Exception\Restricted; use Utopia\Database\Exception\Structure; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; -use Utopia\DSN\DSN; use Utopia\Locale\Locale; use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite; @@ -449,58 +448,10 @@ class Migrations extends Action protected function getDatabaseDSN(string $databaseType): string { - $databases = []; - $databaseKeys = []; - /** - * @var string|null $databaseOverride - */ - $databaseOverride = ''; - $dbScheme = ''; - $region = $this->project->getAttribute('region'); - switch ($databaseType) { - case 'documentsdb': - $databases = Config::getParam('pools-documentsdb', []); - $databaseKeys = System::getEnv('_APP_DATABASE_DOCUMENTSDB_KEYS', ''); - $databaseOverride = System::getEnv('_APP_DATABASE_DOCUMENTSDB_OVERRIDE'); - $dbScheme = System::getEnv('_APP_DB_HOST_DOCUMENTSDB', 'mongodb'); - break; - default: - // legacy/tablesdb case where projects having the location of the database - return $this->project->getAttribute('database'); - } - - if ($region !== 'default') { - $keys = explode(',', $databaseKeys); - $databases = array_filter($keys, function ($value) use ($region) { - return str_contains($value, $region); - }); - } - - $index = \array_search($databaseOverride, $databases); - if ($index !== false) { - $dsn = $databases[$index]; - } else { - $dsn = $databases[array_rand($databases)]; - } - - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - if (\in_array($dsn, $sharedTables)) { - $schema = 'appwrite'; - $database = 'appwrite'; - $namespace = System::getEnv('_APP_DATABASE_SHARED_NAMESPACE', ''); - $dsn = $schema . '://' . $dsn . '?database=' . $database; - - if (!empty($namespace)) { - $dsn .= '&namespace=' . $namespace; - } - } - try { - // for validation - new DSN($dsn); - } catch (\InvalidArgumentException) { - $dsn = $dbScheme.'://' . $dsn; - } - return $dsn; + return match ($databaseType) { + 'documentsdb' => $this->project->getAttribute('documentsDatabase'), + default => $this->project->getAttribute('database'), + }; } /** * Handle actions to be performed when a CSV export migration is successfully completed