diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 8346683187..dcb83a1665 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -86,7 +86,7 @@ App::post('/v1/projects') throw new Exception("'console' is a reserved project.", 400, Exception::PROJECT_RESERVED_PROJECT); } - ['name' => $dbName, 'db' => $projectDB] = $dbPool->getAny(); + ['name' => $dbName, 'db' => $projectDB] = $dbPool->getAnyFromPool(); $project = $dbForConsole->createDocument('projects', new Document([ '$id' => $projectId, diff --git a/app/http.php b/app/http.php index df2dcc1c53..61dc90490d 100644 --- a/app/http.php +++ b/app/http.php @@ -17,9 +17,6 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Swoole\Files; use Appwrite\Utopia\Request; -use Utopia\Cache\Cache; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Database\Adapter\MariaDB; use Utopia\Logger\Log; use Utopia\Logger\Log\User; @@ -69,7 +66,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { do { try { $attempts++; - $consoleDB = $register->get('dbPool')->getConsoleDB(); + $consoleDB = $register->get('dbPool')->getConsoleDBFromPool(); $redis = $register->get('redisPool')->get(); break; // leave the do-while if successful } catch (\Exception $e) { @@ -243,11 +240,11 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $app = new App('UTC'); $dbPool = $register->get('dbPool'); - $consoleDB = $dbPool->getConsoleDB(); + $consoleDB = $dbPool->getConsoleDBFromPool(); $redis = $register->get('redisPool')->get(); - App::setResource('consoleDB', fn() => $consoleDB); App::setResource('dbPool', fn() => $dbPool); + App::setResource('consoleDB', fn() => $consoleDB); App::setResource('cache', fn() => $redis); $projectId = $request->getParam('project', $request->getHeader('x-appwrite-project', 'console')); @@ -257,7 +254,7 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $project = Authorization::skip(fn() => $dbForConsole->getDocument('projects', $projectId)); $dbName = $project->getAttribute('database', ''); if (!empty($dbName)) { - $projectDB = $register->get('dbPool')->get($dbName); + $projectDB = $dbPool->getDBFromPool($dbName); } } diff --git a/app/init.php b/app/init.php index e104f41b54..bd1f40a1f5 100644 --- a/app/init.php +++ b/app/init.php @@ -445,72 +445,28 @@ $register->set('logger', function () { return new Logger($adapter); }); -$register->set('dbMap', function () { - $dbs = App::getEnv('_APP_PROJECT_DB', ''); - $dbs = explode(',', $dbs); - - $dbMap = []; - foreach ($dbs as $db) { - $db = explode('=', $db); - $name = $db[0]; - $dsn = $db[1]; - $dbMap[$name] = $dsn; - } - - return $dbMap; -}); $register->set('dbPool', function () { /** Parse the console databases */ - $consoleDb = App::getEnv('_APP_CONSOLE_DB', ''); - $consoleDb = explode(',', $consoleDb)[0]; - $consoleDb = explode('=', $consoleDb); - $name = $consoleDb[0]; - $dsn = new DSN($consoleDb[1]); - - /** Create a new Database Pool */ - $pool = new DatabasePool(); - - $consolePool = new PDOPool( - (new PDOConfig()) - ->withHost($dsn->getHost()) - ->withPort($dsn->getPort()) - ->withDbName($dsn->getDatabase()) - ->withCharset('utf8mb4') - ->withUsername($dsn->getUser()) - ->withPassword($dsn->getPassword()) - ->withOptions([ - PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed - ]), - 64 - ); - - $pool->add($name, $consolePool); - $pool->setConsoleDB($name); + $consoleDB = App::getEnv('_APP_CONSOLE_DB', ''); + $consoleDB = explode(',', $consoleDB)[0]; + $consoleDB = explode('=', $consoleDB); + $name = $consoleDB[0]; + $dsn = $consoleDB[1]; + $consoleDBs[$name] = $dsn; /** Parse the project databases */ - global $register; - $dbs = $register->get('dbMap'); - - foreach ($dbs as $name => $dsn) { - $dsn = new DSN($dsn); - $projectPool = new PDOPool( - (new PDOConfig()) - ->withHost($dsn->getHost()) - ->withPort($dsn->getPort()) - ->withDbName($dsn->getDatabase()) - ->withCharset('utf8mb4') - ->withUsername($dsn->getUser()) - ->withPassword($dsn->getPassword()) - ->withOptions([ - PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed - ]), - 64 - ); - - $pool->add($name, $projectPool); + $projectDBs = []; + $projectDB = App::getEnv('_APP_PROJECT_DB', ''); + $projectDB = explode(',', $projectDB); + foreach ($projectDB as $db) { + $db = explode('=', $db); + $name = $db[0]; + $dsn = $db[1]; + $projectDBs[$name] = $dsn; } + $pool = new DatabasePool($consoleDBs, $projectDBs); return $pool; }); @@ -592,30 +548,6 @@ $register->set('geodb', function () { return new Reader(__DIR__ . '/db/DBIP/dbip-country-lite-2022-03.mmdb'); }); -$register->set('consoleDB', function () { - /** This is usually for our workers or CLI commands scope */ - $consoleDb = App::getEnv('_APP_CONSOLE_DB', ''); - $consoleDb = explode(',', $consoleDb)[0]; - $consoleDb = explode('=', $consoleDb); - $dsn = new DSN($consoleDb[1]); - - $dbHost = $dsn->getHost(); - $dbPort = $dsn->getPort(); - $dbUser = $dsn->getUser(); - $dbPass = $dsn->getPassword(); - $dbScheme = $dsn->getDatabase(); - - $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( - PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4', - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, - )); - - return $pdo; -}); - $register->set('cache', function () { // This is usually for our workers or CLI commands scope $redis = new Redis(); diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index 760a4dea7c..b4449c4ebf 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -27,16 +27,16 @@ $cli Console::success('Starting Data Migration to version ' . $version); - $consoleDB = $register->get('consoleDB', true); + $db = $register->get('db', true); $cache = $register->get('cache', true); $cache = new Cache(new RedisCache($cache)); // TODO: Iterate through all project DBs - $projectDB = new Database(new MariaDB($consoleDB), $cache); + $projectDB = new Database(new MariaDB($db), $cache); $projectDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $consoleDB = new Database(new MariaDB($consoleDB), $cache); + $consoleDB = new Database(new MariaDB($db), $cache); $consoleDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); $consoleDB->setNamespace('_project_console'); diff --git a/src/Appwrite/Database/DatabasePool.php b/src/Appwrite/Database/DatabasePool.php index e49226ff77..29c2ad61b6 100644 --- a/src/Appwrite/Database/DatabasePool.php +++ b/src/Appwrite/Database/DatabasePool.php @@ -2,44 +2,195 @@ namespace Appwrite\Database; +use Appwrite\DSN\DSN; use Appwrite\Extend\Exception; +use PDO; +use Swoole\Database\PDOConfig; use Swoole\Database\PDOPool; use Swoole\Database\PDOProxy; +use Utopia\App; class DatabasePool { /** * @var array + * + * Array to store mappings from database names to PDOPool instances. */ protected array $pools = []; + /** + * @var array + * + * Array to store mappings from database names to DSNs + */ + protected array $databases = []; + /** * @var string */ protected string $consoleDB = ''; /** - * Function to get the name of the console database. + * Constructor for Database pools + * + * @param array $consoleDB + * @param array $projectDB + * + */ + public function __construct(array $consoleDB, array $projectDB) + { + if(count($consoleDB) != 1) { + throw new Exception('Console DB should contain only one entry', 500); + } + + if(empty($projectDB)) { + throw new Exception('Project DB is not defined', 500); + } + + $this->consoleDB = array_key_first($consoleDB); + $this->databases = array_merge($consoleDB, $projectDB); + + /** Create PDO pool instances for all the databases */ + foreach ($this->databases as $name => $dsn) { + $dsn = new DSN($dsn); + $pool = new PDOPool( + (new PDOConfig()) + ->withHost($dsn->getHost()) + ->withPort($dsn->getPort()) + ->withDbName($dsn->getDatabase()) + ->withCharset('utf8mb4') + ->withUsername($dsn->getUser()) + ->withPassword($dsn->getPassword()) + ->withOptions([ + PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed + ]), + 64 + ); + + $this->pools[$name] = $pool; + } + } + + /** + * Get a single PDO instance + * + * @param string $name + * + * @return ?PDO + */ + public function getDB(string $name): ?PDO + { + $dsn = $this->dsn[$name] ?? false; + + if ($dsn === false) { + throw new Exception("Database with name : $name not found.", 500); + } + + $dsn = new DSN($dsn); + $dbHost = $dsn->getHost(); + $dbPort = $dsn->getPort(); + $dbUser = $dsn->getUser(); + $dbPass = $dsn->getPassword(); + $dbScheme = $dsn->getDatabase(); + + $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( + PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4', + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + )); + + return $pdo; + } + + /** + * Get a PDO instance from the list of available database pools . To be used in co-routines + * + * @param string $name * * @return ?PDOProxy */ - public function getConsoleDB(): ?PDOProxy + public function getDBFromPool(string $name): ?PDOProxy + { + $pool = $this->pools[$name] ?? throw new Exception("Database pool with name : $name not found. Check the value of _APP_PROJECT_DB in .env", 500); + return $pool->get(); + } + + /** + * Return a PDO instance back to its database pool + * + * @param PDOProxy $db + * @param string $name + * + * @return void + */ + public function put(PDOProxy $db, string $name): void + { + $pool = $this->pools[$name] ?? null; + if ($pool === null) { + throw new Exception("Failed to put PDO into database pool. Database pool with name : $name not found", 500); + } + $pool->put($db); + } + + /** + * Function to get a random PDO instance from the available database pools + * + * @return array [PDO, string] + */ + public function getAnyFromPool(): array + { + $key = array_rand($this->pools); + $pool = $this->getDBFromPool($key); + + return [ + 'name' => $key, + 'db' => $pool + ]; + } + + /** + * Convenience methods for console DB + */ + + /** + * Function to get a single instace of the console DB + * + * @return ?PDO + */ + public function getConsoleDB(): ?PDO + { + if (empty($this->consoleDB)) { + throw new Exception('Console DB is not defined', 500); + }; + + return $this->getDB($this->consoleDB); + } + + /** + * Function to get an instance of the console DB from the database pool + * + * @return ?PDOProxy + */ + public function getConsoleDBFromPool(): ?PDOProxy { if (empty($this->consoleDB)) { throw new Exception("Console DB not set", 500); } - return $this->get($this->consoleDB); + return $this->getDBFromPool($this->consoleDB); } /** - * Return a PDO instance back to the console database pool + * Return the console DB back to the console database pool * * @param PDOProxy $db * * @return void */ - public function putConsoleDb(PDOProxy $db): void + public function putConsoleDB(PDOProxy $db): void { $this->put($db, $this->consoleDB); } @@ -59,70 +210,4 @@ class DatabasePool { $this->consoleDB = $consoleDB; } - - /** - * Add a new PDOPool into the list of available pools - * - * @param string $name - * @param PDOPool $dbPool - * - * @return void - */ - public function add(string $name, PDOPool $dbPool): void - { - $this->pools[$name] = $dbPool; - } - - /** - * Get a PDO instance from the list of available database pools - * - * @param string $name - * - * @return ?PDOProxy - */ - public function get(string $name): ?PDOProxy - { - $pool = $this->pools[$name] ?? null; - if ($pool === null) { - throw new Exception("Database pool with name : $name not found. Check the value of _APP_PROJECT_DB in .env", 500); - } - return $pool->get(); - } - - /** - * Function to get a random PDO instance from the available database pools database - * - * @return array [PDO, string] - */ - public function getAny(): ?array - { - if (count($this->pools) === 0) { - throw new Exception("No database pools found. Add pools using DatabasePool::add() method", 500); - } - - $key = array_rand($this->pools); - $pool = $this->pools[$key] ?? null; - - return [ - 'name' => $key, - 'db' => $pool ? $pool->get() : null - ]; - } - - /** - * Return a PDO instance back to its database pool - * - * @param PDOProxy $db - * @param string $name - * - * @return void - */ - public function put(PDOProxy $db, string $name): void - { - $pool = $this->pools[$name] ?? null; - if ($pool === null) { - throw new Exception("Failed to put PDO into database pool. Database pool with name : $name not found", 500); - } - $pool->put($db); - } } \ No newline at end of file