diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index dcb83a1665..c432115e86 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -11,8 +11,6 @@ use Appwrite\Network\Validator\Domain as DomainValidator; use Appwrite\Network\Validator\Origin; use Appwrite\Network\Validator\URL; use Appwrite\Utopia\Database\Validator\CustomId; -use Utopia\Cache\Cache; -use Utopia\Cache\Adapter\Redis as RedisCache; use Appwrite\Utopia\Response; use Utopia\Abuse\Adapters\TimeLimit; use Utopia\App; @@ -26,7 +24,6 @@ use Utopia\Database\Validator\UID; use Utopia\Domains\Domain; use Utopia\Registry\Registry; use Appwrite\Extend\Exception; -use Utopia\Database\Adapter\MariaDB; use Utopia\Validator\ArrayList; use Utopia\Validator\Boolean; use Utopia\Validator\Hostname; @@ -67,7 +64,7 @@ App::post('/v1/projects') ->inject('dbForConsole') ->inject('cache') ->inject('dbPool') - ->action(function (string $projectId, string $name, string $teamId, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Redis $cache, DatabasePool $dbPool) { + ->action(function (string $projectId, string $name, string $teamId, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, \Redis $cache, DatabasePool $dbPool) { $team = $dbForConsole->getDocument('teams', $teamId); @@ -86,7 +83,7 @@ App::post('/v1/projects') throw new Exception("'console' is a reserved project.", 400, Exception::PROJECT_RESERVED_PROJECT); } - ['name' => $dbName, 'db' => $projectDB] = $dbPool->getAnyFromPool(); + [$dbForProject, $returnDatabase, $dbName] = $dbPool->getAnyFromPool($cache); $project = $dbForConsole->createDocument('projects', new Document([ '$id' => $projectId, @@ -115,10 +112,7 @@ App::post('/v1/projects') 'database' => $dbName ])); - $cache = new Cache(new RedisCache($cache)); - $dbForProject = new Database(new MariaDB($projectDB), $cache); - $dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $dbForProject->setNamespace("_{$projectId}"); + $dbForProject->setNamespace("_$projectId"); $dbForProject->create('appwrite'); $audit = new Audit($dbForProject); @@ -164,7 +158,7 @@ App::post('/v1/projects') $dbForProject->createCollection($key, $attributes, $indexes); } - $dbPool->put($projectDB, $dbName); + call_user_func($returnDatabase); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($project, Response::MODEL_PROJECT); diff --git a/app/http.php b/app/http.php index 61dc90490d..477e8ba1dc 100644 --- a/app/http.php +++ b/app/http.php @@ -58,30 +58,13 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $app = new App('UTC'); go(function () use ($register, $app) { - // wait for database to be ready - $attempts = 0; - $max = 10; - $sleep = 1; - do { - try { - $attempts++; - $consoleDB = $register->get('dbPool')->getConsoleDBFromPool(); - $redis = $register->get('redisPool')->get(); - break; // leave the do-while if successful - } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep($sleep); - } - } while ($attempts < $max); - - App::setResource('consoleDB', fn() => $consoleDB); + $redis = $register->get('redisPool')->get(); App::setResource('cache', fn() => $redis); - - $dbForConsole = $app->getResource('dbForConsole'); /** @var Utopia\Database\Database $dbForConsole */ + + $dbPool = $register->get('dbPool'); + [$dbForConsole, $returnDatabase] = $dbPool->getDBFromPool('console', $redis); + App::setResource('dbForConsole', fn() => $dbForConsole); Console::success('[Setup] - Server database init started...'); $collections = Config::getParam('collections', []); /** @var array $collections */ @@ -207,6 +190,8 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $dbForConsole->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes); } + call_user_func($returnDatabase); + Console::success('[Setup] - Server database init completed...'); }); @@ -239,26 +224,18 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $app = new App('UTC'); - $dbPool = $register->get('dbPool'); - $consoleDB = $dbPool->getConsoleDBFromPool(); $redis = $register->get('redisPool')->get(); - - App::setResource('dbPool', fn() => $dbPool); - App::setResource('consoleDB', fn() => $consoleDB); App::setResource('cache', fn() => $redis); - $projectId = $request->getParam('project', $request->getHeader('x-appwrite-project', 'console')); - $projectDB = $consoleDB; - if ($projectId !== 'console') { - $dbForConsole = $app->getResource('dbForConsole'); /** @var Utopia\Database\Database $dbForConsole */ - $project = Authorization::skip(fn() => $dbForConsole->getDocument('projects', $projectId)); - $dbName = $project->getAttribute('database', ''); - if (!empty($dbName)) { - $projectDB = $dbPool->getDBFromPool($dbName); - } - } + $dbPool = $register->get('dbPool'); + App::setResource('dbPool', fn() => $dbPool); - App::setResource('projectDB', fn() => $projectDB); + [$dbForConsole, $returnConsoleDB] = $dbPool->getDBFromPool('console', $redis); + App::setResource('dbForConsole', fn() => $dbForConsole); + + $projectId = $request->getParam('project', $request->getHeader('x-appwrite-project', 'console')); + [$dbForProject, $returnProjectDB] = $dbPool->getDBFromPool($projectId, $redis); + App::setResource('dbForProject', fn() => $dbForProject); try { Authorization::cleanRoles(); @@ -349,12 +326,8 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $swooleResponse->end(\json_encode($output)); } finally { - /** @var PDOPool $consolePool */ - $dbPool->putConsoleDb($consoleDB); - - if (!empty($dbName) && !empty($projectDB)) { - $dbPool->put($projectDB, $dbName); - } + call_user_func($returnConsoleDB); + call_user_func($returnProjectDB); /** @var RedisPool $redisPool */ $redisPool = $register->get('redisPool'); diff --git a/app/init.php b/app/init.php index 2b518c7978..ffb9bcb0d1 100644 --- a/app/init.php +++ b/app/init.php @@ -847,26 +847,6 @@ App::setResource('console', function () { ]); }, []); -App::setResource('dbForConsole', function ($consoleDB, $cache) { - $cache = new Cache(new RedisCache($cache)); - - $database = new Database(new MariaDB($consoleDB), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace('_console'); - - return $database; -}, ['consoleDB', 'cache']); - -App::setResource('dbForProject', function ($projectDB, $cache, $project) { - $cache = new Cache(new RedisCache($cache)); - - $database = new Database(new MariaDB($projectDB), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace("_{$project->getId()}"); - - return $database; -}, ['projectDB', 'cache', 'project']); - App::setResource('deviceLocal', function () { return new Local(); }); diff --git a/phpunit.xml b/phpunit.xml index e3d583022f..87bf3df9cc 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -6,7 +6,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false" + stopOnFailure="true" > diff --git a/src/Appwrite/Database/DatabasePool.php b/src/Appwrite/Database/DatabasePool.php index de7135f76d..70c696c821 100644 --- a/src/Appwrite/Database/DatabasePool.php +++ b/src/Appwrite/Database/DatabasePool.php @@ -9,6 +9,12 @@ use Swoole\Database\PDOConfig; use Swoole\Database\PDOPool; use Swoole\Database\PDOProxy; use Utopia\App; +use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\Cache\Cache; +use Utopia\CLI\Console; +use Utopia\Database\Adapter\MariaDB; +use Utopia\Database\Database; +use Utopia\Database\Validator\Authorization; class DatabasePool { @@ -24,10 +30,12 @@ class DatabasePool { * * Array to store mappings from database names to DSNs */ - protected array $databases = []; + protected array $dsns = []; /** * @var string + * + * The name of the console Database */ protected string $consoleDB = ''; @@ -49,10 +57,10 @@ class DatabasePool { } $this->consoleDB = array_key_first($consoleDB); - $this->databases = array_merge($consoleDB, $projectDB); + $this->dsns = array_merge($consoleDB, $projectDB); - /** Create PDO pool instances for all the databases */ - foreach ($this->databases as $name => $dsn) { + /** Create PDO pool instances for all the dsns */ + foreach ($this->dsns as $name => $dsn) { $dsn = new DSN($dsn); $pool = new PDOPool( (new PDOConfig()) @@ -67,21 +75,20 @@ class DatabasePool { ]), 64 ); - + $this->pools[$name] = $pool; } } /** - * Get a single PDO instance + * Get a PDO instance by database name * * @param string $name - * * @return ?PDO */ - public function getDB(string $name): ?PDO + private function getPDO(string $name): ?PDO { - $dsn = $this->databases[$name] ?? throw new Exception("Database with name : $name not found.", 500); + $dsn = $this->dsns[$name] ?? throw new Exception("Database with name : $name not found.", 500); $dsn = new DSN($dsn); $dbHost = $dsn->getHost(); @@ -102,16 +109,147 @@ class DatabasePool { } /** - * Get a PDO instance from the list of available database pools . To be used in co-routines + * @param string $projectID * - * @param string $name + * @return string * - * @return ?PDOProxy + * Function to return the name of the database from the project ID */ - public function getDBFromPool(string $name): ?PDOProxy + private function getName(string $projectID, \Redis $redis): string { + if ($projectID === 'console') { + return $this->consoleDB; + } + + $pdo = $this->getPDO($this->consoleDB); + $cache = new Cache(new RedisCache($redis)); + + $database = new Database(new MariaDB($pdo), $cache); + $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $database->setNamespace("_console"); + + $project = Authorization::skip(fn() => $database->getDocument('projects', $projectID)); + $database = $project->getAttribute('database', ''); + + return $database; + } + + /** + * Get a single PDO instance for a project + * + * @param string $projectId + * + * @return ?Database + */ + public function getDB(string $projectID, \Redis $cache): ?Database + { + /** Get DB name from the console database */ + $name = $this->getName($projectID, $cache); + + if (empty($name)) { + throw new Exception("Database with name : $name not found.", 500); + } + + /** Get a PDO instance using the databse name */ + $pdo = $this->getPDO($name); + $cache = new Cache(new RedisCache($cache)); + $database = new Database(new MariaDB($pdo), $cache); + $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $database->setNamespace("_{$projectID}"); + + return $database; + } + + + // private function attemptConnection(PDO|PDOProxy $pdo, ?string $namespace, \Redis $cache): Database + // { + + // } + + /** + * Get a PDO instance from the list of available database pools . Meant to be used in co-routines + * + * @param string $projectId + * + * @return array + */ + public function getDBFromPool(string $projectID, \Redis $redis): array + { + /** Get DB name from the console database */ + $name = $this->getName($projectID, $redis); $pool = $this->pools[$name] ?? throw new Exception("Database pool with name : $name not found. Check the value of _APP_PROJECT_DB in .env", 500); - return $pool->get(); + + $namespace = "_$projectID"; + $attempts = 0; + do { + try { + $attempts++; + $pdo = $pool->get(); + $cache = new Cache(new RedisCache($redis)); + $database = new Database(new MariaDB($pdo), $cache); + $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + $database->setNamespace($namespace); + + // if (!$database->exists($database->getDefaultDatabase(), 'metadata')) { + // throw new Exception('Collection not ready'); + // } + break; // leave loop if successful + } catch (\Exception $e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { + throw new \Exception('Failed to connect to database: ' . $e->getMessage()); + } + sleep(DATABASE_RECONNECT_SLEEP); + } + } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); + + return [ + $database, + function () use ($pdo, $name) { + $this->put($pdo, $name); + } + ]; + } + + /** + * Function to get a random PDO instance from the available database pools + * + * @return array [PDO, string] + */ + public function getAnyFromPool(\Redis $redis): array + { + $name = array_rand($this->pools); + $pool = $this->pools[$name] ?? throw new Exception("Database pool with name : $name not found. Check the value of _APP_PROJECT_DB in .env", 500); + + $attempts = 0; + do { + try { + $attempts++; + $pdo = $pool->get(); + $cache = new Cache(new RedisCache($redis)); + $database = new Database(new MariaDB($pdo), $cache); + $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); + + // if (!$database->exists($database->getDefaultDatabase(), 'metadata')) { + // throw new Exception('Collection not ready'); + // } + break; // leave loop if successful + } catch (\Exception $e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { + throw new \Exception('Failed to connect to database: ' . $e->getMessage()); + } + sleep(DATABASE_RECONNECT_SLEEP); + } + } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); + + return [ + $database, + function () use ($pdo, $name) { + $this->put($pdo, $name); + }, + $name + ]; } /** @@ -131,79 +269,63 @@ class DatabasePool { $pool->put($db); } - /** - * Function to get a random PDO instance from the available database pools - * - * @return array [PDO, string] - */ - public function getAnyFromPool(): array - { - $key = array_rand($this->pools); - $pool = $this->getDBFromPool($key); + // /** + // * Convenience methods for console DB + // */ - return [ - 'name' => $key, - 'db' => $pool - ]; - } + // /** + // * Function to get a single instace of the console DB + // * + // * @return ?PDO + // */ + // public function getConsoleDB(): ?PDO + // { + // if (empty($this->consoleDB)) { + // throw new Exception('Console DB is not defined', 500); + // }; - /** - * Convenience methods for console DB - */ + // return $this->getDB($this->consoleDB); + // } - /** - * Function to get a single instace of the console DB - * - * @return ?PDO - */ - public function getConsoleDB(): ?PDO - { - if (empty($this->consoleDB)) { - throw new Exception('Console DB is not defined', 500); - }; + // /** + // * Function to get an instance of the console DB from the database pool + // * + // * @return ?PDOProxy + // */ + // public function getConsoleDBFromPool(): ?PDOProxy + // { + // if (empty($this->consoleDB)) { + // throw new Exception("Console DB not set", 500); + // } - return $this->getDB($this->consoleDB); - } + // return $this->getDBFromPool($this->consoleDB); + // } - /** - * Function to get an instance of the console DB from the database pool - * - * @return ?PDOProxy - */ - public function getConsoleDBFromPool(): ?PDOProxy - { - if (empty($this->consoleDB)) { - throw new Exception("Console DB not set", 500); - } + // /** + // * Return the console DB back to the console database pool + // * + // * @param PDOProxy $db + // * + // * @return void + // */ + // public function putConsoleDB(PDOProxy $db): void + // { + // $this->put($db, $this->consoleDB); + // } - return $this->getDBFromPool($this->consoleDB); - } - - /** - * Return the console DB back to the console database pool - * - * @param PDOProxy $db - * - * @return void - */ - public function putConsoleDB(PDOProxy $db): void - { - $this->put($db, $this->consoleDB); - } - - /** - * Function to set the name of the console database - * - * @param string $consoleDB - * - * @return void - */ - public function setConsoleDB(string $consoleDB): void - { - if(!isset($this->pools[$consoleDB])) { - throw new Exception("Console DB with name : $consoleDB not found. Add it using ", 500); - } + // /** + // * Function to set the name of the console database + // * + // * @param string $consoleDB + // * + // * @return void + // */ + // public function setConsoleDB(string $consoleDB): void + // { + // if(!isset($this->pools[$consoleDB])) { + // throw new Exception("Console DB with name : $consoleDB not found. Add it using ", 500); + // } - $this->consoleDB = $consoleDB; - } + // $this->consoleDB = $consoleDB; + // } } \ No newline at end of file diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 47060d9d8f..e77578f56c 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -161,20 +161,13 @@ abstract class Worker protected function getProjectDB(string $projectId): Database { global $register; - if (!$projectId) { throw new \Exception('ProjectID not provided - cannot get database'); } - - $namespace = "_{$projectId}"; - - $dbForConsole = $this->getConsoleDB(); - $project = $dbForConsole->getDocument('projects', $projectId); - $dbName = $project->getAttribute('database', ''); - - $projectDB = $register->get('dbPool')->getDB($dbName); - - return $this->getDB(self::DATABASE_PROJECT, $projectDB, $namespace); + $cache = $register->get('cache'); + $dbPool = $register->get('dbPool'); + $dbForProject = $dbPool->getDB($projectId, $cache); + return $dbForProject; } /** @@ -182,53 +175,13 @@ abstract class Worker * @return Database */ protected function getConsoleDB(): Database - { - global $register; - $consoleDB = $register->get('dbPool')->getConsoleDB(); - $namespace = "_console"; - $sleep = 5; // ConsoleDB needs extra sleep time to ensure tables are created - - return $this->getDB(self::DATABASE_CONSOLE, $consoleDB, $namespace, $sleep); - } - - /** - * Get console database - * @param string $type One of (internal, external, console) - * @param string $projectId of internal or external DB - * @return Database - */ - private function getDB(string $type, PDO $pdo, string $namespace, int $sleep = DATABASE_RECONNECT_SLEEP): Database { global $register; $cache = $register->get('cache'); - $attempts = 0; - do { - try { - $attempts++; - $cache = new Cache(new RedisCache($cache)); - $database = new Database(new MariaDB($pdo), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace($namespace); // Main DB + $dbPool = $register->get('dbPool'); - if (!empty($projectId) && !$database->getDocument('projects', $projectId)->isEmpty()) { - throw new \Exception("Project does not exist: {$projectId}"); - } - - if ($type === self::DATABASE_CONSOLE && !$database->exists($database->getDefaultDatabase(), '_metadata')) { - throw new \Exception('Console project not ready'); - } - - break; // leave loop if successful - } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep($sleep); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return $database; + $dbForConsole = $dbPool->getDB('console', $cache); + return $dbForConsole; } /**