diff --git a/.env b/.env index 8dcb1c6c17..772302b594 100644 --- a/.env +++ b/.env @@ -41,8 +41,9 @@ _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword -_APP_DB_ADAPTER_DOCUMENTS=mongodb -_APP_DOCUMENTS_DB_HOST=mongo +_APP_DB_ADAPTER_DOCUMENTSDB=mongodb +_APP_DB_HOST_DOCUMENTSDB=mongodb +_APP_DB_HOST_DOCUMENTSDB_PORT=27017 _APP_STORAGE_DEVICE=Local _APP_STORAGE_S3_ACCESS_KEY= _APP_STORAGE_S3_SECRET= diff --git a/app/init/registers.php b/app/init/registers.php index a6b259a5fe..9581ef98f6 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -112,8 +112,8 @@ $register->set('pools', function () { ]); $fallbackForDocumentsDB = 'db_main=' . AppwriteURL::unparse([ 'scheme' => 'mongodb', - 'host' => System::getEnv('_APP_DOCUMENTS_DB_HOST', 'mongodb'), - 'port' => System::getEnv('_APP_DOCUMENTS_DB_PORT', '27017'), + 'host' => System::getEnv('_APP_DB_HOST_DOCUMENTSDB', 'mongodb'), + 'port' => System::getEnv('_APP_DB_HOST_DOCUMENTSDB_PORT', '27017'), 'user' => System::getEnv('_APP_DB_USER', ''), 'pass' => System::getEnv('_APP_DB_PASS', ''), 'path' => System::getEnv('_APP_DB_SCHEMA', ''), @@ -276,9 +276,7 @@ $register->set('pools', function () { default => null }; - if ($dsn->getScheme() !== 'mongodb') { - $adapter->setDatabase($dsn->getPath()); - } + $adapter->setDatabase($dsn->getPath()); return $adapter; case 'pubsub': return match ($dsn->getScheme()) { diff --git a/app/init/resources.php b/app/init/resources.php index 825c436e8d..4b403672d4 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -349,7 +349,7 @@ App::setResource('console', function () { return new Document(Config::getParam('console')); }, []); -App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Database $dbForDocuments, Cache $cache, Document $project, Request $request) { +App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -385,7 +385,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform } return $database; -}, ['pools', 'dbForPlatform', 'dbForDocuments', 'cache', 'project', 'request']); +}, ['pools', 'dbForPlatform', 'cache', 'project']); App::setResource('dbForPlatform', function (Group $pools, Cache $cache) { $adapter = new DatabasePool($pools->get('console')); @@ -401,51 +401,59 @@ App::setResource('dbForPlatform', function (Group $pools, Cache $cache) { return $database; }, ['pools', 'cache']); -App::setResource('dbForDocuments', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project) { - if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForPlatform; - } +App::setResource('getDatabaseDB', function (Group $pools, Cache $cache, Document $project, Request $request) { - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + return function (Document $database) use ($pools, $cache, $project, $request): Database { + $databaseType = $database->getAttribute('database', ''); + $databaseDSN = new DSN($databaseType); + $datatypeType = $databaseDSN->getScheme(); - $adapter = new DatabasePool($pools->get('documentsDb')); - $database = new Database($adapter, $cache); - $database - ->setMetadata('host', \gethostname()) - ->setMetadata('project', $project->getId()) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + $pool = null; + + switch ($datatypeType) { + case System::getEnv('_APP_DB_HOST_DOCUMENTSDB', 'mongodb'): + $pool = $pools->get('documentsDb'); + break; + default: + $pool = $pools->get($dsn->getHost()); + } + + $adapter = new DatabasePool($pool); + $database = new Database($adapter, $cache); + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - if (\in_array($dsn->getHost(), $sharedTables)) { $database - ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } + ->setMetadata('host', \gethostname()) + ->setMetadata('project', $project->getId()) + ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) + ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); - return $database; -}, ['pools', 'dbForPlatform', 'cache', 'project']); + if (\in_array($dsn->getHost(), $sharedTables)) { + $database + ->setSharedTables(true) + ->setTenant((int)$project->getSequence()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getSequence()); + } + $timeout = \intval($request->getHeader('x-appwrite-timeout')); + if (!empty($timeout) && App::isDevelopment()) { + $database->setTimeout($timeout); + } + return $database; + }; -App::setResource('dbForDatabaseRecords', function (Database $dbForProject, Database $dbForDocuments, Request $request) { - - $uri = $request->getURI(); - if (str_starts_with($uri, '/v1/documentsdb')) { - return $dbForDocuments; - } - return $dbForProject; - -}, ['dbForProject','dbForDocuments','request']); +}, ['pools','cache','project','request']); App::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { diff --git a/app/worker.php b/app/worker.php index 5cb156f206..21a8bf3633 100644 --- a/app/worker.php +++ b/app/worker.php @@ -106,37 +106,6 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, return $database; }, ['cache', 'register', 'message', 'project', 'dbForPlatform']); -Server::setResource('dbForDocuments', function (Cache $cache, Registry $register, Document $project, Database $dbForPlatform) { - if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForPlatform; - } - $pools = $register->get('pools'); - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - $adapter = new DatabasePool($pools->get('documentsDb')); - $database = new Database($adapter, $cache); - - if (\in_array($dsn->getHost(), $sharedTables)) { - $database - ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } - $database->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER); - return $database; -}, ['cache', 'register', 'project', 'dbForPlatform']); - Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools @@ -223,17 +192,50 @@ Server::setResource('getLogsDB', function (Group $pools, Cache $cache) { }; }, ['pools', 'cache']); -Server::setResource('getDatabaseRecordsDB', function (Database $dbForProject, Database $dbForDocuments) { - return function (Document $database) use ($dbForProject, $dbForDocuments): Database { +Server::setResource('getDatabaseDB', function (Cache $cache, Registry $register, Document $project) { + return function (Document $database) use ($cache, $register, $project): Database { $databaseType = $database->getAttribute('database', ''); - $dsn = new DSN($databaseType); - $datatypeType = $dsn->getScheme(); - return match ($datatypeType) { - 'mongodb' => $dbForDocuments, - default => $dbForProject, - }; + $databaseDSN = new DSN($databaseType); + $datatypeType = $databaseDSN->getScheme(); + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + $pools = $register->get('pools'); + $pool = null; + + switch ($datatypeType) { + case System::getEnv('_APP_DB_HOST_DOCUMENTSDB', 'mongodb'): + $pool = $pools->get('documentsDb'); + break; + default: + $pool = $pools->get($dsn->getHost()); + } + + $adapter = new DatabasePool($pool); + $database = new Database($adapter, $cache); + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + + if (\in_array($dsn->getHost(), $sharedTables)) { + $database + ->setSharedTables(true) + ->setTenant((int)$project->getSequence()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getSequence()); + } + + $database->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER); + return $database; }; -}, ['dbForProject', 'dbForDocuments']); +}, ['cache', 'register', 'project']); Server::setResource('abuseRetention', function () { return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); // 1 day diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Init/Timeout.php b/src/Appwrite/Platform/Modules/Databases/Http/Init/Timeout.php index 31efc253bd..19e202981b 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Init/Timeout.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Init/Timeout.php @@ -24,13 +24,11 @@ class Timeout extends Action ->groups(['api', 'database']) ->inject('request') ->inject('dbForProject') - ->inject('dbForDocuments') - ->callback(function (Request $request, Database $dbForProject, Database $dbForDocuments) { + ->callback(function (Request $request, Database $dbForProject) { $timeout = \intval($request->getHeader('x-appwrite-timeout')); if (!empty($timeout) && App::isDevelopment()) { $dbForProject->setTimeout($timeout); - $dbForDocuments->setTimeout($timeout); } }); } diff --git a/src/Appwrite/Platform/Modules/Databases/Workers/Databases.php b/src/Appwrite/Platform/Modules/Databases/Workers/Databases.php index 0b2c419174..89cecd3960 100644 --- a/src/Appwrite/Platform/Modules/Databases/Workers/Databases.php +++ b/src/Appwrite/Platform/Modules/Databases/Workers/Databases.php @@ -36,7 +36,7 @@ class Databases extends Action ->inject('project') ->inject('dbForPlatform') ->inject('dbForProject') - ->inject('getDatabaseRecordsDB') + ->inject('getDatabaseDB') ->inject('queueForRealtime') ->inject('log') ->callback($this->action(...)); @@ -52,7 +52,7 @@ class Databases extends Action * @return void * @throws \Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, callable $getDatabaseRecordsDB, Realtime $queueForRealtime, Log $log): void + public function action(Message $message, Document $project, Database $dbForPlatform, Database $dbForProject, callable $getDatabaseDB, Realtime $queueForRealtime, Log $log): void { $payload = $message->getPayload() ?? []; @@ -65,9 +65,9 @@ class Databases extends Action $collection = new Document($payload['table'] ?? $payload['collection'] ?? []); $database = new Document($payload['database'] ?? []); /** - * @var Database $dbForDatabaseRecords + * @var Database $dbForDatabase */ - $dbForDatabaseRecords = call_user_func($getDatabaseRecordsDB, $database); + $dbForDatabase = call_user_func($getDatabaseDB, $database); $log->addTag('projectId', $project->getId()); $log->addTag('type', $type); @@ -79,12 +79,12 @@ class Databases extends Action $log->addTag('databaseId', $database->getId()); match (\strval($type)) { - DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $dbForProject, $dbForDatabaseRecords), - DATABASE_TYPE_DELETE_COLLECTION => $this->deleteCollection($database, $collection, $dbForProject, $dbForDatabaseRecords), + DATABASE_TYPE_DELETE_DATABASE => $this->deleteDatabase($database, $dbForProject, $dbForDatabase), + DATABASE_TYPE_DELETE_COLLECTION => $this->deleteCollection($database, $collection, $dbForProject, $dbForDatabase), DATABASE_TYPE_CREATE_ATTRIBUTE => $this->createAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $queueForRealtime), - DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $dbForDatabaseRecords, $queueForRealtime), - DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $dbForDatabaseRecords, $queueForRealtime), - DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $dbForDatabaseRecords, $queueForRealtime), + DATABASE_TYPE_DELETE_ATTRIBUTE => $this->deleteAttribute($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $dbForDatabase, $queueForRealtime), + DATABASE_TYPE_CREATE_INDEX => $this->createIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $dbForDatabase, $queueForRealtime), + DATABASE_TYPE_DELETE_INDEX => $this->deleteIndex($database, $collection, $document, $project, $dbForPlatform, $dbForProject, $dbForDatabase, $queueForRealtime), default => throw new Exception('No database operation for type: ' . \strval($type)), }; @@ -237,7 +237,7 @@ class Databases extends Action * @param Document $project * @param Database $dbForPlatform * @param Database $dbForProject - * @param Database $dbForDatabaseRecords + * @param Database $dbForDatabase * @param Realtime $queueForRealtime * @return void * @throws Authorization @@ -245,7 +245,7 @@ class Databases extends Action * @throws \Exception * @throws \Throwable **/ - private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForDatabaseRecords, Database $dbForProject, Realtime $queueForRealtime): void + private function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project, Database $dbForPlatform, Database $dbForDatabase, Database $dbForProject, Realtime $queueForRealtime): void { if ($collection->isEmpty()) { throw new Exception('Missing collection/table'); @@ -374,7 +374,7 @@ class Databases extends Action } if ($exists) { // Delete the duplicate if created, else update in db - $this->deleteIndex($database, $collection, $index, $project, $dbForPlatform, $dbForProject, $dbForDatabaseRecords, $queueForRealtime); + $this->deleteIndex($database, $collection, $index, $project, $dbForPlatform, $dbForProject, $dbForDatabase, $queueForRealtime); } else { $dbForProject->updateDocument('indexes', $index->getId(), $index); } @@ -399,7 +399,7 @@ class Databases extends Action * @param Document $project * @param Database $dbForPlatform * @param Database $dbForProject - * @param Database $dbForDatabaseRecords + * @param Database $dbForDatabase * @param Realtime $queueForRealtime * @return void * @throws Authorization @@ -408,7 +408,7 @@ class Databases extends Action * @throws DatabaseException * @throws \Throwable */ - private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject, Database $dbForDatabaseRecords, Realtime $queueForRealtime): void + private function createIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject, Database $dbForDatabase, Realtime $queueForRealtime): void { if ($collection->isEmpty()) { throw new Exception('Missing collection/table'); @@ -428,7 +428,7 @@ class Databases extends Action $project = $dbForPlatform->getDocument('projects', $projectId); try { - if (!$dbForDatabaseRecords->createIndex('database_' . $database->getSequence() . '_collection_' . $collection->getSequence(), $key, $type, $attributes, $lengths, $orders)) { + if (!$dbForDatabase->createIndex('database_' . $database->getSequence() . '_collection_' . $collection->getSequence(), $key, $type, $attributes, $lengths, $orders)) { throw new DatabaseException('Failed to create Index'); } $dbForProject->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'available')); @@ -458,7 +458,7 @@ class Databases extends Action * @param Document $project * @param Database $dbForPlatform * @param Database $dbForProject - * @param Database $dbForDatabaseRecords + * @param Database $dbForDatabase * @param Realtime $queueForRealtime * @return void * @throws Authorization @@ -467,7 +467,7 @@ class Databases extends Action * @throws DatabaseException * @throws \Throwable */ - private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject, Database $dbForDatabaseRecords, Realtime $queueForRealtime): void + private function deleteIndex(Document $database, Document $collection, Document $index, Document $project, Database $dbForPlatform, Database $dbForProject, Database $dbForDatabase, Realtime $queueForRealtime): void { if ($collection->isEmpty()) { throw new Exception('Missing collection/table'); @@ -483,7 +483,7 @@ class Databases extends Action $project = $dbForPlatform->getDocument('projects', $projectId); try { - if ($status !== 'failed' && !$dbForDatabaseRecords->deleteIndex('database_' . $database->getSequence() . '_collection_' . $collection->getSequence(), $key)) { + if ($status !== 'failed' && !$dbForDatabase->deleteIndex('database_' . $database->getSequence() . '_collection_' . $collection->getSequence(), $key)) { throw new DatabaseException('Failed to delete index'); } $dbForProject->deleteDocument('indexes', $index->getId()); @@ -512,14 +512,14 @@ class Databases extends Action /** * @param Document $database * @param Database $dbForProject - * @param Database $dbForDatabaseRecords + * @param Database $dbForDatabase * @return void * @throws Exception */ - protected function deleteDatabase(Document $database, Database $dbForProject, Database $dbForDatabaseRecords): void + protected function deleteDatabase(Document $database, Database $dbForProject, Database $dbForDatabase): void { - $this->deleteByGroup('database_' . $database->getSequence(), [], $dbForProject, function ($collection) use ($database, $dbForProject, $dbForDatabaseRecords) { - $this->deleteCollection($database, $collection, $dbForProject, $dbForDatabaseRecords); + $this->deleteByGroup('database_' . $database->getSequence(), [], $dbForProject, function ($collection) use ($database, $dbForProject, $dbForDatabase) { + $this->deleteCollection($database, $collection, $dbForProject, $dbForDatabase); }); $dbForProject->deleteCollection('database_' . $database->getSequence()); @@ -529,7 +529,7 @@ class Databases extends Action * @param Document $database * @param Document $collection * @param Database $dbForProject - * @param Database $dbForDatabaseRecords + * @param Database $dbForDatabase * @return void * @throws Authorization * @throws Conflict @@ -538,7 +538,7 @@ class Databases extends Action * @throws Structure * @throws Exception */ - protected function deleteCollection(Document $database, Document $collection, Database $dbForProject, Database $dbForDatabaseRecords): void + protected function deleteCollection(Document $database, Document $collection, Database $dbForProject, Database $dbForDatabase): void { if ($collection->isEmpty()) { throw new Exception('Missing collection/table'); @@ -548,7 +548,7 @@ class Databases extends Action $collectionInternalId = $collection->getSequence(); $databaseInternalId = $database->getSequence(); - $dbForDatabaseRecords->deleteCollection('database_' . $databaseInternalId . '_collection_' . $collection->getSequence()); + $dbForDatabase->deleteCollection('database_' . $databaseInternalId . '_collection_' . $collection->getSequence()); /** * Related collections relating to current collection