diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa6dbe2bc3..682c18f85f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -451,6 +451,10 @@ jobs: _APP_BROWSER_HOST: http://invalid-browser/v1 _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'database_db_main' || '' }} + _APP_DATABASE_DOCUMENTSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'documentsdb_db_main' || '' }} + _APP_DATABASE_DOCUMENTSDB_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'documentsdb_db_main' || '' }} + _APP_DATABASE_VECTORSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'vectorsdb_db_main' || '' }} + _APP_DATABASE_VECTORSDB_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'vectorsdb_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -525,6 +529,10 @@ jobs: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'database_db_main' || '' }} + _APP_DATABASE_DOCUMENTSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'documentsdb_db_main' || '' }} + _APP_DATABASE_DOCUMENTSDB_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'documentsdb_db_main' || '' }} + _APP_DATABASE_VECTORSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'vectorsdb_db_main' || '' }} + _APP_DATABASE_VECTORSDB_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'vectorsdb_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -582,6 +590,10 @@ jobs: env: _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'database_db_main' || '' }} + _APP_DATABASE_DOCUMENTSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'documentsdb_db_main' || '' }} + _APP_DATABASE_DOCUMENTSDB_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'documentsdb_db_main' || '' }} + _APP_DATABASE_VECTORSDB_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'vectorsdb_db_main' || '' }} + _APP_DATABASE_VECTORSDB_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'vectorsdb_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable diff --git a/app/init/resources.php b/app/init/resources.php index 3481e73e0b..9f5b02fb91 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -888,8 +888,9 @@ Http::setResource('dbForPlatform', function (Group $pools, Cache $cache, Authori }, ['pools', 'cache', 'authorization']); Http::setResource('getDatabasesDB', function (Group $pools, Cache $cache, Document $project, Request $request, UsageContext $usage, Authorization $authorization) { + $initializedPools = []; - return function (Document $database) use ($pools, $cache, $project, $request, $usage, $authorization): Database { + return function (Document $database) use ($pools, $cache, $project, $request, $usage, $authorization, &$initializedPools): Database { $databaseDSN = $database->getAttribute('database', $project->getAttribute('database', '')); $databaseType = $database->getAttribute('type', ''); @@ -907,11 +908,12 @@ Http::setResource('getDatabasesDB', function (Group $pools, Cache $cache, Docume $dsn = new DSN('mysql://' . $project->getAttribute('database')); } - $pool = $pools->get($databaseDSN->getHost()); + $databaseHost = $databaseDSN->getHost(); + $pool = $pools->get($databaseHost); $adapter = new DatabasePool($pool); $database = new Database($adapter, $cache); - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + $sharedTables = \array_filter(\explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''))); $database ->setDatabase(APP_DATABASE) @@ -922,10 +924,30 @@ Http::setResource('getDatabasesDB', function (Group $pools, Cache $cache, Docume ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); // inside pools authorization needs to be set first $database->getAdapter()->setSupportForAttributes($databaseType !== DOCUMENTSDB); - if (\in_array($dsn->getHost(), $sharedTables)) { + + // When the database uses a separate pool (e.g. vectorsdb on PostgreSQL), + // always use dedicated mode with namespace isolation. Shared tables mode + // can't be used across different engines (e.g. MongoDB UUID tenants are + // incompatible with PostgreSQL's integer _tenant column). + if ($databaseHost !== $dsn->getHost()) { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getSequence()); + + $poolKey = $databaseHost . ':' . $database->getNamespace(); + if (!isset($initializedPools[$poolKey])) { + try { + $database->create(); + } catch (\Utopia\Database\Exception\Duplicate) { + // Schema already exists + } + $initializedPools[$poolKey] = true; + } + } elseif (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant($project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database diff --git a/app/worker.php b/app/worker.php index 71446ee94f..e54819e729 100644 --- a/app/worker.php +++ b/app/worker.php @@ -221,7 +221,9 @@ Server::setResource('getLogsDB', function (Group $pools, Cache $cache, Authoriza }, ['pools', 'cache', 'authorization']); Server::setResource('getDatabasesDB', function (Cache $cache, Registry $register, Document $project, Authorization $authorization) { - return function (Document $database, ?Document $projectDocument = null) use ($cache, $register, $project, $authorization): Database { + $initializedPools = []; + + return function (Document $database, ?Document $projectDocument = null) use ($cache, $register, $project, $authorization, &$initializedPools): Database { $projectDocument ??= $project; $databaseDSN = $database->getAttribute('database', $project->getAttribute('database', '')); $databaseType = $database->getAttribute('type', ''); @@ -246,7 +248,8 @@ Server::setResource('getDatabasesDB', function (Cache $cache, Registry $register } $pools = $register->get('pools'); - $pool = $pools->get($databaseDSN->getHost()); + $databaseHost = $databaseDSN->getHost(); + $pool = $pools->get($databaseHost); $adapter = new DatabasePool($pool); $database = new Database($adapter, $cache); @@ -255,12 +258,30 @@ Server::setResource('getDatabasesDB', function (Cache $cache, Registry $register ->setAuthorization($authorization); $database->getAdapter()->setSupportForAttributes($databaseType !== DOCUMENTSDB); - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + $sharedTables = \array_filter(\explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''))); - if (\in_array($dsn->getHost(), $sharedTables, true)) { + // When using a separate pool, always use dedicated mode with namespace isolation. + // Shared tables mode can't be used across different engines (e.g. MongoDB UUID + // tenants are incompatible with PostgreSQL's integer _tenant column). + if ($databaseHost !== $dsn->getHost()) { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $projectDocument->getSequence()); + + $poolKey = $databaseHost . ':' . $database->getNamespace(); + if (!isset($initializedPools[$poolKey])) { + try { + $database->create(); + } catch (\Utopia\Database\Exception\Duplicate) { + // Schema already exists + } + $initializedPools[$poolKey] = true; + } + } elseif (\in_array($dsn->getHost(), $sharedTables, true)) { $database ->setSharedTables(true) - ->setTenant((int) $projectDocument->getSequence()) + ->setTenant($projectDocument->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database diff --git a/composer.lock b/composer.lock index 1441bf06d1..9a91520978 100644 --- a/composer.lock +++ b/composer.lock @@ -3850,16 +3850,16 @@ }, { "name": "utopia-php/database", - "version": "5.3.17", + "version": "5.3.19", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "cff2b6ed63d3291b74110d086e16ff089fe05993" + "reference": "72ee1614c37e37c7fdd9d4dc87f1f7cdfa1ca691" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/cff2b6ed63d3291b74110d086e16ff089fe05993", - "reference": "cff2b6ed63d3291b74110d086e16ff089fe05993", + "url": "https://api.github.com/repos/utopia-php/database/zipball/72ee1614c37e37c7fdd9d4dc87f1f7cdfa1ca691", + "reference": "72ee1614c37e37c7fdd9d4dc87f1f7cdfa1ca691", "shasum": "" }, "require": { @@ -3903,9 +3903,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/5.3.17" + "source": "https://github.com/utopia-php/database/tree/5.3.19" }, - "time": "2026-03-20T01:18:52+00:00" + "time": "2026-03-31T15:52:08+00:00" }, { "name": "utopia-php/detector", diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php index 3585bc4477..3d07c65250 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Create.php @@ -61,16 +61,16 @@ class Create extends Action $databaseKeys = System::getEnv('_APP_DATABASE_DOCUMENTSDB_KEYS', ''); $databaseOverride = System::getEnv('_APP_DATABASE_DOCUMENTSDB_OVERRIDE'); $dbScheme = System::getEnv('_APP_DB_HOST_DOCUMENTSDB', 'mongodb'); - $databaseSharedTables = \explode(',', System::getEnv('_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES', '')); - $databaseSharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES_V1', '')); + $databaseSharedTables = \array_filter(\explode(',', System::getEnv('_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES', ''))); + $databaseSharedTablesV1 = \array_filter(\explode(',', System::getEnv('_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES_V1', ''))); break; case VECTORSDB: $databases = Config::getParam('pools-vectorsdb', []); $databaseKeys = System::getEnv('_APP_DATABASE_VECTORSDB_KEYS', ''); $databaseOverride = System::getEnv('_APP_DATABASE_VECTORSDB_OVERRIDE'); $dbScheme = System::getEnv('_APP_DB_HOST_VECTORSDB', 'postgresql'); - $databaseSharedTables = \explode(',', System::getEnv('_APP_DATABASE_VECTORSDB_SHARED_TABLES', '')); - $databaseSharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_VECTORSDB_SHARED_TABLES_V1', '')); + $databaseSharedTables = \array_filter(\explode(',', System::getEnv('_APP_DATABASE_VECTORSDB_SHARED_TABLES', ''))); + $databaseSharedTablesV1 = \array_filter(\explode(',', System::getEnv('_APP_DATABASE_VECTORSDB_SHARED_TABLES_V1', ''))); break; default: // legacy/tablesdb @@ -108,7 +108,7 @@ class Create extends Action if ($index !== false) { $selectedDsn = $databases[$index]; } else { - if (!empty($dsn)) { + if (!empty($dsn) && !empty($databaseSharedTables)) { $beforeFilter = \array_values($databases); if ($isSharedTablesV1) { $databases = array_filter($databases, fn ($value) => \in_array($value, $databaseSharedTablesV1)); @@ -118,7 +118,10 @@ class Create extends Action $databases = array_filter($databases, fn ($value) => !\in_array($value, $databaseSharedTables)); } } - $selectedDsn = !empty($databases) ? $databases[array_rand($databases)] : ''; + if (empty($databases)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "No {$databasetype} database pool available for the current shared-tables mode"); + } + $selectedDsn = $databases[array_rand($databases)]; } if (\in_array($selectedDsn, $databaseSharedTables)) { diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php index f6200ed209..a2bf23d402 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php @@ -3039,12 +3039,21 @@ class RealtimeCustomClientTest extends Scope $this->assertEquals(200, $update['headers']['status-code']); - $event = json_decode($client->receive(), true); + // Drain WebSocket messages until the .update event arrives. + // Earlier events (e.g. a late-arriving .create from the row seed above) are skipped. + $updateEvent = "tablesdb.{$databaseId}.tables.{$tableId}.rows.{$rowId}.update"; + $event = null; + $deadline = \time() + 10; + while (\time() < $deadline) { + $raw = $client->receive(); + $msg = json_decode($raw, true); + if (($msg['type'] ?? '') === 'event' && \in_array($updateEvent, $msg['data']['events'] ?? [])) { + $event = $msg; + break; + } + } - $this->assertArrayHasKey('type', $event); - $this->assertArrayHasKey('data', $event); - $this->assertEquals('event', $event['type']); - $this->assertNotEmpty($event['data']); + $this->assertNotNull($event, 'Timed out waiting for the row update event'); $this->assertArrayHasKey('timestamp', $event['data']); $channels = $event['data']['channels'];