From c5926c9134b92238818e3a6df5d8cd5a583187ac Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 19 May 2026 16:06:35 +0530 Subject: [PATCH 1/9] Refactor shared database resources --- app/init/resources.php | 297 +++++++++++++++++++++------------ app/init/resources/request.php | 128 +++----------- app/init/worker/message.php | 145 +++------------- 3 files changed, 241 insertions(+), 329 deletions(-) diff --git a/app/init/resources.php b/app/init/resources.php index 7b7e13482c..f2bf57b644 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -48,17 +48,171 @@ use Utopia\Telemetry\Adapter as Telemetry; use Utopia\Telemetry\Adapter\None as NoTelemetry; use Utopia\VCS\Adapter\Git\GitHub as VcsGitHub; -// Runtime Execution global $register; global $container; + +function getProjectGlobalCollections(): array +{ + /** @var array $collections */ + $collections = Config::getParam('collections', []); + $projectCollections = $collections['projects'] ?? []; + $projectsGlobalCollections = array_keys($projectCollections); + $projectsGlobalCollections[] = 'audit'; + + return $projectsGlobalCollections; +} + +function configureProjectDatabase(Database $database, Document $project, DSN $dsn): Database +{ + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + + if (\in_array($dsn->getHost(), $sharedTables)) { + $database + ->setSharedTables(true) + ->setGlobalCollections(getProjectGlobalCollections()) + ->setTenant($project->getSequence()) + ->setNamespace($dsn->getParam('namespace')); + } else { + $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getSequence()); + } + + return $database; +} + +function createPlatformDatabase( + Group $pools, + Cache $cache, + Authorization $authorization, + int $timeout = 0, + int $maxQueryValues = 0, + bool $metadata = true, +): Database { + $adapter = new DatabasePool($pools->get('console')); + $database = new Database($adapter, $cache); + + $database + ->setDatabase(APP_DATABASE) + ->setAuthorization($authorization) + ->setNamespace('_console'); + + if ($metadata) { + $database + ->setMetadata('host', \gethostname()) + ->setMetadata('project', 'console'); + } + + if ($timeout > 0) { + $database->setTimeout($timeout); + } + + if ($maxQueryValues > 0) { + $database->setMaxQueryValues($maxQueryValues); + } + + $database->setDocumentType('users', User::class); + + return $database; +} + +function createProjectDatabase( + Group $pools, + Cache $cache, + Authorization $authorization, + Document $project, + int $timeout = 0, + int $maxQueryValues = 0, + bool $metadata = true, +): Database { + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + $adapter = new DatabasePool($pools->get($dsn->getHost())); + $database = new Database($adapter, $cache); + + $database + ->setDatabase(APP_DATABASE) + ->setAuthorization($authorization); + + if ($metadata) { + $database + ->setMetadata('host', \gethostname()) + ->setMetadata('project', $project->getId()); + } + + if ($timeout > 0) { + $database->setTimeout($timeout); + } + + if ($maxQueryValues > 0) { + $database->setMaxQueryValues($maxQueryValues); + } + + $database->setDocumentType('users', User::class); + + return configureProjectDatabase($database, $project, $dsn); +} + +function createLogsDatabase( + Group $pools, + Cache $cache, + Authorization $authorization, + int $timeout = 0, + int $maxQueryValues = 0, + ?DatabasePool $adapter = null, +): Database { + /** @var array $collections */ + $collections = Config::getParam('collections', []); + $logsCollections = $collections['logs'] ?? []; + $logsCollections = array_keys($logsCollections); + + $adapter ??= new DatabasePool($pools->get('logs')); + $database = new Database($adapter, $cache); + + $database + ->setDatabase(APP_DATABASE) + ->setAuthorization($authorization) + ->setSharedTables(true) + ->setGlobalCollections($logsCollections) + ->setNamespace('logsV1'); + + if ($timeout > 0) { + $database->setTimeout($timeout); + } + + if ($maxQueryValues > 0) { + $database->setMaxQueryValues($maxQueryValues); + } + + return $database; +} + $container = new Container(); +$container->set('register', fn () => $register); + +$container->set('logger', fn ($register) => $register->get('logger'), ['register']); + +$container->set('hooks', fn ($register) => $register->get('hooks'), ['register']); + $container->set('console', fn () => new Document(Config::getParam('console')), []); +$container->set('platform', fn () => Config::getParam('platform', []), []); + +$container->set('localeCodes', fn () => array_map(fn ($locale) => $locale['code'], Config::getParam('locale-codes', []))); + $container->set('executor', fn () => new Executor(), []); $container->set('telemetry', fn () => new NoTelemetry(), []); +$container->set('authorization', fn () => new Authorization(), []); + $container->set('publisher', fn (Group $pools) => new BrokerPool(publisher: $pools->get('publisher')), ['pools']); $container->set('publisherDatabases', fn (Publisher $publisher) => $publisher, ['publisher']); @@ -140,50 +294,13 @@ $container->set('publisherForMessaging', fn (Publisher $publisher) => new Messag new Queue(System::getEnv('_APP_MESSAGING_QUEUE_NAME', Event::MESSAGING_QUEUE_NAME)) ), ['publisher']); -$container->set('logger', function ($register) { - return $register->get('logger'); -}, ['register']); - -$container->set('hooks', function ($register) { - return $register->get('hooks'); -}, ['register']); - -$container->set('register', fn () => $register); - -$container->set('localeCodes', function () { - return array_map(fn ($locale) => $locale['code'], Config::getParam('locale-codes', [])); -}); - - -/** - * Platform configuration - */ -$container->set('platform', function () { - return Config::getParam('platform', []); -}, []); - -$container->set('authorization', function () { - return new Authorization(); -}, []); - -$container->set('dbForPlatform', function (Group $pools, Cache $cache, Authorization $authorization) { - - $adapter = new DatabasePool($pools->get('console')); - $database = new Database($adapter, $cache); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setNamespace('_console') - ->setMetadata('host', \gethostname()) - ->setMetadata('project', 'console') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); - - $database->setDocumentType('users', User::class); - - return $database; -}, ['pools', 'cache', 'authorization']); +$container->set('dbForPlatform', fn (Group $pools, Cache $cache, Authorization $authorization) => createPlatformDatabase( + $pools, + $cache, + $authorization, + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES +), ['pools', 'cache', 'authorization']); $container->set('getLogsDB', function (Group $pools, Cache $cache, Authorization $authorization) { $database = null; @@ -194,22 +311,13 @@ $container->set('getLogsDB', function (Group $pools, Cache $cache, Authorization return $database; } - $adapter = new DatabasePool($pools->get('logs')); - $database = new Database($adapter, $cache); - - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $logsCollections = $collections['logs'] ?? []; - $logsCollections = array_keys($logsCollections); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setSharedTables(true) - ->setGlobalCollections($logsCollections) - ->setNamespace('logsV1') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); + $database = createLogsDatabase( + $pools, + $cache, + $authorization, + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES + ); // set tenant if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { @@ -234,9 +342,7 @@ $container->set('cache', function (Group $pools, Telemetry $telemetry) { return $cache; }, ['pools', 'telemetry']); -$container->set('cacheControlForStorage', fn () => function (StorageCacheControl $config): string { - return \sprintf('private, max-age=%d', $config->maxAge); -}); +$container->set('cacheControlForStorage', fn () => fn (StorageCacheControl $config): string => \sprintf('private, max-age=%d', $config->maxAge)); $container->set('redis', function () { $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); @@ -253,25 +359,14 @@ $container->set('redis', function () { return $redis; }); -$container->set('locks', function (Group $pools) { - return function (string $key, int $ttl, callable $callback, float $timeout = 0.0) use ($pools): mixed { - return $pools->get('lock')->use(function (\Redis $redis) use ($key, $ttl, $callback, $timeout) { - $lock = new Distributed($redis, $key, ttl: $ttl); +$container->set('locks', fn (Group $pools) => fn (string $key, int $ttl, callable $callback, float $timeout = 0.0): mixed => $pools->get('lock')->use( + fn (\Redis $redis) => (new Distributed($redis, $key, ttl: $ttl))->withLock($callback, timeout: $timeout) +), ['pools']); - return $lock->withLock($callback, timeout: $timeout); - }); - }; -}, ['pools']); +$container->set('timelimit', fn (\Redis $redis) => fn (string $key, int $limit, int $time) => new TimeLimitRedis($key, $limit, $time, $redis), ['redis']); -$container->set('timelimit', function (\Redis $redis) { - return function (string $key, int $limit, int $time) use ($redis) { - return new TimeLimitRedis($key, $limit, $time, $redis); - }; -}, ['redis']); +$container->set('deviceForLocal', fn (Telemetry $telemetry) => new Device\Telemetry($telemetry, new Local()), ['telemetry']); -$container->set('deviceForLocal', function (Telemetry $telemetry) { - return new Device\Telemetry($telemetry, new Local()); -}, ['telemetry']); function getDevice(string $root, string $connection = ''): Device { $connection = ! empty($connection) ? $connection : System::getEnv('_APP_CONNECTIONS_STORAGE', ''); @@ -299,14 +394,14 @@ function getDevice(string $root, string $connection = ''): Device switch ($device) { case Storage::DEVICE_S3: if (! empty($url)) { - $bucketRoot = (! empty($bucket) ? $bucket . '/' : '') . \ltrim($root, '/'); + $bucketRoot = (! empty($bucket) ? "{$bucket}/" : '') . \ltrim($root, '/'); return new S3($bucketRoot, $accessKey, $accessSecret, $url, $region, $acl); } else { return new AWS($root, $accessKey, $accessSecret, $bucket, $region, $acl); } // no break - case STORAGE::DEVICE_DO_SPACES: + case Storage::DEVICE_DO_SPACES: $device = new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); $device->setHttpVersion(S3::HTTP_VERSION_1_1); @@ -323,9 +418,6 @@ function getDevice(string $root, string $connection = ''): Device } } else { switch (strtolower(System::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL))) { - case Storage::DEVICE_LOCAL: - default: - return new Local($root); case Storage::DEVICE_S3: $s3AccessKey = System::getEnv('_APP_STORAGE_S3_ACCESS_KEY', ''); $s3SecretKey = System::getEnv('_APP_STORAGE_S3_SECRET', ''); @@ -334,7 +426,7 @@ function getDevice(string $root, string $connection = ''): Device $s3Acl = 'private'; $s3EndpointUrl = System::getEnv('_APP_STORAGE_S3_ENDPOINT', ''); if (! empty($s3EndpointUrl)) { - $bucketRoot = (! empty($s3Bucket) ? $s3Bucket . '/' : '') . \ltrim($root, '/'); + $bucketRoot = (! empty($s3Bucket) ? "{$s3Bucket}/" : '') . \ltrim($root, '/'); return new S3($bucketRoot, $s3AccessKey, $s3SecretKey, $s3EndpointUrl, $s3Region, $s3Acl); } else { @@ -375,46 +467,33 @@ function getDevice(string $root, string $connection = ''): Device $wasabiAcl = 'private'; return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); + case Storage::DEVICE_LOCAL: + default: + return new Local($root); } } } -$container->set('geodb', function ($register) { - /** @var Utopia\Registry\Registry $register */ - return $register->get('geodb'); -}, ['register']); +$container->set('geodb', fn ($register) => $register->get('geodb'), ['register']); -$container->set('passwordsDictionary', function ($register) { - /** @var Utopia\Registry\Registry $register */ - return $register->get('passwordsDictionary'); -}, ['register']); +$container->set('passwordsDictionary', fn ($register) => $register->get('passwordsDictionary'), ['register']); $container->set('servers', function () { $platforms = Config::getParam('sdks'); $server = $platforms[APP_SDK_PLATFORM_SERVER]; - $languages = array_map(function ($language) { - return strtolower($language['name']); - }, $server['sdks']); + $languages = array_map(fn ($language) => strtolower($language['name']), $server['sdks']); return $languages; }); -$container->set('promiseAdapter', function ($register) { - return $register->get('promiseAdapter'); -}, ['register']); +$container->set('promiseAdapter', fn ($register) => $register->get('promiseAdapter'), ['register']); -$container->set('gitHub', function (Cache $cache) { - return new VcsGitHub($cache); -}, ['cache']); +$container->set('gitHub', fn (Cache $cache) => new VcsGitHub($cache), ['cache']); -$container->set('plan', function () { - return []; -}); +$container->set('plan', fn () => []); -$container->set('smsRates', function () { - return []; -}); +$container->set('smsRates', fn () => []); $container->set( 'isResourceBlocked', diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 85d8db3698..11f9acfd56 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -116,23 +116,13 @@ return function (Container $context): void { new Queue(System::getEnv('_APP_FUNCTIONS_QUEUE_NAME', Event::FUNCTIONS_QUEUE_NAME), 'utopia-queue', Event::FUNCTIONS_QUEUE_TTL) ), ['publisher']); $context->set('eventProcessor', fn () => new EventProcessor(), []); - $context->set('dbForPlatform', function (Group $pools, Cache $cache, Authorization $authorization) { - $adapter = new DatabasePool($pools->get('console')); - $database = new Database($adapter, $cache); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setNamespace('_console') - ->setMetadata('host', \gethostname()) - ->setMetadata('project', 'console') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); - - $database->setDocumentType('users', User::class); - - return $database; - }, ['pools', 'cache', 'authorization']); + $context->set('dbForPlatform', fn (Group $pools, Cache $cache, Authorization $authorization) => createPlatformDatabase( + $pools, + $cache, + $authorization, + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES + ), ['pools', 'cache', 'authorization']); $context->set('getProjectDB', function (Group $pools, Database $dbForPlatform, Cache $cache, Authorization $authorization) { $adapters = []; @@ -166,28 +156,7 @@ return function (Container $context): void { ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES) ->setDocumentType('users', User::class); - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - if (\in_array($dsn->getHost(), $sharedTables)) { - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $projectCollections = $collections['projects'] ?? []; - $projectsGlobalCollections = array_keys($projectCollections); - $projectsGlobalCollections[] = 'audit'; - - $database - ->setSharedTables(true) - ->setTenant($project->getSequence()) - ->setGlobalCollections($projectsGlobalCollections) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } - - return $database; + return configureProjectDatabase($database, $project, $dsn); }; }, ['pools', 'dbForPlatform', 'cache', 'authorization']); @@ -195,22 +164,14 @@ return function (Container $context): void { $adapter = null; return function (?Document $project = null) use ($pools, $cache, $authorization, &$adapter) { - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $logsCollections = $collections['logs'] ?? []; - $logsCollections = array_keys($logsCollections); - - $adapter ??= new DatabasePool($pools->get('logs')); - $database = new Database($adapter, $cache); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setSharedTables(true) - ->setGlobalCollections($logsCollections) - ->setNamespace('logsV1') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); + $database = createLogsDatabase( + $pools, + $cache, + $authorization, + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES, + $adapter ??= new DatabasePool($pools->get('logs')) + ); if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { $database->setTenant($project->getSequence()); @@ -646,45 +607,14 @@ return function (Container $context): void { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Project database is not configured'); } - try { - $dsn = new DSN($database); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $database); - } - - $adapter = new DatabasePool($pools->get($dsn->getHost())); - $database = new Database($adapter, $cache); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setMetadata('host', \gethostname()) - ->setMetadata('project', $project->getId()) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); - $database->setDocumentType('users', User::class); - - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - if (\in_array($dsn->getHost(), $sharedTables)) { - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $projectCollections = $collections['projects'] ?? []; - $projectsGlobalCollections = array_keys($projectCollections); - $projectsGlobalCollections[] = 'audit'; - - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($project->getSequence()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } + $database = createProjectDatabase( + $pools, + $cache, + $authorization, + $project, + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES + ); /** * This isolated event handling for `users.*.create` which is based on a `Database::EVENT_DOCUMENT_CREATE` listener may look odd, but it is **intentional**. @@ -1257,13 +1187,13 @@ return function (Container $context): void { $databaseDSN = $database->getAttribute('database') ?: $project->getAttribute('database', ''); $databaseType = $database->getAttribute('type', ''); + // databaseDSN determines the adapter try { $databaseDSN = new DSN($databaseDSN); } catch (\InvalidArgumentException) { - // for old databases migrated through patch script - // databaseDSN determines the adapter $databaseDSN = new DSN('mysql://' . $databaseDSN); } + try { $dsn = new DSN($project->getAttribute('database')); } catch (\InvalidArgumentException) { @@ -1278,11 +1208,7 @@ return function (Container $context): void { $database = new Database($adapter, $cache); $sharedTables = \array_filter(\explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''))); - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $projectCollections = $collections['projects'] ?? []; - $projectsGlobalCollections = array_keys($projectCollections); - $projectsGlobalCollections[] = 'audit'; + $projectsGlobalCollections = getProjectGlobalCollections(); $database ->setDatabase(APP_DATABASE) diff --git a/app/init/worker/message.php b/app/init/worker/message.php index 5cabfc7859..257e228fef 100644 --- a/app/init/worker/message.php +++ b/app/init/worker/message.php @@ -4,7 +4,6 @@ use Appwrite\Event\Event; use Appwrite\Event\Realtime; use Appwrite\Event\Webhook; use Appwrite\Usage\Context; -use Appwrite\Utopia\Database\Documents\User; use Utopia\Audit\Adapter\Database as AdapterDatabase; use Utopia\Audit\Audit as UtopiaAudit; use Utopia\Cache\Cache; @@ -42,18 +41,12 @@ return function (Container $container): void { return $authorization; }, []); - $container->set('dbForPlatform', function (Cache $cache, Group $pools, Authorization $authorization) { - $adapter = new DatabasePool($pools->get('console')); - $dbForPlatform = new Database($adapter, $cache); - - $dbForPlatform - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setNamespace('_console') - ->setDocumentType('users', User::class); - - return $dbForPlatform; - }, ['cache', 'pools', 'authorization']); + $container->set('dbForPlatform', fn (Cache $cache, Group $pools, Authorization $authorization) => createPlatformDatabase( + $pools, + $cache, + $authorization, + metadata: false + ), ['cache', 'pools', 'authorization']); $container->set('project', function ($message, Database $dbForPlatform) { $payload = $message->getPayload() ?? []; @@ -66,50 +59,16 @@ return function (Container $container): void { return $dbForPlatform->getDocument('projects', $project->getId()); }, ['message', 'dbForPlatform']); - $container->set('dbForProject', function (Cache $cache, Group $pools, Document $project, Database $dbForPlatform, Authorization $authorization) { - if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForPlatform; - } - - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - - $adapter = new DatabasePool($pools->get($dsn->getHost())); - $database = new Database($adapter, $cache); - $database->setDocumentType('users', User::class); - - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - if (\in_array($dsn->getHost(), $sharedTables)) { - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $projectCollections = $collections['projects'] ?? []; - $projectsGlobalCollections = array_keys($projectCollections); - $projectsGlobalCollections[] = 'audit'; - - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($project->getSequence()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER); - - return $database; - }, ['cache', 'pools', 'project', 'dbForPlatform', 'authorization']); + $container->set('dbForProject', fn (Cache $cache, Group $pools, Document $project, Database $dbForPlatform, Authorization $authorization) => $project->isEmpty() || $project->getId() === 'console' + ? $dbForPlatform + : createProjectDatabase( + $pools, + $cache, + $authorization, + $project, + APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER, + metadata: false + ), ['cache', 'pools', 'project', 'dbForPlatform', 'authorization']); $container->set('getProjectDB', function (Group $pools, Database $dbForPlatform, Cache $cache, Authorization $authorization) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools @@ -129,28 +88,8 @@ return function (Container $container): void { if (isset($databases[$dsn->getHost()])) { $database = $databases[$dsn->getHost()]; $database->setAuthorization($authorization); - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - if (\in_array($dsn->getHost(), $sharedTables)) { - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $projectCollections = $collections['projects'] ?? []; - $projectsGlobalCollections = array_keys($projectCollections); - $projectsGlobalCollections[] = 'audit'; - - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($project->getSequence()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } - - return $database; + return configureProjectDatabase($database, $project, $dsn); } $adapter = new DatabasePool($pools->get($dsn->getHost())); @@ -158,26 +97,7 @@ return function (Container $container): void { $databases[$dsn->getHost()] = $database; - $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); - - if (\in_array($dsn->getHost(), $sharedTables)) { - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $projectCollections = $collections['projects'] ?? []; - $projectsGlobalCollections = array_keys($projectCollections); - $projectsGlobalCollections[] = 'audit'; - - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($project->getSequence()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } + $database = configureProjectDatabase($database, $project, $dsn); $database ->setDatabase(APP_DATABASE) @@ -226,11 +146,7 @@ return function (Container $container): void { $sharedTables = \array_filter(\explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''))); - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $projectCollections = $collections['projects'] ?? []; - $projectsGlobalCollections = array_keys($projectCollections); - $projectsGlobalCollections[] = 'audit'; + $projectsGlobalCollections = getProjectGlobalCollections(); $database->setGlobalCollections($projectsGlobalCollections); @@ -283,22 +199,13 @@ return function (Container $container): void { return $database; } - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $logsCollections = $collections['logs'] ?? []; - $logsCollections = array_keys($logsCollections); - - $adapter = new DatabasePool($pools->get('logs')); - $database = new Database($adapter, $cache); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setSharedTables(true) - ->setGlobalCollections($logsCollections) - ->setNamespace('logsV1') - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES_WORKER); + $database = createLogsDatabase( + $pools, + $cache, + $authorization, + APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER, + APP_DATABASE_QUERY_MAX_VALUES_WORKER + ); if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { $database->setTenant($project->getSequence()); From a96fb2445b023067d399c1ec535c8a2a78c1762c Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 19 May 2026 18:18:09 +0530 Subject: [PATCH 2/9] Refactor database initialization factory --- app/init/resources.php | 175 ++--------------- app/init/resources/request.php | 165 ++++------------ app/init/worker/message.php | 164 ++++------------ src/Appwrite/Database/DatabaseFactory.php | 227 ++++++++++++++++++++++ 4 files changed, 315 insertions(+), 416 deletions(-) create mode 100644 src/Appwrite/Database/DatabaseFactory.php diff --git a/app/init/resources.php b/app/init/resources.php index f2bf57b644..b388eed0c6 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -1,5 +1,6 @@ getHost(), $sharedTables)) { - $database - ->setSharedTables(true) - ->setGlobalCollections(getProjectGlobalCollections()) - ->setTenant($project->getSequence()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } - - return $database; -} - -function createPlatformDatabase( - Group $pools, - Cache $cache, - Authorization $authorization, - int $timeout = 0, - int $maxQueryValues = 0, - bool $metadata = true, -): Database { - $adapter = new DatabasePool($pools->get('console')); - $database = new Database($adapter, $cache); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setNamespace('_console'); - - if ($metadata) { - $database - ->setMetadata('host', \gethostname()) - ->setMetadata('project', 'console'); - } - - if ($timeout > 0) { - $database->setTimeout($timeout); - } - - if ($maxQueryValues > 0) { - $database->setMaxQueryValues($maxQueryValues); - } - - $database->setDocumentType('users', User::class); - - return $database; -} - -function createProjectDatabase( - Group $pools, - Cache $cache, - Authorization $authorization, - Document $project, - int $timeout = 0, - int $maxQueryValues = 0, - bool $metadata = true, -): Database { - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - - $adapter = new DatabasePool($pools->get($dsn->getHost())); - $database = new Database($adapter, $cache); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization); - - if ($metadata) { - $database - ->setMetadata('host', \gethostname()) - ->setMetadata('project', $project->getId()); - } - - if ($timeout > 0) { - $database->setTimeout($timeout); - } - - if ($maxQueryValues > 0) { - $database->setMaxQueryValues($maxQueryValues); - } - - $database->setDocumentType('users', User::class); - - return configureProjectDatabase($database, $project, $dsn); -} - -function createLogsDatabase( - Group $pools, - Cache $cache, - Authorization $authorization, - int $timeout = 0, - int $maxQueryValues = 0, - ?DatabasePool $adapter = null, -): Database { - /** @var array $collections */ - $collections = Config::getParam('collections', []); - $logsCollections = $collections['logs'] ?? []; - $logsCollections = array_keys($logsCollections); - - $adapter ??= new DatabasePool($pools->get('logs')); - $database = new Database($adapter, $cache); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setSharedTables(true) - ->setGlobalCollections($logsCollections) - ->setNamespace('logsV1'); - - if ($timeout > 0) { - $database->setTimeout($timeout); - } - - if ($maxQueryValues > 0) { - $database->setMaxQueryValues($maxQueryValues); - } - - return $database; -} - $container = new Container(); $container->set('register', fn () => $register); @@ -294,39 +150,36 @@ $container->set('publisherForMessaging', fn (Publisher $publisher) => new Messag new Queue(System::getEnv('_APP_MESSAGING_QUEUE_NAME', Event::MESSAGING_QUEUE_NAME)) ), ['publisher']); -$container->set('dbForPlatform', fn (Group $pools, Cache $cache, Authorization $authorization) => createPlatformDatabase( +$container->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory( $pools, $cache, - $authorization, - APP_DATABASE_TIMEOUT_MILLISECONDS_API, - APP_DATABASE_QUERY_MAX_VALUES + $authorization ), ['pools', 'cache', 'authorization']); -$container->set('getLogsDB', function (Group $pools, Cache $cache, Authorization $authorization) { +$container->set('dbForPlatform', fn (DatabaseFactory $databaseFactory) => $databaseFactory->platform( + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES, + ['host' => \gethostname(), 'project' => 'console'] +), ['databaseFactory']); + +$container->set('getLogsDB', function (DatabaseFactory $databaseFactory) { $database = null; - return function (?Document $project = null) use ($pools, $cache, $authorization, &$database) { + return function (?Document $project = null) use ($databaseFactory, &$database) { if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { $database->setTenant($project->getSequence()); return $database; } - $database = createLogsDatabase( - $pools, - $cache, - $authorization, + $database = $databaseFactory->logs( + $project, APP_DATABASE_TIMEOUT_MILLISECONDS_API, APP_DATABASE_QUERY_MAX_VALUES ); - // set tenant - if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant($project->getSequence()); - } - return $database; }; -}, ['pools', 'cache', 'authorization']); +}, ['databaseFactory']); $container->set('cache', function (Group $pools, Telemetry $telemetry) { $list = Config::getParam('pools-cache', []); diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 11f9acfd56..cf7bb2a3fc 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -3,6 +3,7 @@ use Ahc\Jwt\JWT; use Ahc\Jwt\JWTException; use Appwrite\Auth\Key; +use Appwrite\Database\DatabaseFactory; use Appwrite\Databases\TransactionState; use Appwrite\Event\Context\Audit as AuditContext; use Appwrite\Event\Event; @@ -33,7 +34,6 @@ use Utopia\Auth\Proofs\Token; use Utopia\Auth\Store; use Utopia\Cache\Cache; use Utopia\Config\Config; -use Utopia\Database\Adapter\Pool as DatabasePool; use Utopia\Database\Database; use Utopia\Database\DateTime as DatabaseDateTime; use Utopia\Database\Document; @@ -41,7 +41,6 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\DI\Container; use Utopia\Domains\Domain; -use Utopia\DSN\DSN; use Utopia\Http\Http; use Utopia\Locale\Locale; use Utopia\Logger\Log; @@ -116,18 +115,21 @@ return function (Container $context): void { new Queue(System::getEnv('_APP_FUNCTIONS_QUEUE_NAME', Event::FUNCTIONS_QUEUE_NAME), 'utopia-queue', Event::FUNCTIONS_QUEUE_TTL) ), ['publisher']); $context->set('eventProcessor', fn () => new EventProcessor(), []); - $context->set('dbForPlatform', fn (Group $pools, Cache $cache, Authorization $authorization) => createPlatformDatabase( + $context->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory( $pools, $cache, - $authorization, - APP_DATABASE_TIMEOUT_MILLISECONDS_API, - APP_DATABASE_QUERY_MAX_VALUES + $authorization ), ['pools', 'cache', 'authorization']); - $context->set('getProjectDB', function (Group $pools, Database $dbForPlatform, Cache $cache, Authorization $authorization) { - $adapters = []; + $context->set('dbForPlatform', fn (DatabaseFactory $databaseFactory) => $databaseFactory->platform( + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES, + ['host' => \gethostname(), 'project' => 'console'] + ), ['databaseFactory']); - return function (Document $project) use ($pools, $dbForPlatform, $cache, $authorization, &$adapters) { + $context->set('getProjectDB', function (DatabaseFactory $databaseFactory, Database $dbForPlatform) { + + return function (Document $project) use ($databaseFactory, $dbForPlatform) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -137,49 +139,25 @@ return function (Container $context): void { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Project database is not configured'); } - try { - $dsn = new DSN($database); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $database); - } - - $adapter = $adapters[$dsn->getHost()] ??= new DatabasePool($pools->get($dsn->getHost())); - $database = new Database($adapter, $cache); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setMetadata('host', \gethostname()) - ->setMetadata('project', $project->getId()) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES) - ->setDocumentType('users', User::class); - - return configureProjectDatabase($database, $project, $dsn); - }; - }, ['pools', 'dbForPlatform', 'cache', 'authorization']); - - $context->set('getLogsDB', function (Group $pools, Cache $cache, Authorization $authorization) { - $adapter = null; - - return function (?Document $project = null) use ($pools, $cache, $authorization, &$adapter) { - $database = createLogsDatabase( - $pools, - $cache, - $authorization, + return $databaseFactory->project( + $project, APP_DATABASE_TIMEOUT_MILLISECONDS_API, APP_DATABASE_QUERY_MAX_VALUES, - $adapter ??= new DatabasePool($pools->get('logs')) + ['host' => \gethostname(), 'project' => $project->getId()] ); - - if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant($project->getSequence()); - } - - return $database; }; - }, ['pools', 'cache', 'authorization']); + }, ['databaseFactory', 'dbForPlatform']); + + $context->set('getLogsDB', function (DatabaseFactory $databaseFactory) { + + return function (?Document $project = null) use ($databaseFactory) { + return $databaseFactory->logs( + $project, + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES + ); + }; + }, ['databaseFactory']); /** * List of allowed request hostnames for the request. @@ -597,7 +575,7 @@ return function (Container $context): void { return; }, ['user', 'store', 'proofForToken']); - $context->set('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project, Response $response, Publisher $publisher, Publisher $publisherFunctions, Publisher $publisherWebhooks, Event $queueForEvents, FunctionPublisher $publisherForFunctions, Webhook $queueForWebhooks, Realtime $queueForRealtime, UsageContext $usage, Authorization $authorization, Request $request) { + $context->set('dbForProject', function (DatabaseFactory $databaseFactory, Database $dbForPlatform, Document $project, Response $response, Publisher $publisher, Publisher $publisherFunctions, Publisher $publisherWebhooks, Event $queueForEvents, FunctionPublisher $publisherForFunctions, Webhook $queueForWebhooks, Realtime $queueForRealtime, UsageContext $usage, Request $request) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -607,13 +585,11 @@ return function (Container $context): void { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Project database is not configured'); } - $database = createProjectDatabase( - $pools, - $cache, - $authorization, + $database = $databaseFactory->project( $project, APP_DATABASE_TIMEOUT_MILLISECONDS_API, - APP_DATABASE_QUERY_MAX_VALUES + APP_DATABASE_QUERY_MAX_VALUES, + ['host' => \gethostname(), 'project' => $project->getId()] ); /** @@ -845,7 +821,7 @@ return function (Container $context): void { ->on(Database::EVENT_DOCUMENT_DELETE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)); return $database; - }, ['pools', 'dbForPlatform', 'cache', 'project', 'response', 'publisher', 'publisherFunctions', 'publisherWebhooks', 'queueForEvents', 'publisherForFunctions', 'queueForWebhooks', 'queueForRealtime', 'usage', 'authorization', 'request']); + }, ['databaseFactory', 'dbForPlatform', 'project', 'response', 'publisher', 'publisherFunctions', 'publisherWebhooks', 'queueForEvents', 'publisherForFunctions', 'queueForWebhooks', 'queueForRealtime', 'usage', 'request']); $context->set('schema', function ($utopia, $dbForProject, $authorization) { @@ -1181,78 +1157,19 @@ return function (Container $context): void { return new Document([]); }, ['project', 'dbForProject', 'request', 'authorization']); - $context->set('getDatabasesDB', function (Group $pools, Cache $cache, Document $project, Request $request, UsageContext $usage, Authorization $authorization) { + $context->set('getDatabasesDB', function (DatabaseFactory $databaseFactory, Document $project, Request $request, UsageContext $usage) { - return function (Document $database) use ($pools, $cache, $project, $request, $usage, $authorization): Database { - $databaseDSN = $database->getAttribute('database') ?: $project->getAttribute('database', ''); + return function (Document $database) use ($databaseFactory, $project, $request, $usage): Database { $databaseType = $database->getAttribute('type', ''); - // databaseDSN determines the adapter - try { - $databaseDSN = new DSN($databaseDSN); - } catch (\InvalidArgumentException) { - $databaseDSN = new DSN('mysql://' . $databaseDSN); - } + $database = $databaseFactory->tenant( + $database, + $project, + APP_DATABASE_TIMEOUT_MILLISECONDS_API, + APP_DATABASE_QUERY_MAX_VALUES, + ['host' => \gethostname(), 'project' => $project->getId()] + ); - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - - $databaseHost = $databaseDSN->getHost(); - $pool = $pools->get($databaseHost); - - $adapter = new DatabasePool($pool); - $database = new Database($adapter, $cache); - $sharedTables = \array_filter(\explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''))); - - $projectsGlobalCollections = getProjectGlobalCollections(); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setMetadata('host', \gethostname()) - ->setMetadata('project', $project->getId()) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) - ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); - // inside pools authorization needs to be set first - $database->getAdapter()->setSupportForAttributes($databaseType !== DOCUMENTSDB); - - // For separate pools (documentsdb/vectorsdb), check their own shared tables config. - // If not configured, use dedicated mode to avoid cross-engine tenant type mismatches. - if ($databaseHost !== $dsn->getHost()) { - $dbTypeSharedTables = match ($databaseType) { - DOCUMENTSDB => \array_filter(\explode(',', System::getEnv('_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES', ''))), - VECTORSDB => \array_filter(\explode(',', System::getEnv('_APP_DATABASE_VECTORSDB_SHARED_TABLES', ''))), - default => [], - }; - - if (\in_array($databaseHost, $dbTypeSharedTables)) { - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($project->getSequence()) - ->setNamespace($databaseDSN->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } - } elseif (\in_array($dsn->getHost(), $sharedTables)) { - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($project->getSequence()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $project->getSequence()); - } $timeout = \intval($request->getHeader('x-appwrite-timeout')); if (!empty($timeout) && Http::isDevelopment()) { $database->setTimeout($timeout); @@ -1337,7 +1254,7 @@ return function (Container $context): void { return $database; }; - }, ['pools', 'cache', 'project', 'request', 'usage', 'authorization']); + }, ['databaseFactory', 'project', 'request', 'usage']); $context->set( 'transactionState', diff --git a/app/init/worker/message.php b/app/init/worker/message.php index 257e228fef..6875351def 100644 --- a/app/init/worker/message.php +++ b/app/init/worker/message.php @@ -1,5 +1,6 @@ set('dbForPlatform', fn (Cache $cache, Group $pools, Authorization $authorization) => createPlatformDatabase( + $container->set('databaseFactory', fn (Group $pools, Cache $cache, Authorization $authorization) => new DatabaseFactory( $pools, $cache, - $authorization, - metadata: false - ), ['cache', 'pools', 'authorization']); + $authorization + ), ['pools', 'cache', 'authorization']); + + $container->set('dbForPlatform', fn (DatabaseFactory $databaseFactory) => $databaseFactory->platform(), ['databaseFactory']); $container->set('project', function ($message, Database $dbForPlatform) { $payload = $message->getPayload() ?? []; @@ -59,161 +58,64 @@ return function (Container $container): void { return $dbForPlatform->getDocument('projects', $project->getId()); }, ['message', 'dbForPlatform']); - $container->set('dbForProject', fn (Cache $cache, Group $pools, Document $project, Database $dbForPlatform, Authorization $authorization) => $project->isEmpty() || $project->getId() === 'console' + $container->set('dbForProject', fn (DatabaseFactory $databaseFactory, Document $project, Database $dbForPlatform) => $project->isEmpty() || $project->getId() === 'console' ? $dbForPlatform - : createProjectDatabase( - $pools, - $cache, - $authorization, + : $databaseFactory->project( $project, APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER, - metadata: false - ), ['cache', 'pools', 'project', 'dbForPlatform', 'authorization']); + ), ['databaseFactory', 'project', 'dbForPlatform']); - $container->set('getProjectDB', function (Group $pools, Database $dbForPlatform, Cache $cache, Authorization $authorization) { - $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - - return function (Document $project) use ($pools, $dbForPlatform, $cache, $authorization, &$databases): Database { + $container->set('getProjectDB', function (DatabaseFactory $databaseFactory, Database $dbForPlatform) { + return function (Document $project) use ($databaseFactory, $dbForPlatform): Database { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - - if (isset($databases[$dsn->getHost()])) { - $database = $databases[$dsn->getHost()]; - $database->setAuthorization($authorization); - - return configureProjectDatabase($database, $project, $dsn); - } - - $adapter = new DatabasePool($pools->get($dsn->getHost())); - $database = new Database($adapter, $cache); - - $databases[$dsn->getHost()] = $database; - - $database = configureProjectDatabase($database, $project, $dsn); - - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization) - ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER); - - return $database; + return $databaseFactory->project( + $project, + APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER + ); }; - }, ['pools', 'dbForPlatform', 'cache', 'authorization']); + }, ['databaseFactory', 'dbForPlatform']); - $container->set('getDatabasesDB', function (Cache $cache, Registry $register, Document $project, Authorization $authorization) { - return function (Document $database, ?Document $projectDocument = null) use ($cache, $register, $project, $authorization): Database { + $container->set('getDatabasesDB', function (DatabaseFactory $databaseFactory, Document $project) { + return function (Document $database, ?Document $projectDocument = null) use ($databaseFactory, $project): Database { $projectDocument ??= $project; - $databaseDSN = $database->getAttribute('database', $project->getAttribute('database', '')); - $databaseType = $database->getAttribute('type', ''); // Backwards-compatibility: older or seeded legacy databases may not have a DSN stored // in the "database" attribute. In that case, fall back to the project's database DSN. - if ($databaseDSN === '') { - $databaseDSN = $projectDocument->getAttribute('database', ''); - } + $databaseConfig = $database->getAttribute('database', '') === '' + ? new Document(\array_merge($database->getArrayCopy(), ['database' => $projectDocument->getAttribute('database', '')])) + : $database; - try { - $databaseDSN = new DSN($databaseDSN); - } catch (\InvalidArgumentException) { - $databaseDSN = new DSN('mysql://' . $databaseDSN); - } - - try { - $dsn = new DSN($projectDocument->getAttribute('database')); - } catch (\InvalidArgumentException) { - // Temporary fallback until all projects use shared tables - $dsn = new DSN('mysql://' . $projectDocument->getAttribute('database')); - } - - $pools = $register->get('pools'); - $databaseHost = $databaseDSN->getHost(); - $pool = $pools->get($databaseHost); - - $adapter = new DatabasePool($pool); - $database = new Database($adapter, $cache); - $database - ->setDatabase(APP_DATABASE) - ->setAuthorization($authorization); - $database->getAdapter()->setSupportForAttributes($databaseType !== DOCUMENTSDB); - - $sharedTables = \array_filter(\explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''))); - - $projectsGlobalCollections = getProjectGlobalCollections(); - - $database->setGlobalCollections($projectsGlobalCollections); - - // For separate pools (documentsdb/vectorsdb), check their own shared tables config. - // If not configured, use dedicated mode to avoid cross-engine tenant type mismatches. - if ($databaseHost !== $dsn->getHost()) { - $dbTypeSharedTables = match ($databaseType) { - DOCUMENTSDB => \array_filter(\explode(',', System::getEnv('_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES', ''))), - VECTORSDB => \array_filter(\explode(',', System::getEnv('_APP_DATABASE_VECTORSDB_SHARED_TABLES', ''))), - default => [], - }; - - if (\in_array($databaseHost, $dbTypeSharedTables)) { - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($projectDocument->getSequence()) - ->setNamespace($databaseDSN->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $projectDocument->getSequence()); - } - } elseif (\in_array($dsn->getHost(), $sharedTables, true)) { - $database - ->setSharedTables(true) - ->setGlobalCollections($projectsGlobalCollections) - ->setTenant($projectDocument->getSequence()) - ->setNamespace($dsn->getParam('namespace')); - } else { - $database - ->setSharedTables(false) - ->setTenant(null) - ->setNamespace('_' . $projectDocument->getSequence()); - } - - $database->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER); - return $database; + return $databaseFactory->tenant( + $databaseConfig, + $projectDocument, + APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER, + preserveDates: true, + ); }; - }, ['cache', 'register', 'project', 'authorization']); + }, ['databaseFactory', 'project']); - $container->set('getLogsDB', function (Group $pools, Cache $cache, Authorization $authorization) { + $container->set('getLogsDB', function (DatabaseFactory $databaseFactory) { $database = null; - return function (?Document $project = null) use ($pools, $cache, $authorization, &$database) { + return function (?Document $project = null) use ($databaseFactory, &$database) { if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { $database->setTenant($project->getSequence()); return $database; } - $database = createLogsDatabase( - $pools, - $cache, - $authorization, + $database = $databaseFactory->logs( + $project, APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER, APP_DATABASE_QUERY_MAX_VALUES_WORKER ); - if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant($project->getSequence()); - } - return $database; }; - }, ['pools', 'cache', 'authorization']); + }, ['databaseFactory']); $container->set('abuseRetention', function () { return \time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); // 1 day diff --git a/src/Appwrite/Database/DatabaseFactory.php b/src/Appwrite/Database/DatabaseFactory.php new file mode 100644 index 0000000000..f1e36c95f8 --- /dev/null +++ b/src/Appwrite/Database/DatabaseFactory.php @@ -0,0 +1,227 @@ +newDatabase(new DatabasePool($this->pools->get('console'))); + + $database + ->setDatabase($this->database) + ->setAuthorization($this->authorization) + ->setNamespace($this->platformNamespace); + + $this->configureDocumentTypes($database); + $this->configureOptions($database, $timeout, $maxQueryValues, $metadata); + + return $database; + } + + public function project( + Document $project, + int $timeout = 0, + int $maxQueryValues = 0, + array $metadata = [], + ): Database { + $dsn = $this->dsn($project->getAttribute('database')); + $database = $this->newDatabase( + new DatabasePool($this->pools->get($dsn->getHost())), + $this->destinationFor($dsn) + ); + + $database + ->setDatabase($this->database) + ->setAuthorization($this->authorization); + + $this->configureDocumentTypes($database); + $this->configureOptions($database, $timeout, $maxQueryValues, $metadata); + + return $this->configureProject($database, $project, $dsn); + } + + public function logs( + ?Document $project = null, + int $timeout = 0, + int $maxQueryValues = 0, + array $metadata = [], + ): Database { + /** @var array $collections */ + $collections = Config::getParam('collections', []); + $logsCollections = \array_keys($collections['logs'] ?? []); + + $database = $this->newDatabase(new DatabasePool($this->pools->get('logs'))); + + $database + ->setDatabase($this->database) + ->setAuthorization($this->authorization) + ->setSharedTables(true) + ->setGlobalCollections($logsCollections) + ->setNamespace('logsV1'); + + if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant($project->getSequence()); + } + + $this->configureOptions($database, $timeout, $maxQueryValues, $metadata); + + return $database; + } + + public function tenant( + Document $databaseDocument, + Document $project, + int $timeout = 0, + int $maxQueryValues = 0, + array $metadata = [], + bool $preserveDates = false, + ): Database { + $databaseDsn = $this->dsn($databaseDocument->getAttribute('database') ?: $project->getAttribute('database', '')); + $projectDsn = $this->dsn($project->getAttribute('database')); + $databaseType = $databaseDocument->getAttribute('type', ''); + + $database = $this->newDatabase(new DatabasePool($this->pools->get($databaseDsn->getHost()))); + + $database + ->setDatabase($this->database) + ->setAuthorization($this->authorization); + + $database->getAdapter()->setSupportForAttributes($databaseType !== DOCUMENTSDB); + + if ($preserveDates) { + $database->setPreserveDates(true); + } + + $this->configureOptions($database, $timeout, $maxQueryValues, $metadata); + + $sharedTables = \array_filter(\explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''))); + $globalCollections = $this->projectGlobalCollections(); + $databaseHost = $databaseDsn->getHost(); + + if ($databaseHost !== $projectDsn->getHost()) { + $dbTypeSharedTables = match ($databaseType) { + DOCUMENTSDB => \array_filter(\explode(',', System::getEnv('_APP_DATABASE_DOCUMENTSDB_SHARED_TABLES', ''))), + VECTORSDB => \array_filter(\explode(',', System::getEnv('_APP_DATABASE_VECTORSDB_SHARED_TABLES', ''))), + default => [], + }; + + if (\in_array($databaseHost, $dbTypeSharedTables, true)) { + return $database + ->setSharedTables(true) + ->setGlobalCollections($globalCollections) + ->setTenant($project->getSequence()) + ->setNamespace($databaseDsn->getParam('namespace')); + } + + return $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getSequence()); + } + + if (\in_array($projectDsn->getHost(), $sharedTables, true)) { + return $database + ->setSharedTables(true) + ->setGlobalCollections($globalCollections) + ->setTenant($project->getSequence()) + ->setNamespace($projectDsn->getParam('namespace')); + } + + return $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getSequence()); + } + + protected function newDatabase(Adapter $adapter, ?Database $destination = null): Database + { + return new Database($adapter, $this->cache); + } + + protected function configureDocumentTypes(Database $database): Database + { + return $database->setDocumentType('users', User::class); + } + + protected function destinationFor(DSN $dsn): ?Database + { + return null; + } + + protected function configureProject(Database $database, Document $project, DSN $dsn): Database + { + $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); + + if (\in_array($dsn->getHost(), $sharedTables, true)) { + return $database + ->setSharedTables(true) + ->setGlobalCollections($this->projectGlobalCollections()) + ->setTenant($project->getSequence()) + ->setNamespace($dsn->getParam('namespace')); + } + + return $database + ->setSharedTables(false) + ->setTenant(null) + ->setNamespace('_' . $project->getSequence()); + } + + private function configureOptions(Database $database, int $timeout, int $maxQueryValues, array $metadata): void + { + foreach ($metadata as $key => $value) { + $database->setMetadata($key, $value); + } + + if ($timeout > 0) { + $database->setTimeout($timeout); + } + + if ($maxQueryValues > 0) { + $database->setMaxQueryValues($maxQueryValues); + } + } + + private function dsn(?string $value): DSN + { + try { + return new DSN($value ?? ''); + } catch (\InvalidArgumentException) { + return new DSN('mysql://' . $value); + } + } + + private function projectGlobalCollections(): array + { + /** @var array $collections */ + $collections = Config::getParam('collections', []); + $globalCollections = \array_keys($collections['projects'] ?? []); + $globalCollections[] = 'audit'; + + return $globalCollections; + } +} From 329935aa7f7879bf651e277baebf5ea8babed38e Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 19 May 2026 19:23:56 +0530 Subject: [PATCH 3/9] Cache database factory adapters --- src/Appwrite/Database/DatabaseFactory.php | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Database/DatabaseFactory.php b/src/Appwrite/Database/DatabaseFactory.php index f1e36c95f8..afe512ce65 100644 --- a/src/Appwrite/Database/DatabaseFactory.php +++ b/src/Appwrite/Database/DatabaseFactory.php @@ -16,6 +16,11 @@ use Utopia\System\System; class DatabaseFactory { + /** + * @var array + */ + protected array $adapters = []; + public function __construct( protected Group $pools, protected Cache $cache, @@ -30,7 +35,7 @@ class DatabaseFactory int $maxQueryValues = 0, array $metadata = [], ): Database { - $database = $this->newDatabase(new DatabasePool($this->pools->get('console'))); + $database = $this->newDatabase($this->adapter('console')); $database ->setDatabase($this->database) @@ -51,7 +56,7 @@ class DatabaseFactory ): Database { $dsn = $this->dsn($project->getAttribute('database')); $database = $this->newDatabase( - new DatabasePool($this->pools->get($dsn->getHost())), + $this->adapter($dsn->getHost()), $this->destinationFor($dsn) ); @@ -75,7 +80,7 @@ class DatabaseFactory $collections = Config::getParam('collections', []); $logsCollections = \array_keys($collections['logs'] ?? []); - $database = $this->newDatabase(new DatabasePool($this->pools->get('logs'))); + $database = $this->newDatabase($this->adapter('logs')); $database ->setDatabase($this->database) @@ -105,7 +110,7 @@ class DatabaseFactory $projectDsn = $this->dsn($project->getAttribute('database')); $databaseType = $databaseDocument->getAttribute('type', ''); - $database = $this->newDatabase(new DatabasePool($this->pools->get($databaseDsn->getHost()))); + $database = $this->newDatabase($this->adapter($databaseDsn->getHost())); $database ->setDatabase($this->database) @@ -163,6 +168,11 @@ class DatabaseFactory return new Database($adapter, $this->cache); } + protected function adapter(string $name): DatabasePool + { + return $this->adapters[$name] ??= new DatabasePool($this->pools->get($name)); + } + protected function configureDocumentTypes(Database $database): Database { return $database->setDocumentType('users', User::class); From f18af04eaeb18d0071e369d012468686b67f7d73 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 20 May 2026 09:58:35 +0530 Subject: [PATCH 4/9] Fix tenant database adapter reuse --- src/Appwrite/Database/DatabaseFactory.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Database/DatabaseFactory.php b/src/Appwrite/Database/DatabaseFactory.php index afe512ce65..14ce0839ff 100644 --- a/src/Appwrite/Database/DatabaseFactory.php +++ b/src/Appwrite/Database/DatabaseFactory.php @@ -110,12 +110,15 @@ class DatabaseFactory $projectDsn = $this->dsn($project->getAttribute('database')); $databaseType = $databaseDocument->getAttribute('type', ''); - $database = $this->newDatabase($this->adapter($databaseDsn->getHost())); + $database = $this->newDatabase(new DatabasePool($this->pools->get($databaseDsn->getHost()))); $database ->setDatabase($this->database) ->setAuthorization($this->authorization); + // This flag is adapter-level mutable state. Tenant databases can share the same + // pool name while requiring different attribute support (for example TablesDB + // versus DocumentsDB), so this adapter must not be reused from the factory cache. $database->getAdapter()->setSupportForAttributes($databaseType !== DOCUMENTSDB); if ($preserveDates) { From fda5f1e0250091e029f2cda6c45014e22ad8387a Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 20 May 2026 10:32:25 +0530 Subject: [PATCH 5/9] Restore worker database date handling --- app/init/worker/message.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/init/worker/message.php b/app/init/worker/message.php index 6875351def..4826cd3d1e 100644 --- a/app/init/worker/message.php +++ b/app/init/worker/message.php @@ -92,7 +92,6 @@ return function (Container $container): void { $databaseConfig, $projectDocument, APP_DATABASE_TIMEOUT_MILLISECONDS_WORKER, - preserveDates: true, ); }; }, ['databaseFactory', 'project']); From 87cc1ea13316ec657d2e2e16b723263c4dd652c7 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 20 May 2026 10:46:25 +0530 Subject: [PATCH 6/9] Avoid sharing mutable database adapters --- src/Appwrite/Database/DatabaseFactory.php | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/Appwrite/Database/DatabaseFactory.php b/src/Appwrite/Database/DatabaseFactory.php index 14ce0839ff..0fe4af56e6 100644 --- a/src/Appwrite/Database/DatabaseFactory.php +++ b/src/Appwrite/Database/DatabaseFactory.php @@ -16,11 +16,6 @@ use Utopia\System\System; class DatabaseFactory { - /** - * @var array - */ - protected array $adapters = []; - public function __construct( protected Group $pools, protected Cache $cache, @@ -110,15 +105,12 @@ class DatabaseFactory $projectDsn = $this->dsn($project->getAttribute('database')); $databaseType = $databaseDocument->getAttribute('type', ''); - $database = $this->newDatabase(new DatabasePool($this->pools->get($databaseDsn->getHost()))); + $database = $this->newDatabase($this->adapter($databaseDsn->getHost())); $database ->setDatabase($this->database) ->setAuthorization($this->authorization); - // This flag is adapter-level mutable state. Tenant databases can share the same - // pool name while requiring different attribute support (for example TablesDB - // versus DocumentsDB), so this adapter must not be reused from the factory cache. $database->getAdapter()->setSupportForAttributes($databaseType !== DOCUMENTSDB); if ($preserveDates) { @@ -173,7 +165,7 @@ class DatabaseFactory protected function adapter(string $name): DatabasePool { - return $this->adapters[$name] ??= new DatabasePool($this->pools->get($name)); + return new DatabasePool($this->pools->get($name)); } protected function configureDocumentTypes(Database $database): Database From ef7770faab33790a16102f6a993c270726a3de08 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Tue, 26 May 2026 17:14:43 +0530 Subject: [PATCH 7/9] Fix OAuth2 provider list ordering --- .../Modules/Project/Http/Project/OAuth2/XList.php | 9 +++++++-- tests/e2e/Services/Project/OAuth2Base.php | 13 +++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Project/Http/Project/OAuth2/XList.php b/src/Appwrite/Platform/Modules/Project/Http/Project/OAuth2/XList.php index 6a84868286..9a0739e0e6 100644 --- a/src/Appwrite/Platform/Modules/Project/Http/Project/OAuth2/XList.php +++ b/src/Appwrite/Platform/Modules/Project/Http/Project/OAuth2/XList.php @@ -76,8 +76,13 @@ class XList extends Action $actions = Base::getProviderActions(); $documents = []; - foreach ($actions as $providerId => $updateClass) { - if (!($providers[$providerId]['enabled'] ?? false)) { + foreach ($providers as $providerId => $config) { + $updateClass = $actions[$providerId] ?? null; + if ($updateClass === null) { + continue; + } + + if (!($config['enabled'] ?? false)) { // Disabled by Appwrite configuration, exclude from response continue; } diff --git a/tests/e2e/Services/Project/OAuth2Base.php b/tests/e2e/Services/Project/OAuth2Base.php index 5959a584ea..8a317f3e1d 100644 --- a/tests/e2e/Services/Project/OAuth2Base.php +++ b/tests/e2e/Services/Project/OAuth2Base.php @@ -81,6 +81,19 @@ trait OAuth2Base } } + public function testListOAuth2ProvidersSortedAlphabetically(): void + { + $response = $this->listOAuth2Providers(); + + $this->assertSame(200, $response['headers']['status-code']); + + $ids = \array_column($response['body']['providers'], '$id'); + $sorted = $ids; + \sort($sorted); + + $this->assertSame($sorted, $ids); + } + /** * Pin the exact set of registered providers — adding or removing a * provider must be a deliberate change to this assertion. Catches From 4e74c4fcf9a150f9d224b077a88fa359b2814747 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 27 May 2026 09:34:48 +0530 Subject: [PATCH 8/9] Rename database factory class --- app/init/resources.php | 2 +- app/init/resources/request.php | 2 +- app/init/worker/message.php | 2 +- src/Appwrite/Database/{DatabaseFactory.php => Factory.php} | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename src/Appwrite/Database/{DatabaseFactory.php => Factory.php} (99%) diff --git a/app/init/resources.php b/app/init/resources.php index b388eed0c6..34f66d3de7 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -1,6 +1,6 @@ Date: Wed, 27 May 2026 11:30:04 +0530 Subject: [PATCH 9/9] Fix Swagger enum keys metadata --- .../SDK/Specification/Format/Swagger2.php | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/Appwrite/SDK/Specification/Format/Swagger2.php b/src/Appwrite/SDK/Specification/Format/Swagger2.php index 1232ea5408..6d456e1abc 100644 --- a/src/Appwrite/SDK/Specification/Format/Swagger2.php +++ b/src/Appwrite/SDK/Specification/Format/Swagger2.php @@ -611,19 +611,15 @@ class Swagger2 extends Format } $enumKeys = []; - if (!empty($enum->map)) { - foreach ($enumValues as $enumValue) { - $enumKeys[] = $enum->map[$enumValue] ?? $enumValue; - } + foreach ($enumValues as $enumValue) { + $enumKeys[] = $enum->map[$enumValue] ?? $enumValue; } $node['items']['enum'] = $enumValues; if (!empty($enum->name)) { $node['items']['x-enum-name'] = $enum->name; } - if (!empty($enumKeys)) { - $node['items']['x-enum-keys'] = $enumKeys; - } + $node['items']['x-enum-keys'] = $enumKeys; } } if ($validator->getType() === 'integer') { @@ -652,19 +648,15 @@ class Swagger2 extends Format } $enumKeys = []; - if (!empty($enum->map)) { - foreach ($enumValues as $enumValue) { - $enumKeys[] = $enum->map[$enumValue] ?? $enumValue; - } + foreach ($enumValues as $enumValue) { + $enumKeys[] = $enum->map[$enumValue] ?? $enumValue; } $node['enum'] = $enumValues; if (!empty($enum->name)) { $node['x-enum-name'] = $enum->name; } - if (!empty($enumKeys)) { - $node['x-enum-keys'] = $enumKeys; - } + $node['x-enum-keys'] = $enumKeys; } } if ($validator->getType() === 'integer') {