diff --git a/.env b/.env index bc2b32cde8..c3785cb49a 100644 --- a/.env +++ b/.env @@ -17,13 +17,18 @@ _APP_REDIS_HOST=redis _APP_REDIS_PORT=6379 _APP_REDIS_PASS= _APP_REDIS_USER= -_APP_DB_HOST=mariadb +_APP_DB_HOST=mysql _APP_DB_PORT=3306 _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password _APP_DB_ROOT_PASS=rootsecretpassword _APP_DB_MAX_CONNECTIONS=1001 +_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mysql://user:password@mysql:3306/appwrite +_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mysql://user:password@mysql:3306/appwrite +_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379 +_APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379 +_APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379 _APP_STORAGE_DEVICE=Local _APP_STORAGE_S3_ACCESS_KEY= _APP_STORAGE_S3_SECRET= diff --git a/Dockerfile b/Dockerfile index d284e2a90d..3910bc61a1 100755 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,7 @@ ENV PHP_REDIS_VERSION=5.3.7 \ PHP_IMAGICK_VERSION=3.7.0 \ PHP_YAML_VERSION=2.2.2 \ PHP_MAXMINDDB_VERSION=v1.11.0 \ + PHP_MEMCACHED_VERSION=v3.2.0 \ PHP_ZSTD_VERSION="4504e4186e79b197cfcb75d4d09aa47ef7d92fe9 " RUN \ @@ -52,6 +53,7 @@ RUN \ imagemagick \ imagemagick-dev \ libmaxminddb-dev \ + libmemcached-dev \ zstd-dev RUN docker-php-ext-install sockets @@ -125,6 +127,15 @@ RUN \ ./configure && \ make && make install +# Memcached Extension +FROM compile as memcached +RUN \ + git clone --depth 1 --branch $PHP_MEMCACHED_VERSION https://github.com/php-memcached-dev/php-memcached.git && \ + cd php-memcached && \ + phpize && \ + ./configure && \ + make && make install + # Zstd Compression FROM compile as zstd RUN git clone --recursive -n https://github.com/kjdev/php-ext-zstd.git \ @@ -134,7 +145,6 @@ RUN git clone --recursive -n https://github.com/kjdev/php-ext-zstd.git \ && ./configure --with-libzstd \ && make && make install - # Rust Extensions Compile Image FROM php:8.0.18-cli as rust_compile @@ -305,6 +315,7 @@ COPY --from=imagick /usr/local/lib/php/extensions/no-debug-non-zts-20200930/imag COPY --from=yaml /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yaml.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=maxmind /usr/local/lib/php/extensions/no-debug-non-zts-20200930/maxminddb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=mongodb /usr/local/lib/php/extensions/no-debug-non-zts-20200930/mongodb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ +COPY --from=memcached /usr/local/lib/php/extensions/no-debug-non-zts-20200930/memcached.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=scrypt /usr/local/lib/php/extensions/php-scrypt/target/libphp_scrypt.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ COPY --from=zstd /usr/local/lib/php/extensions/no-debug-non-zts-20200930/zstd.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/ diff --git a/app/cli.php b/app/cli.php index 55704e47eb..1b51282348 100644 --- a/app/cli.php +++ b/app/cli.php @@ -6,7 +6,78 @@ require_once __DIR__ . '/controllers/general.php'; use Utopia\App; use Utopia\CLI\CLI; use Utopia\CLI\Console; +use Utopia\Cache\Adapter\Sharding; +use Utopia\Cache\Cache; +use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\Database\Validator\Authorization; +use InfluxDB\Database as InfluxDatabase; + +function getInfluxDB(): InfluxDatabase +{ + global $register; + + $client = $register->get('influxdb'); /** @var InfluxDB\Client $client */ + $attempts = 0; + $max = 10; + $sleep = 1; + + do { // check if telegraf database is ready + try { + $attempts++; + $database = $client->selectDB('telegraf'); + if (in_array('telegraf', $client->listDatabases())) { + break; // leave the do-while if successful + } + } catch (\Throwable $th) { + Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('InfluxDB database not ready yet'); + } + sleep($sleep); + } + } while ($attempts < $max); + return $database; +} + +function getConsoleDB(): Database +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, getCache()); + + $database->setNamespace('console'); + + return $database; +} + +function getCache(): Cache +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +} Authorization::disable(); @@ -30,4 +101,13 @@ $cli Console::log(App::getEnv('_APP_VERSION', 'UNKNOWN')); }); +$cli + ->error(function ($error) { + if (App::getEnv('_APP_ENV', 'development')) { + Console::error($error); + } else { + Console::error($error->getMessage()); + } + }); + $cli->run(); diff --git a/app/config/collections.php b/app/config/collections.php index 6fc7ec1e37..b4fabf0e65 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -534,6 +534,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => ID::custom('database'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], [ '$id' => ID::custom('logo'), 'type' => Database::VAR_STRING, @@ -992,7 +1003,7 @@ $collections = [ '$id' => ID::custom('secret'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 512, // var_dump of \bin2hex(\random_bytes(128)) => string(256) doubling for encryption + 'size' => 512, // Output of \bin2hex(\random_bytes(128)) => string(256) doubling for encryption 'signed' => true, 'required' => true, 'default' => null, diff --git a/app/config/variables.php b/app/config/variables.php index ba6aa9a14a..7cacd39ca4 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -315,6 +315,24 @@ return [ 'question' => '', 'filter' => '' ], + // [ + // 'name' => '_APP_CONNECTIONS_DB_PROJECT', + // 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.', + // 'introduction' => 'TBD', + // 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + // 'required' => true, + // 'question' => '', + // 'filter' => '' + // ], + // [ + // 'name' => '_APP_CONNECTIONS_DB_CONSOLE', + // 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.', + // 'introduction' => 'TBD', + // 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite', + // 'required' => true, + // 'question' => '', + // 'filter' => '' + // ] ], ], [ diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index ace987d726..c3056db574 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -1574,7 +1574,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes') Query::equal('databaseInternalId', [$db->getInternalId()]) ], 61); - $limit = 64 - MariaDB::getCountOfDefaultIndexes(); + $limit = $dbForProject->getLimitForIndexes(); if ($count >= $limit) { throw new Exception(Exception::INDEX_LIMIT_EXCEEDED, 'Index limit exceeded'); diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 55997ce029..188c5e453c 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -25,7 +25,6 @@ use Appwrite\Task\Validator\Cron; use Appwrite\Utopia\Database\Validator\Queries\Deployments; use Appwrite\Utopia\Database\Validator\Queries\Executions; use Appwrite\Utopia\Database\Validator\Queries\Functions; -use Appwrite\Utopia\Database\Validator\Queries\Variables; use Utopia\App; use Utopia\Database\Database; use Utopia\Database\Document; @@ -33,7 +32,6 @@ use Utopia\Database\DateTime; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Validator\ArrayList; -use Utopia\Validator\Assoc; use Utopia\Validator\Text; use Utopia\Validator\Range; use Utopia\Validator\WhiteList; diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 4384fcbcd5..f65e65ba23 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -5,7 +5,9 @@ use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\Utopia\Response; use Utopia\App; +use Utopia\Config\Config; use Utopia\Database\Document; +use Utopia\Pools\Group; use Utopia\Registry\Registry; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; @@ -26,6 +28,7 @@ App::get('/v1/health') ->action(function (Response $response) { $output = [ + 'name' => 'http', 'status' => 'pass', 'ping' => 0 ]; @@ -42,7 +45,6 @@ App::get('/v1/health/version') ->label('sdk.response.model', Response::MODEL_HEALTH_VERSION) ->inject('response') ->action(function (Response $response) { - $response->dynamic(new Document([ 'version' => APP_VERSION_STABLE ]), Response::MODEL_HEALTH_VERSION); }); @@ -58,30 +60,50 @@ App::get('/v1/health/db') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) ->inject('response') - ->inject('utopia') - ->action(function (Response $response, App $utopia) { + ->inject('pools') + ->action(function (Response $response, Group $pools) { - $checkStart = \microtime(true); + $output = []; - try { - $db = $utopia->getResource('db'); /* @var $db PDO */ - - // Run a small test to check the connection - $statement = $db->prepare("SELECT 1;"); - - $statement->closeCursor(); - - $statement->execute(); - } catch (Exception $_e) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Database is not available'); - } - - $output = [ - 'status' => 'pass', - 'ping' => \round((\microtime(true) - $checkStart) / 1000) + $configs = [ + 'Console.DB' => Config::getParam('pools-console'), + 'Projects.DB' => Config::getParam('pools-database'), ]; - $response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS); + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); }); App::get('/v1/health/cache') @@ -96,23 +118,163 @@ App::get('/v1/health/cache') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) ->inject('response') - ->inject('utopia') - ->action(function (Response $response, App $utopia) { + ->inject('pools') + ->action(function (Response $response, Group $pools) { - $checkStart = \microtime(true); + $output = []; - $redis = $utopia->getResource('cache'); - - if (!$redis->ping(true)) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Cache is not available'); - } - - $output = [ - 'status' => 'pass', - 'ping' => \round((\microtime(true) - $checkStart) / 1000) + $configs = [ + 'Cache' => Config::getParam('pools-cache'), ]; - $response->dynamic(new Document($output), Response::MODEL_HEALTH_STATUS); + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); + }); + +App::get('/v1/health/queue') + ->desc('Get Queue') + ->groups(['api', 'health']) + ->label('scope', 'health.read') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'health') + ->label('sdk.method', 'getQueue') + ->label('sdk.description', '/docs/references/health/get-queue.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) + ->inject('response') + ->inject('pools') + ->action(function (Response $response, Group $pools) { + + $output = []; + + $configs = [ + 'Queue' => Config::getParam('pools-queue'), + ]; + + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); + }); + +App::get('/v1/health/pubsub') + ->desc('Get PubSub') + ->groups(['api', 'health']) + ->label('scope', 'health.read') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'health') + ->label('sdk.method', 'getPubSub') + ->label('sdk.description', '/docs/references/health/get-pubsub.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_HEALTH_STATUS) + ->inject('response') + ->inject('pools') + ->action(function (Response $response, Group $pools) { + + $output = []; + + $configs = [ + 'PubSub' => Config::getParam('pools-pubsub'), + ]; + + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + $checkStart = \microtime(true); + + if ($adapter->ping()) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'pass', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } else { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } catch (\Throwable $th) { + $output[] = new Document([ + 'name' => $key . " ($database)", + 'status' => 'fail', + 'ping' => \round((\microtime(true) - $checkStart) / 1000) + ]); + } + } + } + + $response->dynamic(new Document([ + 'statuses' => $output, + 'total' => count($output), + ]), Response::MODEL_HEALTH_STATUS_LIST); }); App::get('/v1/health/time') diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index b0f0a7383d..3935a71a9d 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -29,6 +29,8 @@ use Utopia\Domains\Domain; use Utopia\Registry\Registry; use Appwrite\Extend\Exception; use Appwrite\Utopia\Database\Validator\Queries\Projects; +use Utopia\Cache\Cache; +use Utopia\Pools\Group; use Utopia\Validator\ArrayList; use Utopia\Validator\Boolean; use Utopia\Validator\Hostname; @@ -69,8 +71,10 @@ App::post('/v1/projects') ->param('legalTaxId', '', new Text(256), 'Project legal Tax ID. Max length: 256 chars.', true) ->inject('response') ->inject('dbForConsole') - ->inject('dbForProject') - ->action(function (string $projectId, string $name, string $teamId, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Database $dbForProject) { + ->inject('cache') + ->inject('pools') + ->action(function (string $projectId, string $name, string $teamId, string $description, string $logo, string $url, string $legalName, string $legalCountry, string $legalState, string $legalCity, string $legalAddress, string $legalTaxId, Response $response, Database $dbForConsole, Cache $cache, Group $pools) { + $team = $dbForConsole->getDocument('teams', $teamId); @@ -85,6 +89,8 @@ App::post('/v1/projects') } $projectId = ($projectId == 'unique()') ? ID::unique() : $projectId; + $databases = Config::getParam('pools-database', []); + $database = $databases[array_rand($databases)]; if ($projectId === 'console') { throw new Exception(Exception::PROJECT_RESERVED_PROJECT, "'console' is a reserved project."); @@ -120,10 +126,10 @@ App::post('/v1/projects') 'domains' => null, 'auths' => $auths, 'search' => implode(' ', [$projectId, $name]), + 'database' => $database, ])); - /** @var array $collections */ - $collections = Config::getParam('collections', []); + $dbForProject = new Database($pools->get($database)->pop()->getResource(), $cache); $dbForProject->setNamespace("_{$project->getInternalId()}"); $dbForProject->create(); @@ -133,6 +139,9 @@ App::post('/v1/projects') $adapter = new TimeLimit('', 0, 1, $dbForProject); $adapter->setup(); + /** @var array $collections */ + $collections = Config::getParam('collections', []); + foreach ($collections as $key => $collection) { if (($collection['$collection'] ?? '') !== Database::METADATA) { continue; diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 60536f3151..b87017dbd2 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -20,7 +20,6 @@ use Appwrite\Utopia\Response; use MaxMind\Db\Reader; use Utopia\App; use Utopia\Audit\Audit; -use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Document; @@ -36,9 +35,7 @@ use Utopia\Database\Validator\Key; use Utopia\Database\Validator\UID; use Utopia\Locale\Locale; use Utopia\Validator\Text; -use Utopia\Validator\Range; use Utopia\Validator\ArrayList; -use Utopia\Validator\WhiteList; App::post('/v1/teams') ->desc('Create Team') diff --git a/app/http.php b/app/http.php index f27bf82353..b497c466f3 100644 --- a/app/http.php +++ b/app/http.php @@ -22,6 +22,7 @@ use Utopia\Swoole\Files; use Appwrite\Utopia\Request; use Utopia\Logger\Log; use Utopia\Logger\Log\User; +use Utopia\Pools\Group; $http = new Server("0.0.0.0", App::getEnv('PORT', 80)); @@ -60,6 +61,9 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $app = new App('UTC'); go(function () use ($register, $app) { + $pools = $register->get('pools'); /** @var Group $pools */ + App::setResource('pools', fn() => $pools); + // wait for database to be ready $attempts = 0; $max = 10; @@ -68,8 +72,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { do { try { $attempts++; - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); + $dbForConsole = $app->getResource('dbForConsole'); /** @var Utopia\Database\Database $dbForConsole */ break; // leave the do-while if successful } catch (\Exception $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); @@ -80,19 +83,14 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { } } while ($attempts < $max); - App::setResource('db', fn () => $db); - App::setResource('cache', fn () => $redis); - - /** @var Utopia\Database\Database $dbForConsole */ - $dbForConsole = $app->getResource('dbForConsole'); - Console::success('[Setup] - Server database init started...'); /** @var array $collections */ $collections = Config::getParam('collections', []); try { - $redis->flushAll(); + $cache = $app->getResource('cache'); /** @var Utopia\Cache\Cache $cache */ + $cache->flush(); Console::success('[Setup] - Creating database: appwrite...'); $dbForConsole->create(); } catch (\Exception $e) { @@ -116,10 +114,11 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { if (!$dbForConsole->getCollection($key)->isEmpty()) { continue; } + /** * Skip to prevent 0.16 migration issues. */ - if (in_array($key, ['cache', 'variables']) && $dbForConsole->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), 'bucket_1')) { + if (in_array($key, ['cache', 'variables']) && $dbForConsole->exists($dbForConsole->getDefaultDatabase(), 'bucket_1')) { continue; } @@ -155,7 +154,7 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $dbForConsole->createCollection($key, $attributes, $indexes); } - if ($dbForConsole->getDocument('buckets', 'default')->isEmpty() && !$dbForConsole->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), 'bucket_1')) { + if ($dbForConsole->getDocument('buckets', 'default')->isEmpty() && !$dbForConsole->exists($dbForConsole->getDefaultDatabase(), 'bucket_1')) { Console::success('[Setup] - Creating default bucket...'); $dbForConsole->createDocument('buckets', new Document([ '$id' => ID::custom('default'), @@ -215,6 +214,8 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) { $dbForConsole->createCollection('bucket_' . $bucket->getInternalId(), $attributes, $indexes); } + $pools->reclaim(); + Console::success('[Setup] - Server database init completed...'); }); @@ -246,11 +247,8 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $app = new App('UTC'); - $db = $register->get('dbPool')->get(); - $redis = $register->get('redisPool')->get(); - - App::setResource('db', fn () => $db); - App::setResource('cache', fn () => $redis); + $pools = $register->get('pools'); + App::setResource('pools', fn() => $pools); try { Authorization::cleanRoles(); @@ -334,13 +332,7 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $swooleResponse->end(\json_encode($output)); } finally { - /** @var PDOPool $dbPool */ - $dbPool = $register->get('dbPool'); - $dbPool->put($db); - - /** @var RedisPool $redisPool */ - $redisPool = $register->get('redisPool'); - $redisPool->put($redis); + $pools->reclaim(); } }); diff --git a/app/init.php b/app/init.php index 70997ebbf9..6d7028e884 100644 --- a/app/init.php +++ b/app/init.php @@ -18,9 +18,6 @@ ini_set('display_startup_errors', 1); ini_set('default_socket_timeout', -1); error_reporting(E_ALL); -use Appwrite\Extend\PDO; -use Ahc\Jwt\JWT; -use Ahc\Jwt\JWTException; use Appwrite\Extend\Exception; use Appwrite\Auth\Auth; use Appwrite\SMS\Adapter\Mock; @@ -32,39 +29,31 @@ use Appwrite\SMS\Adapter\Vonage; use Appwrite\DSN\DSN; use Appwrite\Event\Audit; use Appwrite\Event\Database as EventDatabase; -use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Phone; +use Appwrite\Event\Delete; use Appwrite\Network\Validator\Email; use Appwrite\Network\Validator\IP; use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; +use Appwrite\URL\URL as AppwriteURL; use Appwrite\Usage\Stats; use Appwrite\Utopia\View; use Utopia\App; +use Utopia\Validator\Range; +use Utopia\Validator\WhiteList; use Utopia\Database\ID; +use Utopia\Database\Document; +use Utopia\Database\Database; +use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\DatetimeValidator; +use Utopia\Database\Validator\Structure; use Utopia\Logger\Logger; use Utopia\Config\Config; use Utopia\Locale\Locale; use Utopia\Registry\Registry; -use MaxMind\Db\Reader; -use PHPMailer\PHPMailer\PHPMailer; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Cache\Cache; -use Utopia\Database\Adapter\MySQL; -use Utopia\Database\Document; -use Utopia\Database\Database; -use Utopia\Database\Validator\Structure; -use Utopia\Database\Validator\Authorization; -use Utopia\Validator\Range; -use Utopia\Validator\WhiteList; -use Swoole\Database\PDOConfig; -use Swoole\Database\PDOPool; -use Swoole\Database\RedisConfig; -use Swoole\Database\RedisPool; -use Utopia\Database\Query; -use Utopia\Database\Validator\DatetimeValidator; use Utopia\Storage\Device; use Utopia\Storage\Storage; use Utopia\Storage\Device\Backblaze; @@ -73,6 +62,18 @@ use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; use Utopia\Storage\Device\Linode; use Utopia\Storage\Device\Wasabi; +use Utopia\Cache\Adapter\Redis as RedisCache; +use Utopia\Cache\Adapter\Sharding; +use Utopia\Cache\Cache; +use Utopia\Database\Adapter\MariaDB; +use Utopia\Database\Adapter\MySQL; +use Utopia\Pools\Group; +use Utopia\Pools\Pool; +use Ahc\Jwt\JWT; +use Ahc\Jwt\JWTException; +use MaxMind\Db\Reader; +use PHPMailer\PHPMailer\PHPMailer; +use Swoole\Database\PDOProxy; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; @@ -496,56 +497,173 @@ $register->set('logger', function () { $adapter = new $classname($providerConfig); return new Logger($adapter); }); -$register->set('dbPool', function ($size = APP_DATABASE_DEFAULT_POOL_SIZE) { - // Register DB connection - $dbHost = App::getEnv('_APP_DB_HOST', ''); - $dbPort = App::getEnv('_APP_DB_PORT', ''); - $dbUser = App::getEnv('_APP_DB_USER', ''); - $dbPass = App::getEnv('_APP_DB_PASS', ''); - $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); +$register->set('pools', function ($size = APP_DATABASE_DEFAULT_POOL_SIZE) { + $group = new Group(); - $pool = new PDOPool( - (new PDOConfig()) - ->withHost($dbHost) - ->withPort($dbPort) - ->withDbName($dbScheme) - ->withCharset('utf8mb4') - ->withUsername($dbUser) - ->withPassword($dbPass) - ->withOptions([ - PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true, - ]), - $size - ); + $fallbackForDB = AppwriteURL::unparse([ + 'scheme' => 'mariadb', + 'host' => App::getEnv('_APP_DB_HOST', 'mariadb'), + 'port' => App::getEnv('_APP_DB_PORT', '3306'), + 'user' => App::getEnv('_APP_DB_USER', ''), + 'pass' => App::getEnv('_APP_DB_PASS', ''), + ]); + $fallbackForRedis = AppwriteURL::unparse([ + 'scheme' => 'redis', + 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), + 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), + 'user' => App::getEnv('_APP_REDIS_USER', ''), + 'pass' => App::getEnv('_APP_REDIS_PASS', ''), + ]); - return $pool; -}); -$register->set('redisPool', function ($size = APP_DATABASE_DEFAULT_POOL_SIZE) { - $redisHost = App::getEnv('_APP_REDIS_HOST', ''); - $redisPort = App::getEnv('_APP_REDIS_PORT', ''); - $redisUser = App::getEnv('_APP_REDIS_USER', ''); - $redisPass = App::getEnv('_APP_REDIS_PASS', ''); - $redisAuth = ''; + $connections = [ + 'console' => [ + 'type' => 'database', + 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), + 'multiple' => false, + 'schemes' => ['mariadb', 'mysql'], + ], + 'database' => [ + 'type' => 'database', + 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), + 'multiple' => true, + 'schemes' => ['mariadb', 'mysql'], + ], + 'queue' => [ + 'type' => 'queue', + 'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'pubsub' => [ + 'type' => 'pubsub', + 'dsns' => App::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'cache' => [ + 'type' => 'cache', + 'dsns' => App::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), + 'multiple' => true, + 'schemes' => ['redis'], + ], + ]; - if ($redisUser && $redisPass) { - $redisAuth = $redisUser . ':' . $redisPass; + foreach ($connections as $key => $connection) { + $type = $connection['type'] ?? ''; + $dsns = $connection['dsns'] ?? ''; + $multipe = $connection['multiple'] ?? false; + $schemes = $connection['schemes'] ?? []; + $config = []; + $dsns = explode(',', $connection['dsns'] ?? ''); + + foreach ($dsns as &$dsn) { + $dsn = explode('=', $dsn); + $name = ($multipe) ? $key . '_' . $dsn[0] : $key; + $dsn = $dsn[1] ?? ''; + $config[] = $name; + + if (empty($dsn)) { + //throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}"); + continue; + } + + $dsn = new DSN($dsn); + $dsnHost = $dsn->getHost(); + $dsnPort = $dsn->getPort(); + $dsnUser = $dsn->getUser(); + $dsnPass = $dsn->getPassword(); + $dsnScheme = $dsn->getScheme(); + $dsnDatabase = $dsn->getDatabase(); + + if (!in_array($dsnScheme, $schemes)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); + } + + /** + * Get Resource + * + * Creation could be reused accross connection types like database, cache, queue, etc. + * + * Resource assignment to an adapter will happen below. + */ + + switch ($dsnScheme) { + case 'mysql': + case 'mariadb': + $resource = function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + )); + }); + }; + break; + case 'redis': + $resource = function () use ($dsnHost, $dsnPort, $dsnPass) { + $redis = new Redis(); + @$redis->pconnect($dsnHost, (int)$dsnPort); + if ($dsnPass) { + $redis->auth($dsnPass); + } + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + + return $redis; + }; + break; + + default: + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid scheme"); + break; + } + + $pool = new Pool($name, $size, function () use ($type, $resource, $dsn) { + // Get Adapter + $adapter = null; + + switch ($type) { + case 'database': + $adapter = match ($dsn->getScheme()) { + 'mariadb' => new MariaDB($resource()), + 'mysql' => new MySQL($resource()), + default => null + }; + + $adapter->setDefaultDatabase($dsn->getDatabase()); + + break; + case 'queue': + $adapter = $resource(); + break; + case 'pubsub': + $adapter = $resource(); + break; + case 'cache': + $adapter = match ($dsn->getScheme()) { + 'redis' => new RedisCache($resource()), + default => null + }; + break; + + default: + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation."); + break; + } + + return $adapter; + }); + + $group->add($pool); + } + + Config::setParam('pools-' . $key, $config); } - $pool = new RedisPool( - (new RedisConfig()) - ->withHost($redisHost) - ->withPort($redisPort) - ->withAuth($redisAuth) - ->withDbIndex(0), - $size - ); - - return $pool; + return $group; }); $register->set('influxdb', function () { // Register DB connection @@ -562,7 +680,7 @@ $register->set('influxdb', function () { return $client; }); $register->set('statsd', function () { - // Register DB connection + // Register DB connection $host = App::getEnv('_APP_STATSD_HOST', 'telegraf'); $port = App::getEnv('_APP_STATSD_PORT', 8125); @@ -602,33 +720,6 @@ $register->set('smtp', function () { $register->set('geodb', function () { return new Reader(__DIR__ . '/db/DBIP/dbip-country-lite-2022-06.mmdb'); }); -$register->set('db', function () { - // This is usually for our workers or CLI commands scope - $dbHost = App::getEnv('_APP_DB_HOST', ''); - $dbPort = App::getEnv('_APP_DB_PORT', ''); - $dbUser = App::getEnv('_APP_DB_USER', ''); - $dbPass = App::getEnv('_APP_DB_PASS', ''); - $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); - - $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true, - )); - - return $pdo; -}); -$register->set('cache', function () { - // This is usually for our workers or CLI commands scope - $redis = new Redis(); - $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); - - return $redis; -}); /* * Localization @@ -926,26 +1017,51 @@ App::setResource('console', function () { ]); }, []); -App::setResource('dbForProject', function ($db, $cache, Document $project) { - $cache = new Cache(new RedisCache($cache)); +App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForConsole; + } - $database = new Database(new MySQL($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace("_{$project->getInternalId()}"); + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, $cache); + $database->setNamespace('_' . $project->getInternalId()); return $database; -}, ['db', 'cache', 'project']); +}, ['pools', 'dbForConsole', 'cache', 'project']); -App::setResource('dbForConsole', function ($db, $cache) { - $cache = new Cache(new RedisCache($cache)); +App::setResource('dbForConsole', function (Group $pools, Cache $cache) { + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; - $database = new Database(new MySQL($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace('_console'); + $database = new Database($dbAdapter, $cache); + + $database->setNamespace('console'); return $database; -}, ['db', 'cache']); +}, ['pools', 'cache']); +App::setResource('cache', function (Group $pools) { + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +}, ['pools']); App::setResource('deviceLocal', function () { return new Local(); diff --git a/app/preload.php b/app/preload.php index 8789936417..4935db3da4 100644 --- a/app/preload.php +++ b/app/preload.php @@ -35,7 +35,7 @@ foreach ( realpath(__DIR__ . '/../vendor/symfony'), realpath(__DIR__ . '/../vendor/mongodb'), realpath(__DIR__ . '/../vendor/utopia-php/websocket'), // TODO: remove workerman autoload - realpath(__DIR__ . '/../vendor/utopia-php/cache'), // TODO: remove memcache autoload + realpath(__DIR__ . '/../vendor/utopia-php/cache'), // TODO: remove memcached autoload ] as $key => $value ) { if ($value !== false) { diff --git a/app/realtime.php b/app/realtime.php index 854b5e3b7f..4771669476 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -17,16 +17,15 @@ use Utopia\CLI\Console; use Utopia\Database\ID; use Utopia\Database\Role; use Utopia\Logger\Log; -use Utopia\Database\Database; use Utopia\Database\DateTime; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Cache\Cache; -use Utopia\Database\Adapter\MySQL; use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; -use Utopia\Registry\Registry; use Appwrite\Utopia\Request; +use Utopia\Cache\Adapter\Sharding; +use Utopia\Cache\Cache; +use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\WebSocket\Server; use Utopia\WebSocket\Adapter; @@ -34,6 +33,67 @@ require_once __DIR__ . '/init.php'; Runtime::enableCoroutine(SWOOLE_HOOK_ALL); +function getConsoleDB(): Database +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, getCache()); + + $database->setNamespace('console'); + + return $database; +} + +function getProjectDB(Document $project): Database +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + if ($project->isEmpty() || $project->getId() === 'console') { + return getConsoleDB(); + } + + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, getCache()); + $database->setNamespace('_' . $project->getInternalId()); + + return $database; +} + +function getCache(): Cache +{ + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Cache(new Sharding($adapters)); +} + $realtime = new Realtime(); $dbPool = $register->get('dbPool', args: [getWorkerPoolSize()]); @@ -99,46 +159,7 @@ $logError = function (Throwable $error, string $action) use ($register) { $server->error($logError); -function getDatabase(ConnectionPool $dbPool, ConnectionPool $redisPool, string $namespace) -{ - $attempts = 0; - - do { - try { - $attempts++; - - $db = $dbPool->get(); - $redis = $redisPool->get(); - - $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MySQL($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace($namespace); - - if (!$database->exists($database->getDefaultDatabase(), 'realtime')) { - throw new Exception('Collection not ready'); - } - - break; // leave loop if successful - } catch (\Throwable $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep(DATABASE_RECONNECT_SLEEP); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return [ - $database, - function () use ($dbPool, $redisPool, $db, $redis) { - $dbPool->put($db); - $redisPool->put($redis); - } - ]; -} - -$server->onStart(function () use ($stats, $dbPool, $redisPool, $containerId, &$statsDocument, $logError) { +$server->onStart(function () use ($stats, $register, $containerId, &$statsDocument, $logError) { sleep(5); // wait for the initial database schema to be ready Console::success('Server started successfully'); @@ -147,7 +168,8 @@ $server->onStart(function () use ($stats, $dbPool, $redisPool, $containerId, &$s */ go(function () use ($dbPool, $redisPool, $containerId, &$statsDocument) { $attempts = 0; - [$database, $returnDatabase] = getDatabase($dbPool, $redisPool, '_console'); + $database = getConsoleDB(); + do { try { $attempts++; @@ -167,7 +189,7 @@ $server->onStart(function () use ($stats, $dbPool, $redisPool, $containerId, &$s sleep(DATABASE_RECONNECT_SLEEP); } } while (true); - call_user_func($returnDatabase); + $register->get('pools')->reclaim(); }); /** @@ -183,7 +205,7 @@ $server->onStart(function () use ($stats, $dbPool, $redisPool, $containerId, &$s } try { - [$database, $returnDatabase] = getDatabase($dbPool, $redisPool, '_console'); + $database = getConsoleDB(); $statsDocument ->setAttribute('timestamp', DateTime::now()) @@ -193,7 +215,7 @@ $server->onStart(function () use ($stats, $dbPool, $redisPool, $containerId, &$s } catch (\Throwable $th) { call_user_func($logError, $th, "updateWorkerDocument"); } finally { - call_user_func($returnDatabase); + $register->get('pools')->reclaim(); } }); }); @@ -209,7 +231,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $dbPool, $redisPoo * Sending current connections to project channels on the console project every 5 seconds. */ if ($realtime->hasSubscriber('console', Role::users()->toString(), 'project')) { - [$database, $returnDatabase] = getDatabase($dbPool, $redisPool, '_console'); + $database = getConsoleDB(); $payload = []; @@ -254,7 +276,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $dbPool, $redisPoo ])); } - call_user_func($returnDatabase); + $register->get('pools')->reclaim(); } /** * Sending test message for SDK E2E tests every 5 seconds. @@ -289,8 +311,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $dbPool, $redisPoo } $start = time(); - /** @var Redis $redis */ - $redis = $redisPool->get(); + $redis = $register->get('pools')->get('pubsub')->pop()->getResource(); /** @var Redis $redis */ $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); if ($redis->ping(true)) { @@ -309,9 +330,9 @@ $server->onWorkerStart(function (int $workerId) use ($server, $dbPool, $redisPoo if ($realtime->hasSubscriber($projectId, 'user:' . $userId)) { $connection = array_key_first(reset($realtime->subscriptions[$projectId]['user:' . $userId])); - [$consoleDatabase, $returnConsoleDatabase] = getDatabase($dbPool, $redisPool, '_console'); + $consoleDatabase = getConsoleDB(); $project = Authorization::skip(fn() => $consoleDatabase->getDocument('projects', $projectId)); - [$database, $returnDatabase] = getDatabase($dbPool, $redisPool, "_{$project->getInternalId()}"); + $database = getProjectDB($project); $user = $database->getDocument('users', $userId); @@ -319,8 +340,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $dbPool, $redisPoo $realtime->subscribe($projectId, $connection, $roles, $realtime->connections[$connection]['channels']); - call_user_func($returnDatabase); - call_user_func($returnConsoleDatabase); + $register->get('pools')->reclaim(); } } @@ -348,6 +368,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $dbPool, $redisPoo call_user_func($logError, $th, "pubSubConnection"); Console::error('Pub/sub error: ' . $th->getMessage()); + $register->get('pools')->reclaim(); $attempts++; sleep(DATABASE_RECONNECT_SLEEP); continue; @@ -364,33 +385,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $request = new Request($request); $response = new Response(new SwooleResponse()); - /** @var PDO $db */ - $db = $dbPool->get(); - /** @var Redis $redis */ - $redis = $redisPool->get(); - Console::info("Connection open (user: {$connection})"); - App::setResource('db', fn () => $db); - App::setResource('cache', fn () => $redis); - App::setResource('request', fn () => $request); - App::setResource('response', fn () => $response); + App::setResource('pools', fn() => $register->get('pools')); + App::setResource('request', fn() => $request); + App::setResource('response', fn() => $response); try { - /** @var \Utopia\Database\Document $user */ - $user = $app->getResource('user'); - /** @var \Utopia\Database\Document $project */ $project = $app->getResource('project'); - /** @var \Utopia\Database\Document $console */ - $console = $app->getResource('console'); - - $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MySQL($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace("_{$project->getInternalId()}"); - /* * Project Check */ @@ -398,12 +402,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new Exception('Missing or unknown project ID', 1008); } + $dbForProject = getProjectDB($project); + $console = $app->getResource('console'); /** @var \Utopia\Database\Document $console */ + $user = $app->getResource('user'); /** @var \Utopia\Database\Document $user */ + /* * Abuse Check * * Abuse limits are connecting 128 times per minute and ip address. */ - $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $database); + $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $dbForProject); $timeLimit ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getURI()); @@ -474,34 +482,20 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, Console::error('[Error] Code: ' . $response['data']['code']); Console::error('[Error] Message: ' . $response['data']['message']); } - - if ($th instanceof PDOException) { - $db = null; - } } finally { - /** - * Put used PDO and Redis Connections back into their pools. - */ - $dbPool->put($db); - $redisPool->put($redis); + $register->get('pools')->reclaim(); } }); $server->onMessage(function (int $connection, string $message) use ($server, $dbPool, $redisPool, $realtime, $containerId) { try { $response = new Response(new SwooleResponse()); - $db = $dbPool->get(); - $redis = $redisPool->get(); - - $cache = new Cache(new RedisCache($redis)); - $database = new Database(new MySQL($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace("_console"); $projectId = $realtime->connections[$connection]['projectId']; + $database = getConsoleDB(); if ($projectId !== 'console') { $project = Authorization::skip(fn() => $database->getDocument('projects', $projectId)); - $database->setNamespace("_{$project->getInternalId()}"); + $database = getProjectDB($project); } /* @@ -585,8 +579,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $db $server->close($connection, $th->getCode()); } } finally { - $dbPool->put($db); - $redisPool->put($redis); + $register->get('pools')->reclaim(); } }); diff --git a/app/tasks/doctor.php b/app/tasks/doctor.php index d79e8d530f..daeaaa3f52 100644 --- a/app/tasks/doctor.php +++ b/app/tasks/doctor.php @@ -8,6 +8,7 @@ use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; use Utopia\App; use Utopia\CLI\Console; +use Utopia\Config\Config; use Utopia\Domains\Domain; $cli @@ -21,7 +22,7 @@ $cli Console::log("\n" . '👩‍⚕️ Running ' . APP_NAME . ' Doctor for version ' . App::getEnv('_APP_VERSION', 'UNKNOWN') . ' ...' . "\n"); - Console::log('Checking for production best practices...'); + Console::log('[Settings]'); $domain = new Domain(App::getEnv('_APP_DOMAIN')); @@ -77,7 +78,6 @@ $cli Console::log('🟢 HTTPS force option is enabled'); } - $providerName = App::getEnv('_APP_LOGGING_PROVIDER', ''); $providerConfig = App::getEnv('_APP_LOGGING_CONFIG', ''); @@ -90,30 +90,55 @@ $cli \sleep(0.2); try { - Console::log("\n" . 'Checking connectivity...'); + Console::log("\n" . '[Connectivity]'); } catch (\Throwable $th) { //throw $th; } - try { - $register->get('db'); /* @var $db PDO */ - Console::success('Database............connected 👍'); - } catch (\Throwable $th) { - Console::error('Database.........disconnected 👎'); + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $configs = [ + 'Console.DB' => Config::getParam('pools-console'), + 'Projects.DB' => Config::getParam('pools-database'), + ]; + + foreach ($configs as $key => $config) { + foreach ($config as $database) { + try { + $adapter = $pools->get($database)->pop()->getResource(); + + if ($adapter->ping()) { + Console::success('🟢 ' . str_pad("{$key}({$database})", 50, '.') . 'connected'); + } else { + Console::error('🔴 ' . str_pad("{$key}({$database})", 47, '.') . 'disconnected'); + } + } catch (\Throwable $th) { + Console::error('🔴 ' . str_pad("{$key}.({$database})", 47, '.') . 'disconnected'); + } + } } - try { - $register->get('cache'); - Console::success('Queue...............connected 👍'); - } catch (\Throwable $th) { - Console::error('Queue............disconnected 👎'); - } + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + $configs = [ + 'Cache' => Config::getParam('pools-cache'), + 'Queue' => Config::getParam('pools-queue'), + 'PubSub' => Config::getParam('pools-pubsub'), + ]; - try { - $register->get('cache'); - Console::success('Cache...............connected 👍'); - } catch (\Throwable $th) { - Console::error('Cache............disconnected 👎'); + foreach ($configs as $key => $config) { + foreach ($config as $pool) { + try { + $adapter = $pools->get($pool)->pop()->getResource(); + + if ($adapter->ping()) { + Console::success('🟢 ' . str_pad("{$key}({$pool})", 50, '.') . 'connected'); + } else { + Console::error('🔴 ' . str_pad("{$key}({$pool})", 47, '.') . 'disconnected'); + } + } catch (\Throwable $th) { + Console::error('🔴 ' . str_pad("{$key}({$pool})", 47, '.') . 'disconnected'); + } + } } if (App::getEnv('_APP_STORAGE_ANTIVIRUS') === 'enabled') { // Check if scans are enabled @@ -124,12 +149,12 @@ $cli ); if ((@$antivirus->ping())) { - Console::success('Antivirus...........connected 👍'); + Console::success('🟢 ' . str_pad("Antivirus", 50, '.') . 'connected'); } else { - Console::error('Antivirus........disconnected 👎'); + Console::error('🔴 ' . str_pad("Antivirus", 47, '.') . 'disconnected'); } } catch (\Throwable $th) { - Console::error('Antivirus........disconnected 👎'); + Console::error('🔴 ' . str_pad("Antivirus", 47, '.') . 'disconnected'); } } @@ -142,35 +167,35 @@ $cli $mail->AltBody = 'Hello World'; $mail->send(); - Console::success('SMTP................connected 👍'); + Console::success('🟢 ' . str_pad("SMTP", 50, '.') . 'connected'); } catch (\Throwable $th) { - Console::error('SMTP.............disconnected 👎'); + Console::error('🔴 ' . str_pad("SMTP", 47, '.') . 'disconnected'); } $host = App::getEnv('_APP_STATSD_HOST', 'telegraf'); $port = App::getEnv('_APP_STATSD_PORT', 8125); if ($fp = @\fsockopen('udp://' . $host, $port, $errCode, $errStr, 2)) { - Console::success('StatsD..............connected 👍'); + Console::success('🟢 ' . str_pad("StatsD", 50, '.') . 'connected'); \fclose($fp); } else { - Console::error('StatsD...........disconnected 👎'); + Console::error('🔴 ' . str_pad("StatsD", 47, '.') . 'disconnected'); } $host = App::getEnv('_APP_INFLUXDB_HOST', ''); $port = App::getEnv('_APP_INFLUXDB_PORT', ''); if ($fp = @\fsockopen($host, $port, $errCode, $errStr, 2)) { - Console::success('InfluxDB............connected 👍'); + Console::success('🟢 ' . str_pad("InfluxDB", 50, '.') . 'connected'); \fclose($fp); } else { - Console::error('InfluxDB.........disconnected 👎'); + Console::error('🔴 ' . str_pad("InfluxDB", 47, '.') . 'disconnected'); } \sleep(0.2); Console::log(''); - Console::log('Checking volumes...'); + Console::log('[Volumes]'); foreach ( [ @@ -198,7 +223,7 @@ $cli \sleep(0.2); Console::log(''); - Console::log('Checking disk space usage...'); + Console::log('[Disk Space]'); foreach ( [ diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index b8ad5adcbd..1c16ca6911 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -1,52 +1,16 @@ get('cache'))); - $database = new Database(new MySQL($register->get('db')), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace('_console'); // Main DB - - if (!$database->exists($database->getDefaultDatabase(), 'certificates')) { - throw new \Exception('Console project not ready'); - } - - break; // leave loop if successful - } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep(DATABASE_RECONNECT_SLEEP); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return $database; -} - $cli ->task('maintenance') ->desc('Schedules maintenance tasks and publishes them to resque') diff --git a/app/tasks/migrate.php b/app/tasks/migrate.php index c30cd33459..bf3d1ff602 100644 --- a/app/tasks/migrate.php +++ b/app/tasks/migrate.php @@ -7,9 +7,6 @@ use Appwrite\Migration\Migration; use Utopia\App; use Utopia\Cache\Cache; use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Database\Adapter\MySQL; -use Utopia\Database\Database; -use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Validator\Text; @@ -28,17 +25,13 @@ $cli Console::success('Starting Data Migration to version ' . $version); - $db = $register->get('db', true); + $dbPool = $register->get('dbPool', true); $redis = $register->get('cache', true); $redis->flushAll(); $cache = new Cache(new RedisCache($redis)); - $projectDB = new Database(new MySQL($db), $cache); - $projectDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - - $consoleDB = new Database(new MySQL($db), $cache); - $consoleDB->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $consoleDB->setNamespace('_project_console'); + $dbForConsole = $dbPool->getDB('console', $cache); + $dbForConsole->setNamespace('_project_console'); $console = $app->getResource('console'); @@ -52,10 +45,10 @@ $cli $count = 0; try { - $totalProjects = $consoleDB->count('projects') + 1; + $totalProjects = $dbForConsole->count('projects') + 1; } catch (\Throwable $th) { - $consoleDB->setNamespace('_console'); - $totalProjects = $consoleDB->count('projects') + 1; + $dbForConsole->setNamespace('_console'); + $totalProjects = $dbForConsole->count('projects') + 1; } $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version]; @@ -71,8 +64,10 @@ $cli } try { + // TODO: Iterate through all project DBs + $projectDB = $dbPool->getDB($project->getId(), $cache); $migration - ->setProject($project, $projectDB, $consoleDB) + ->setProject($project, $projectDB, $dbForConsole) ->execute(); } catch (\Throwable $th) { throw $th; @@ -81,7 +76,7 @@ $cli } $sum = \count($projects); - $projects = $consoleDB->find('projects', [Query::limit($limit), Query::offset($offset)]); + $projects = $dbForConsole->find('projects', limit: $limit, offset: $offset); $offset = $offset + $limit; $count = $count + $sum; diff --git a/app/tasks/specs.php b/app/tasks/specs.php index 57c6bf29e1..b3ac763b39 100644 --- a/app/tasks/specs.php +++ b/app/tasks/specs.php @@ -9,8 +9,12 @@ use Appwrite\Specification\Specification; use Appwrite\Utopia\Response; use Swoole\Http\Response as HttpResponse; use Utopia\App; +use Utopia\Cache\Adapter\None; +use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Config\Config; +use Utopia\Database\Adapter\MySQL; +use Utopia\Database\Database; use Utopia\Request; use Utopia\Validator\WhiteList; @@ -19,16 +23,15 @@ $cli ->param('version', 'latest', new Text(16), 'Spec version', true) ->param('mode', 'normal', new WhiteList(['normal', 'mocks']), 'Spec Mode', true) ->action(function ($version, $mode) use ($register) { - $db = $register->get('db'); - $redis = $register->get('cache'); $appRoutes = App::getRoutes(); $response = new Response(new HttpResponse()); $mocks = ($mode === 'mocks'); + // Mock dependencies App::setResource('request', fn () => new Request()); App::setResource('response', fn () => $response); - App::setResource('db', fn () => $db); - App::setResource('cache', fn () => $redis); + App::setResource('dbForConsole', fn () => new Database(new MySQL(''), new Cache(new None()))); + App::setResource('dbForProject', fn () => new Database(new MySQL(''), new Cache(new None()))); $platforms = [ 'client' => APP_PLATFORM_CLIENT, @@ -203,7 +206,7 @@ $cli unset($models[$key]); } } - // var_dump($models); + $arguments = [new App('UTC'), $services, $routes, $models, $keys[$platform], $authCounts[$platform] ?? 0]; foreach (['swagger2', 'open-api3'] as $format) { $formatInstance = match ($format) { diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 34519bcce4..d1aeab2e84 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -2,84 +2,20 @@ global $cli, $register; -use Appwrite\Stats\Usage; -use Appwrite\Stats\UsageDB; use Appwrite\Usage\Calculators\Aggregator; use Appwrite\Usage\Calculators\Database; use Appwrite\Usage\Calculators\TimeSeries; use InfluxDB\Database as InfluxDatabase; use Utopia\App; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\Cache\Cache; use Utopia\CLI\Console; -use Utopia\Database\Adapter\MySQL; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Validator\Authorization; -use Utopia\Registry\Registry; use Utopia\Logger\Log; use Utopia\Validator\WhiteList; Authorization::disable(); Authorization::setDefaultStatus(false); -function getDatabase(Registry &$register, string $namespace): UtopiaDatabase -{ - $attempts = 0; - - do { - try { - $attempts++; - - $db = $register->get('db'); - $redis = $register->get('cache'); - - $cache = new Cache(new RedisCache($redis)); - $database = new UtopiaDatabase(new MySQL($db), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace($namespace); - - if (!$database->exists($database->getDefaultDatabase(), 'projects')) { - throw new Exception('Projects collection not ready'); - } - break; // leave loop if successful - } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep(DATABASE_RECONNECT_SLEEP); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return $database; -} - -function getInfluxDB(Registry &$register): InfluxDatabase -{ - /** @var InfluxDB\Client $client */ - $client = $register->get('influxdb'); - $attempts = 0; - $max = 10; - $sleep = 1; - - do { // check if telegraf database is ready - try { - $attempts++; - $database = $client->selectDB('telegraf'); - if (in_array('telegraf', $client->listDatabases())) { - break; // leave the do-while if successful - } - } catch (\Throwable $th) { - Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); - if ($attempts >= $max) { - throw new \Exception('InfluxDB database not ready yet'); - } - sleep($sleep); - } - } while ($attempts < $max); - return $database; -} - $logError = function (Throwable $error, string $action = 'syncUsageStats') use ($register) { $logger = $register->get('logger'); @@ -114,7 +50,6 @@ $logError = function (Throwable $error, string $action = 'syncUsageStats') use ( Console::warning($error->getTraceAsString()); }; - function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void { $interval = (int) App::getEnv('_APP_USAGE_TIMESERIES_INTERVAL', '30'); // 30 seconds (by default) @@ -156,12 +91,12 @@ $cli ->task('usage') ->param('type', 'timeseries', new WhiteList(['timeseries', 'database'])) ->desc('Schedules syncing data from influxdb to Appwrite console db') - ->action(function (string $type) use ($register, $logError) { + ->action(function (string $type) use ($logError) { Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); - $database = getDatabase($register, '_console'); - $influxDB = getInfluxDB($register); + $database = getConsoleDB(); + $influxDB = getInfluxDB(); switch ($type) { case 'timeseries': diff --git a/app/tasks/volume-sync.php b/app/tasks/volume-sync.php index fe0eee07b9..8d5fec201b 100644 --- a/app/tasks/volume-sync.php +++ b/app/tasks/volume-sync.php @@ -18,6 +18,11 @@ $cli Console::title('RSync V1'); Console::success(APP_NAME . ' rsync process v1 has started'); + if (!file_exists($source)) { + Console::error('Source directory does not exist. Exiting ... '); + Console::exit(0); + } + Console::loop(function () use ($interval, $source, $destination) { $time = DateTime::now(); diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index 0442d8456b..1d7fdcd046 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -88,15 +88,15 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -184,13 +184,15 @@ services: - _APP_WORKER_PER_CORE - _APP_OPTIONS_ABUSE - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_USAGE_STATS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -209,15 +211,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -263,15 +265,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_STORAGE_DEVICE - _APP_STORAGE_S3_ACCESS_KEY - _APP_STORAGE_S3_SECRET @@ -312,15 +314,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -340,15 +342,15 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -372,15 +374,15 @@ services: - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -399,15 +401,15 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_FUNCTIONS_TIMEOUT - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST @@ -535,15 +537,15 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE @@ -571,14 +573,14 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_TIMESERIES_INTERVAL + - _APP_USAGE_DATABASE_INTERVAL - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -603,14 +605,14 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_TIMESERIES_INTERVAL + - _APP_USAGE_DATABASE_INTERVAL - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG diff --git a/app/workers/audits.php b/app/workers/audits.php index b86649543b..90ac020536 100644 --- a/app/workers/audits.php +++ b/app/workers/audits.php @@ -1,6 +1,5 @@ getAttribute('name', ''); $userEmail = $user->getAttribute('email', ''); - $dbForProject = $this->getProjectDB($project->getId()); + $dbForProject = $this->getProjectDB($project); $audit = new Audit($dbForProject); $audit->log( userId: $user->getId(), diff --git a/app/workers/builds.php b/app/workers/builds.php index bf780c6464..235ad0da72 100644 --- a/app/workers/builds.php +++ b/app/workers/builds.php @@ -7,7 +7,6 @@ use Appwrite\Utopia\Response\Model\Deployment; use Cron\CronExpression; use Executor\Executor; use Appwrite\Usage\Stats; -use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\App; use Utopia\CLI\Console; @@ -15,7 +14,6 @@ use Utopia\Database\ID; use Utopia\Storage\Storage; use Utopia\Database\Document; use Utopia\Config\Config; -use Utopia\Database\Query; require_once __DIR__ . '/../init.php'; @@ -59,7 +57,7 @@ class BuildsV1 extends Worker protected function buildDeployment(Document $project, Document $function, Document $deployment) { - $dbForProject = $this->getProjectDB($project->getId()); + $dbForProject = $this->getProjectDB($project); $function = $dbForProject->getDocument('functions', $function->getId()); if ($function->isEmpty()) { diff --git a/app/workers/certificates.php b/app/workers/certificates.php index e7885bc669..b4f0701c46 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -1,6 +1,5 @@ createAttribute($database, $collection, $document, $project->getId()); + $this->createAttribute($database, $collection, $document, $project); break; case DATABASE_TYPE_DELETE_ATTRIBUTE: - $this->deleteAttribute($database, $collection, $document, $project->getId()); + $this->deleteAttribute($database, $collection, $document, $project); break; case DATABASE_TYPE_CREATE_INDEX: - $this->createIndex($database, $collection, $document, $project->getId()); + $this->createIndex($database, $collection, $document, $project); break; case DATABASE_TYPE_DELETE_INDEX: - $this->deleteIndex($database, $collection, $document, $project->getId()); + $this->deleteIndex($database, $collection, $document, $project); break; default: @@ -61,12 +61,13 @@ class DatabaseV1 extends Worker * @param Document $database * @param Document $collection * @param Document $attribute - * @param string $projectId + * @param Document $project */ - protected function createAttribute(Document $database, Document $collection, Document $attribute, string $projectId): void + protected function createAttribute(Document $database, Document $collection, Document $attribute, Document $project): void { + $projectId = $project->getId(); $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].update', [ 'databaseId' => $database->getId(), @@ -128,12 +129,13 @@ class DatabaseV1 extends Worker * @param Document $database * @param Document $collection * @param Document $attribute - * @param string $projectId + * @param Document $project */ - protected function deleteAttribute(Document $database, Document $collection, Document $attribute, string $projectId): void + protected function deleteAttribute(Document $database, Document $collection, Document $attribute, Document $project): void { + $projectId = $project->getId(); $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete', [ 'databaseId' => $database->getId(), @@ -225,7 +227,7 @@ class DatabaseV1 extends Worker } if ($exists) { // Delete the duplicate if created, else update in db - $this->deleteIndex($database, $collection, $index, $projectId); + $this->deleteIndex($database, $collection, $index, $project); } else { $dbForProject->updateDocument('indexes', $index->getId(), $index); } @@ -241,12 +243,13 @@ class DatabaseV1 extends Worker * @param Document $database * @param Document $collection * @param Document $index - * @param string $projectId + * @param Document $project */ - protected function createIndex(Document $database, Document $collection, Document $index, string $projectId): void + protected function createIndex(Document $database, Document $collection, Document $index, Document $project): void { + $projectId = $project->getId(); $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].update', [ 'databaseId' => $database->getId(), @@ -298,12 +301,13 @@ class DatabaseV1 extends Worker * @param Document $database * @param Document $collection * @param Document $index - * @param string $projectId + * @param Document $project */ - protected function deleteIndex(Document $database, Document $collection, Document $index, string $projectId): void + protected function deleteIndex(Document $database, Document $collection, Document $index, Document $project): void { + $projectId = $project->getId(); $dbForConsole = $this->getConsoleDB(); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $events = Event::generateEvents('databases.[databaseId].collections.[collectionId].indexes.[indexId].delete', [ 'databaseId' => $database->getId(), diff --git a/app/workers/deletes.php b/app/workers/deletes.php index b015043b1d..9ef593dbb9 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -40,28 +40,28 @@ class DeletesV1 extends Worker switch ($document->getCollection()) { case DELETE_TYPE_DATABASES: - $this->deleteDatabase($document, $project->getId()); + $this->deleteDatabase($document, $project); break; case DELETE_TYPE_COLLECTIONS: - $this->deleteCollection($document, $project->getId()); + $this->deleteCollection($document, $project); break; case DELETE_TYPE_PROJECTS: $this->deleteProject($document); break; case DELETE_TYPE_FUNCTIONS: - $this->deleteFunction($document, $project->getId()); + $this->deleteFunction($document, $project); break; case DELETE_TYPE_DEPLOYMENTS: - $this->deleteDeployment($document, $project->getId()); + $this->deleteDeployment($document, $project); break; case DELETE_TYPE_USERS: - $this->deleteUser($document, $project->getId()); + $this->deleteUser($document, $project); break; case DELETE_TYPE_TEAMS: - $this->deleteMemberships($document, $project->getId()); + $this->deleteMemberships($document, $project); break; case DELETE_TYPE_BUCKETS: - $this->deleteBucket($document, $project->getId()); + $this->deleteBucket($document, $project); break; default: Console::error('No lazy delete operation available for document of type: ' . $document->getCollection()); @@ -82,7 +82,7 @@ class DeletesV1 extends Worker $document = new Document($this->args['document'] ?? []); if (!$document->isEmpty()) { - $this->deleteAuditLogsByResource('document/' . $document->getId(), $project->getId()); + $this->deleteAuditLogsByResource('document/' . $document->getId(), $project); } break; @@ -109,7 +109,7 @@ class DeletesV1 extends Worker break; case DELETE_TYPE_CACHE_BY_RESOURCE: - $this->deleteCacheByResource($project->getId()); + $this->deleteCacheByResource($this->args['resource']); break; case DELETE_TYPE_CACHE_BY_TIMESTAMP: $this->deleteCacheByDate(); @@ -125,12 +125,12 @@ class DeletesV1 extends Worker } /** - * @param string $projectId + * @param string $resource */ - protected function deleteCacheByResource(string $projectId): void + protected function deleteCacheByResource(string $resource): void { $this->deleteCacheFiles([ - Query::equal('resource', [$this->args['resource']]), + Query::equal('resource', [$resource]), ]); } @@ -143,9 +143,10 @@ class DeletesV1 extends Worker protected function deleteCacheFiles($query): void { - $this->deleteForProjectIds(function (string $projectId) use ($query) { + $this->deleteForProjectIds(function (Document $project) use ($query) { - $dbForProject = $this->getProjectDB($projectId); + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $cache = new Cache( new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $projectId) ); @@ -169,34 +170,35 @@ class DeletesV1 extends Worker /** * @param Document $document database document - * @param string $projectId + * @param Document $projectId */ - protected function deleteDatabase(Document $document, string $projectId): void + protected function deleteDatabase(Document $document, Document $project): void { $databaseId = $document->getId(); + $projectId = $project->getId(); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); - $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($projectId) { - $this->deleteCollection($document, $projectId); + $this->deleteByGroup('database_' . $document->getInternalId(), [], $dbForProject, function ($document) use ($project) { + $this->deleteCollection($document, $project); }); $dbForProject->deleteCollection('database_' . $document->getInternalId()); - $this->deleteAuditLogsByResource('database/' . $databaseId, $projectId); + $this->deleteAuditLogsByResource('database/' . $databaseId, $project); } /** * @param Document $document teams document - * @param string $projectId + * @param Document $project */ - protected function deleteCollection(Document $document, string $projectId): void + protected function deleteCollection(Document $document, Document $project): void { $collectionId = $document->getId(); $databaseId = $document->getAttribute('databaseId'); $databaseInternalId = $document->getAttribute('databaseInternalId'); - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $dbForProject->deleteCollection('database_' . $databaseInternalId . '_collection_' . $document->getInternalId()); @@ -210,7 +212,7 @@ class DeletesV1 extends Worker Query::equal('collectionId', [$collectionId]) ], $dbForProject); - $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $projectId); + $this->deleteAuditLogsByResource('database/' . $databaseId . '/collection/' . $collectionId, $project); } /** @@ -219,8 +221,8 @@ class DeletesV1 extends Worker */ protected function deleteUsageStats(string $datetime1d, string $datetime30m) { - $this->deleteForProjectIds(function (string $projectId) use ($datetime1d, $datetime30m) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime1d, $datetime30m) { + $dbForProject = $this->getProjectDB($project); // Delete Usage stats $this->deleteByGroup('stats', [ Query::lessThan('time', $datetime1d), @@ -236,16 +238,16 @@ class DeletesV1 extends Worker /** * @param Document $document teams document - * @param string $projectId + * @param Document $project */ - protected function deleteMemberships(Document $document, string $projectId): void + protected function deleteMemberships(Document $document, Document $project): void { $teamId = $document->getAttribute('teamId', ''); // Delete Memberships $this->deleteByGroup('memberships', [ Query::equal('teamId', [$teamId]) - ], $this->getProjectDB($projectId)); + ], $this->getProjectDB($project)); } /** @@ -256,7 +258,7 @@ class DeletesV1 extends Worker $projectId = $document->getId(); // Delete all DBs - $this->getProjectDB($projectId)->delete($projectId); + $this->getProjectDB($document)->delete($projectId); // Delete all storage directories $uploads = new Local(APP_STORAGE_UPLOADS . '/app-' . $document->getId()); @@ -268,30 +270,30 @@ class DeletesV1 extends Worker /** * @param Document $document user document - * @param string $projectId + * @param Document $project */ - protected function deleteUser(Document $document, string $projectId): void + protected function deleteUser(Document $document, Document $project): void { $userId = $document->getId(); // Delete all sessions of this user from the sessions table and update the sessions field of the user record $this->deleteByGroup('sessions', [ Query::equal('userId', [$userId]) - ], $this->getProjectDB($projectId)); + ], $this->getProjectDB($project)); - $this->getProjectDB($projectId)->deleteCachedDocument('users', $userId); + $this->getProjectDB($project)->deleteCachedDocument('users', $userId); // Delete Memberships and decrement team membership counts $this->deleteByGroup('memberships', [ Query::equal('userId', [$userId]) - ], $this->getProjectDB($projectId), function (Document $document) use ($projectId) { + ], $this->getProjectDB($project), function (Document $document) use ($project) { if ($document->getAttribute('confirm')) { // Count only confirmed members $teamId = $document->getAttribute('teamId'); - $team = $this->getProjectDB($projectId)->getDocument('teams', $teamId); + $team = $this->getProjectDB($project)->getDocument('teams', $teamId); if (!$team->isEmpty()) { $team = $this - ->getProjectDB($projectId) + ->getProjectDB($project) ->updateDocument( 'teams', $teamId, @@ -305,7 +307,7 @@ class DeletesV1 extends Worker // Delete tokens $this->deleteByGroup('tokens', [ Query::equal('userId', [$userId]) - ], $this->getProjectDB($projectId)); + ], $this->getProjectDB($project)); } /** @@ -313,8 +315,8 @@ class DeletesV1 extends Worker */ protected function deleteExecutionLogs(string $datetime): void { - $this->deleteForProjectIds(function (string $projectId) use ($datetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $dbForProject = $this->getProjectDB($project); // Delete Executions $this->deleteByGroup('executions', [ Query::lessThan('$createdAt', $datetime) @@ -327,8 +329,8 @@ class DeletesV1 extends Worker */ protected function deleteExpiredSessions(string $datetime): void { - $this->deleteForProjectIds(function (string $projectId) use ($datetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $dbForProject = $this->getProjectDB($project); // Delete Sessions $this->deleteByGroup('sessions', [ Query::lessThan('expire', $datetime) @@ -341,8 +343,8 @@ class DeletesV1 extends Worker */ protected function deleteRealtimeUsage(string $datetime): void { - $this->deleteForProjectIds(function (string $projectId) use ($datetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $dbForProject = $this->getProjectDB($project); // Delete Dead Realtime Logs $this->deleteByGroup('realtime', [ Query::lessThan('timestamp', $datetime) @@ -360,8 +362,9 @@ class DeletesV1 extends Worker throw new Exception('Failed to delete audit logs. No datetime provided'); } - $this->deleteForProjectIds(function (string $projectId) use ($datetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $timeLimit = new TimeLimit("", 0, 1, $dbForProject); $abuse = new Abuse($timeLimit); $status = $abuse->cleanup($datetime); @@ -381,8 +384,9 @@ class DeletesV1 extends Worker throw new Exception('Failed to delete audit logs. No datetime provided'); } - $this->deleteForProjectIds(function (string $projectId) use ($datetime) { - $dbForProject = $this->getProjectDB($projectId); + $this->deleteForProjectIds(function (Document $project) use ($datetime) { + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $audit = new Audit($dbForProject); $status = $audit->cleanup($datetime); if (!$status) { @@ -393,11 +397,11 @@ class DeletesV1 extends Worker /** * @param string $resource - * @param string $projectId + * @param Document $project */ - protected function deleteAuditLogsByResource(string $resource, string $projectId): void + protected function deleteAuditLogsByResource(string $resource, Document $project): void { - $dbForProject = $this->getProjectDB($projectId); + $dbForProject = $this->getProjectDB($project); $this->deleteByGroup(Audit::COLLECTION, [ Query::equal('resource', [$resource]) @@ -406,11 +410,12 @@ class DeletesV1 extends Worker /** * @param Document $document function document - * @param string $projectId + * @param Document $project */ - protected function deleteFunction(Document $document, string $projectId): void + protected function deleteFunction(Document $document, Document $project): void { - $dbForProject = $this->getProjectDB($projectId); + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $functionId = $document->getId(); /** @@ -479,11 +484,12 @@ class DeletesV1 extends Worker /** * @param Document $document deployment document - * @param string $projectId + * @param Document $project */ - protected function deleteDeployment(Document $document, string $projectId): void + protected function deleteDeployment(Document $document, Document $project): void { - $dbForProject = $this->getProjectDB($projectId); + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $deploymentId = $document->getId(); $functionId = $document->getAttribute('resourceId'); @@ -568,13 +574,11 @@ class DeletesV1 extends Worker $chunk++; /** @var string[] $projectIds */ - $projectIds = array_map(fn (Document $project) => $project->getId(), $projects); - $sum = count($projects); Console::info('Executing delete function for chunk #' . $chunk . '. Found ' . $sum . ' projects'); - foreach ($projectIds as $projectId) { - $callback($projectId); + foreach ($projects as $project) { + $callback($project); $count++; } } @@ -666,9 +670,10 @@ class DeletesV1 extends Worker } } - protected function deleteBucket(Document $document, string $projectId) + protected function deleteBucket(Document $document, Document $project) { - $dbForProject = $this->getProjectDB($projectId); + $projectId = $project->getId(); + $dbForProject = $this->getProjectDB($project); $dbForProject->deleteCollection('bucket_' . $document->getInternalId()); $device = $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId); diff --git a/app/workers/functions.php b/app/workers/functions.php index 1df776383f..1ba4b6575b 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -52,7 +52,7 @@ class FunctionsV1 extends Worker return; } - $database = $this->getProjectDB($project->getId()); + $database = $this->getProjectDB($project); /** * Handle Event execution. diff --git a/composer.json b/composer.json index 6a80960aec..91c561d7d1 100644 --- a/composer.json +++ b/composer.json @@ -43,12 +43,12 @@ "ext-sockets": "*", "appwrite/php-clamav": "1.1.*", "appwrite/php-runtimes": "0.11.*", - "utopia-php/framework": "0.21.*", + "utopia-php/framework": "0.23.*", "utopia-php/logger": "0.3.*", "utopia-php/abuse": "0.16.*", "utopia-php/analytics": "0.2.*", - "utopia-php/audit": "0.17.*", "utopia-php/cache": "0.8.*", + "utopia-php/audit": "0.17.*", "utopia-php/cli": "0.13.*", "utopia-php/config": "0.2.*", "utopia-php/database": "0.28.*", @@ -56,11 +56,12 @@ "utopia-php/registry": "dev-feat-allow-params as 0.5.0", "utopia-php/preloader": "0.2.*", "utopia-php/domains": "1.1.*", - "utopia-php/swoole": "0.3.*", + "utopia-php/swoole": "0.5.*", "utopia-php/storage": "0.11.*", "utopia-php/websocket": "0.1.0", "utopia-php/image": "0.5.*", "utopia-php/orchestration": "0.6.*", + "utopia-php/pools": "dev-feat-optimize-filling as 0.3.0", "resque/php-resque": "1.3.6", "matomo/device-detector": "6.0.0", "dragonmantank/cron-expression": "3.3.1", diff --git a/composer.lock b/composer.lock index 3d29f7563a..b245d26383 100644 --- a/composer.lock +++ b/composer.lock @@ -1945,24 +1945,24 @@ }, { "name": "utopia-php/framework", - "version": "0.21.1", + "version": "0.23.4", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "c81789b87a917da2daf336738170ebe01f50ea18" + "reference": "97f64aa1732af92b967c3576f16967dc762ad47b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/c81789b87a917da2daf336738170ebe01f50ea18", - "reference": "c81789b87a917da2daf336738170ebe01f50ea18", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/97f64aa1732af92b967c3576f16967dc762ad47b", + "reference": "97f64aa1732af92b967c3576f16967dc762ad47b", "shasum": "" }, "require": { "php": ">=8.0.0" }, "require-dev": { - "phpunit/phpunit": "^9.5.10", - "vimeo/psalm": "4.13.1" + "phpunit/phpunit": "^9.5.25", + "vimeo/psalm": "^4.27.0" }, "type": "library", "autoload": { @@ -1974,12 +1974,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "eldad@appwrite.io" - } - ], "description": "A simple, light and advanced PHP framework", "keywords": [ "framework", @@ -1988,9 +1982,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.21.1" + "source": "https://github.com/utopia-php/framework/tree/0.23.4" }, - "time": "2022-09-07T09:56:28+00:00" + "time": "2022-10-31T11:57:14+00:00" }, { "name": "utopia-php/image", @@ -2216,6 +2210,60 @@ }, "time": "2022-07-13T16:47:18+00:00" }, + { + "name": "utopia-php/pools", + "version": "dev-feat-optimize-filling", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/pools.git", + "reference": "be603898142f55df9db5058f81a610ead7769d7d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/be603898142f55df9db5058f81a610ead7769d7d", + "reference": "be603898142f55df9db5058f81a610ead7769d7d", + "shasum": "" + }, + "require": { + "ext-mongodb": "*", + "ext-pdo": "*", + "ext-redis": "*", + "php": ">=8.0", + "utopia-php/cli": "^0.13.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.4", + "vimeo/psalm": "4.0.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Pools\\": "src/Pools" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Team Appwrite", + "email": "team@appwrite.io" + } + ], + "description": "A simple library to manage connection pools", + "keywords": [ + "framework", + "php", + "pools", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/pools/issues", + "source": "https://github.com/utopia-php/pools/tree/feat-optimize-filling" + }, + "time": "2022-11-03T18:50:28+00:00" + }, { "name": "utopia-php/preloader", "version": "0.2.4", @@ -2378,16 +2426,16 @@ }, { "name": "utopia-php/swoole", - "version": "0.3.3", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/swoole.git", - "reference": "8312df69233b5dcd3992de88f131f238002749de" + "reference": "c2a3a4f944a2f22945af3cbcb95b13f0769628b1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/swoole/zipball/8312df69233b5dcd3992de88f131f238002749de", - "reference": "8312df69233b5dcd3992de88f131f238002749de", + "url": "https://api.github.com/repos/utopia-php/swoole/zipball/c2a3a4f944a2f22945af3cbcb95b13f0769628b1", + "reference": "c2a3a4f944a2f22945af3cbcb95b13f0769628b1", "shasum": "" }, "require": { @@ -2396,6 +2444,7 @@ "utopia-php/framework": "0.*.*" }, "require-dev": { + "laravel/pint": "1.2.*", "phpunit/phpunit": "^9.3", "swoole/ide-helper": "4.8.3", "vimeo/psalm": "4.15.0" @@ -2410,12 +2459,6 @@ "license": [ "MIT" ], - "authors": [ - { - "name": "Eldad Fux", - "email": "team@appwrite.io" - } - ], "description": "An extension for Utopia Framework to work with PHP Swoole as a PHP FPM alternative", "keywords": [ "framework", @@ -2428,9 +2471,9 @@ ], "support": { "issues": "https://github.com/utopia-php/swoole/issues", - "source": "https://github.com/utopia-php/swoole/tree/0.3.3" + "source": "https://github.com/utopia-php/swoole/tree/0.5.0" }, - "time": "2022-01-20T09:58:43+00:00" + "time": "2022-10-19T22:19:07+00:00" }, { "name": "utopia-php/system", @@ -5142,6 +5185,7 @@ ], "minimum-stability": "stable", "stability-flags": { + "utopia-php/pools": 20 "utopia-php/registry": 20 }, "prefer-stable": false, diff --git a/docker-compose.yml b/docker-compose.yml index 7fdf459a45..82d7b59c04 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -119,15 +119,20 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_DOMAIN - _APP_DOMAIN_TARGET - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE + - _APP_CONNECTIONS_PUBSUB - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -220,13 +225,19 @@ services: - _APP_WORKER_PER_CORE - _APP_OPTIONS_ABUSE - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_PUBSUB - _APP_USAGE_STATS - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -247,15 +258,19 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -281,6 +296,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -305,15 +321,19 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_STORAGE_DEVICE - _APP_STORAGE_S3_ACCESS_KEY - _APP_STORAGE_S3_SECRET @@ -350,22 +370,25 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - redis - mysql environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -387,15 +410,19 @@ services: - _APP_OPENSSL_KEY_V1 - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -420,15 +447,19 @@ services: - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -449,15 +480,19 @@ services: environment: - _APP_ENV - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE - _APP_FUNCTIONS_TIMEOUT - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST @@ -549,6 +584,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_SMTP_HOST - _APP_SMTP_PORT - _APP_SMTP_SECURE @@ -575,6 +611,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE - _APP_SMS_PROVIDER - _APP_SMS_FROM - _APP_LOGGING_PROVIDER @@ -590,7 +627,6 @@ services: volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src - #- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database depends_on: - redis environment: @@ -598,15 +634,18 @@ services: - _APP_DOMAIN - _APP_DOMAIN_TARGET - _APP_OPENSSL_KEY_V1 - - _APP_REDIS_HOST - - _APP_REDIS_PORT - - _APP_REDIS_USER - - _APP_REDIS_PASS - _APP_DB_HOST - _APP_DB_PORT - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE - _APP_MAINTENANCE_INTERVAL - _APP_MAINTENANCE_RETENTION_EXECUTION - _APP_MAINTENANCE_RETENTION_CACHE @@ -618,7 +657,6 @@ services: <<: *x-logging container_name: appwrite-volume-sync image: appwrite-dev - restart: unless-stopped command: - --source=/data/src/ --destination=/data/dest/ --interval=10 networks: @@ -651,14 +689,17 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_USAGE_TIMESERIES_INTERVAL + - _APP_USAGE_DATABASE_INTERVAL - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -686,14 +727,17 @@ services: - _APP_DB_SCHEMA - _APP_DB_USER - _APP_DB_PASS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - - _APP_USAGE_TIMESERIES_INTERVAL - - _APP_USAGE_DATABASE_INTERVAL - _APP_REDIS_HOST - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_CACHE + - _APP_USAGE_TIMESERIES_INTERVAL + - _APP_USAGE_DATABASE_INTERVAL - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG @@ -715,6 +759,7 @@ services: - _APP_REDIS_PORT - _APP_REDIS_USER - _APP_REDIS_PASS + - _APP_CONNECTIONS_QUEUE mysql: image: mysql:8.0.28-oracle # fix issues when upgrading using: mysql_upgrade -u root -p @@ -737,7 +782,6 @@ services: # smtp: # image: appwrite/smtp:1.2.0 # container_name: appwrite-smtp - # restart: unless-stopped # networks: # - appwrite # environment: @@ -824,7 +868,6 @@ services: image: adminer container_name: appwrite-adminer <<: *x-logging - restart: always ports: - 9506:8080 networks: @@ -832,7 +875,6 @@ services: # redis-commander: # image: rediscommander/redis-commander:latest - # restart: unless-stopped # networks: # - appwrite # environment: @@ -842,7 +884,6 @@ services: # resque: # image: appwrite/resque-web:1.1.0 - # restart: unless-stopped # networks: # - appwrite # ports: @@ -856,7 +897,6 @@ services: # chronograf: # image: chronograf:1.6 # container_name: appwrite-chronograf - # restart: unless-stopped # networks: # - appwrite # volumes: diff --git a/docs/references/health/get-cache.md b/docs/references/health/get-cache.md index 91abcd6bc5..632c02208d 100644 --- a/docs/references/health/get-cache.md +++ b/docs/references/health/get-cache.md @@ -1 +1 @@ -Check the Appwrite in-memory cache server is up and connection is successful. \ No newline at end of file +Check the Appwrite in-memory cache servers are up and connection is successful. \ No newline at end of file diff --git a/docs/references/health/get-db.md b/docs/references/health/get-db.md index 9652d0d3e3..7381e51f70 100644 --- a/docs/references/health/get-db.md +++ b/docs/references/health/get-db.md @@ -1 +1 @@ -Check the Appwrite database server is up and connection is successful. \ No newline at end of file +Check the Appwrite database servers are up and connection is successful. \ No newline at end of file diff --git a/docs/references/health/get-pubsub.md b/docs/references/health/get-pubsub.md new file mode 100644 index 0000000000..8f86411e8f --- /dev/null +++ b/docs/references/health/get-pubsub.md @@ -0,0 +1 @@ +Check the Appwrite pub-sub servers are up and connection is successful. \ No newline at end of file diff --git a/docs/references/health/get-queue.md b/docs/references/health/get-queue.md new file mode 100644 index 0000000000..e4558f941f --- /dev/null +++ b/docs/references/health/get-queue.md @@ -0,0 +1 @@ +Check the Appwrite queue messaging servers are up and connection is successful. \ No newline at end of file diff --git a/phpunit.xml b/phpunit.xml index 4012c8c276..927e91567a 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertWarningsToExceptions="true" processIsolation="false" stopOnFailure="false" -> + > diff --git a/src/Appwrite/Extend/PDO.php b/src/Appwrite/Extend/PDO.php deleted file mode 100644 index bc3f42afb9..0000000000 --- a/src/Appwrite/Extend/PDO.php +++ /dev/null @@ -1,110 +0,0 @@ -dsn = $dsn; - $this->username = $username; - $this->passwd = $passwd; - $this->options = $options; - - $this->pdo = new PDONative($dsn, $username, $passwd, $options); - } - - public function setAttribute($attribute, $value) - { - return $this->pdo->setAttribute($attribute, $value); - } - - public function prepare($statement, $driver_options = null) - { - return new PDOStatement($this, $this->pdo->prepare($statement, [])); - } - - public function quote($string, $parameter_type = PDONative::PARAM_STR) - { - return $this->pdo->quote($string, $parameter_type); - } - - public function beginTransaction() - { - try { - $result = $this->pdo->beginTransaction(); - } catch (\Throwable $th) { - $this->pdo = $this->reconnect(); - $result = $this->pdo->beginTransaction(); - } - - return $result; - } - - public function rollBack() - { - try { - $result = $this->pdo->rollBack(); - } catch (\Throwable $th) { - $this->pdo = $this->reconnect(); - return false; - } - - return $result; - } - - public function commit() - { - try { - $result = $this->pdo->commit(); - } catch (\Throwable $th) { - $this->pdo = $this->reconnect(); - $result = $this->pdo->commit(); - } - - return $result; - } - - public function reconnect(): PDONative - { - $this->pdo = new PDONative($this->dsn, $this->username, $this->passwd, $this->options); - - echo '[PDO] MySQL connection restarted' . PHP_EOL; - - // Connection settings - $this->pdo->setAttribute(PDONative::ATTR_DEFAULT_FETCH_MODE, PDONative::FETCH_ASSOC); // Return arrays - $this->pdo->setAttribute(PDONative::ATTR_ERRMODE, PDONative::ERRMODE_EXCEPTION); // Handle all errors with exceptions - - return $this->pdo; - } -} diff --git a/src/Appwrite/Extend/PDOStatement.php b/src/Appwrite/Extend/PDOStatement.php deleted file mode 100644 index 9c5a83ec34..0000000000 --- a/src/Appwrite/Extend/PDOStatement.php +++ /dev/null @@ -1,115 +0,0 @@ -pdo = &$pdo; - $this->PDOStatement = $PDOStatement; - } - - public function bindValue($parameter, $value, $data_type = PDONative::PARAM_STR) - { - $this->values[$parameter] = ['value' => $value, 'data_type' => $data_type]; - - $result = $this->PDOStatement->bindValue($parameter, $value, $data_type); - - return $result; - } - - public function bindParam($parameter, &$variable, $data_type = PDONative::PARAM_STR, $length = null, $driver_options = null) - { - $this->params[$parameter] = ['value' => &$variable, 'data_type' => $data_type, 'length' => $length, 'driver_options' => $driver_options]; - - $result = $this->PDOStatement->bindParam($parameter, $variable, $data_type, $length, $driver_options); - - return $result; - } - - public function bindColumn($column, &$param, $type = null, $maxlen = null, $driverdata = null) - { - $this->columns[$column] = ['param' => &$param, 'type' => $type, 'maxlen' => $maxlen, 'driverdata' => $driverdata]; - - $result = $this->PDOStatement->bindColumn($column, $param, $type, $maxlen, $driverdata); - - return $result; - } - - public function execute($input_parameters = null) - { - try { - $result = $this->PDOStatement->execute($input_parameters); - } catch (\Throwable $th) { - $this->pdo = $this->pdo->reconnect(); - $this->PDOStatement = $this->pdo->prepare($this->PDOStatement->queryString, []); - - foreach ($this->values as $key => $set) { - $this->PDOStatement->bindValue($key, $set['value'], $set['data_type']); - } - - foreach ($this->params as $key => $set) { - $this->PDOStatement->bindParam($key, $set['variable'], $set['data_type'], $set['length'], $set['driver_options']); - } - - foreach ($this->columns as $key => $set) { - $this->PDOStatement->bindColumn($key, $set['param'], $set['type'], $set['maxlen'], $set['driverdata']); - } - - $result = $this->PDOStatement->execute($input_parameters); - } - - return $result; - } - - public function fetch($fetch_style = PDONative::FETCH_ASSOC, $cursor_orientation = PDONative::FETCH_ORI_NEXT, $cursor_offset = 0) - { - $result = $this->PDOStatement->fetch($fetch_style, $cursor_orientation, $cursor_offset); - - return $result; - } - - /** - * Fetch All - * - * @param int $fetch_style - * @param mixed $fetch_args - * - * @return array|false - */ - public function fetchAll(int $fetch_style = PDO::FETCH_BOTH, mixed ...$fetch_args) - { - $result = $this->PDOStatement->fetchAll(); - - return $result; - } -} diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index c87c907bf0..2ac2641b95 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -86,8 +86,6 @@ abstract class Migration { $this->project = $project; $this->projectDB = $projectDB; - $this->projectDB->setNamespace('_' . $this->project->getId()); - $this->consoleDB = $consoleDB; return $this; diff --git a/src/Appwrite/Resque/Worker.php b/src/Appwrite/Resque/Worker.php index 8398813fff..5d05d77576 100644 --- a/src/Appwrite/Resque/Worker.php +++ b/src/Appwrite/Resque/Worker.php @@ -2,12 +2,12 @@ namespace Appwrite\Resque; +use Exception; use Utopia\App; use Utopia\Cache\Cache; -use Utopia\Cache\Adapter\Redis as RedisCache; -use Utopia\CLI\Console; +use Utopia\Config\Config; +use Utopia\Cache\Adapter\Sharding; use Utopia\Database\Database; -use Utopia\Database\Adapter\MySQL; use Utopia\Storage\Device; use Utopia\Storage\Storage; use Utopia\Storage\Device\Local; @@ -16,7 +16,7 @@ use Utopia\Storage\Device\Linode; use Utopia\Storage\Device\Wasabi; use Utopia\Storage\Device\Backblaze; use Utopia\Storage\Device\S3; -use Exception; +use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; abstract class Worker @@ -136,7 +136,12 @@ abstract class Worker */ public function tearDown(): void { + global $register; + try { + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + $pools->reclaim(); + $this->shutdown(); } catch (\Throwable $error) { foreach (self::$errorCallbacks as $errorCallback) { @@ -158,23 +163,32 @@ abstract class Worker { \array_push(self::$errorCallbacks, $callback); } + /** * Get internal project database - * @param string $projectId + * @param Document $project * @return Database */ - protected function getProjectDB(string $projectId): Database + protected function getProjectDB(Document $project): Database { - $consoleDB = $this->getConsoleDB(); + global $register; - if ($projectId === 'console') { - return $consoleDB; + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + if ($project->isEmpty() || $project->getId() === 'console') { + return $this->getConsoleDB(); } - /** @var Document $project */ - $project = Authorization::skip(fn() => $consoleDB->getDocument('projects', $projectId)); + $dbAdapter = $pools + ->get($project->getAttribute('database')) + ->pop() + ->getResource() + ; - return $this->getDB(self::DATABASE_PROJECT, $projectId, $project->getInternalId()); + $database = new Database($dbAdapter, $this->getCache()); + $database->setNamespace('_' . $project->getInternalId()); + + return $database; } /** @@ -183,67 +197,46 @@ abstract class Worker */ protected function getConsoleDB(): Database { - return $this->getDB(self::DATABASE_CONSOLE); + global $register; + + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ + + $dbAdapter = $pools + ->get('console') + ->pop() + ->getResource() + ; + + $database = new Database($dbAdapter, $this->getCache()); + + $database->setNamespace('console'); + + return $database; } + /** - * Get console database - * @param string $type One of (internal, external, console) - * @param string $projectId of internal or external DB - * @return Database + * Get Cache + * @return Cache */ - private function getDB(string $type, string $projectId = '', string $projectInternalId = ''): Database + protected function getCache(): Cache { global $register; - $namespace = ''; - $sleep = DATABASE_RECONNECT_SLEEP; // overwritten when necessary + $pools = $register->get('pools'); /** @var \Utopia\Pools\Group $pools */ - switch ($type) { - case self::DATABASE_PROJECT: - if (!$projectId) { - throw new \Exception('ProjectID not provided - cannot get database'); - } - $namespace = "_{$projectInternalId}"; - break; - case self::DATABASE_CONSOLE: - $namespace = "_console"; - $sleep = 5; // ConsoleDB needs extra sleep time to ensure tables are created - break; - default: - throw new \Exception('Unknown database type: ' . $type); - break; + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; } - $attempts = 0; - - do { - try { - $attempts++; - $cache = new Cache(new RedisCache($register->get('cache'))); - $database = new Database(new MySQL($register->get('db')), $cache); - $database->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite')); - $database->setNamespace($namespace); // Main DB - - if (!empty($projectId) && !$database->getDocument('projects', $projectId)->isEmpty()) { - throw new \Exception("Project does not exist: {$projectId}"); - } - - if ($type === self::DATABASE_CONSOLE && !$database->exists($database->getDefaultDatabase(), Database::METADATA)) { - throw new \Exception('Console project not ready'); - } - - break; // leave loop if successful - } catch (\Exception $e) { - Console::warning("Database not ready. Retrying connection ({$attempts})..."); - if ($attempts >= DATABASE_RECONNECT_MAX_ATTEMPTS) { - throw new \Exception('Failed to connect to database: ' . $e->getMessage()); - } - sleep($sleep); - } - } while ($attempts < DATABASE_RECONNECT_MAX_ATTEMPTS); - - return $database; + return new Cache(new Sharding($adapters)); } /** @@ -266,7 +259,6 @@ abstract class Worker return $this->getDevice(APP_STORAGE_UPLOADS . '/app-' . $projectId); } - /** * Get Builds Storage Device * @param string $projectId of the project diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index e23335365a..07134c6ea9 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -207,6 +207,7 @@ class Response extends SwooleResponse public const MODEL_HEALTH_QUEUE = 'healthQueue'; public const MODEL_HEALTH_TIME = 'healthTime'; public const MODEL_HEALTH_ANTIVIRUS = 'healthAntivirus'; + public const MODEL_HEALTH_STATUS_LIST = 'healthStatusList'; // Deprecated public const MODEL_PERMISSIONS = 'permissions'; @@ -268,6 +269,7 @@ class Response extends SwooleResponse ->setModel(new BaseList('Phones List', self::MODEL_PHONE_LIST, 'phones', self::MODEL_PHONE)) ->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metrics', self::MODEL_METRIC, true, false)) ->setModel(new BaseList('Variables List', self::MODEL_VARIABLE_LIST, 'variables', self::MODEL_VARIABLE)) + ->setModel(new BaseList('Status List', self::MODEL_HEALTH_STATUS_LIST, 'statuses', self::MODEL_HEALTH_STATUS)) // Entities ->setModel(new Database()) ->setModel(new Collection()) diff --git a/src/Appwrite/Utopia/Response/Model/HealthStatus.php b/src/Appwrite/Utopia/Response/Model/HealthStatus.php index 23756de131..ba340107ac 100644 --- a/src/Appwrite/Utopia/Response/Model/HealthStatus.php +++ b/src/Appwrite/Utopia/Response/Model/HealthStatus.php @@ -10,6 +10,12 @@ class HealthStatus extends Model public function __construct() { $this + ->addRule('name', [ + 'type' => self::TYPE_STRING, + 'description' => 'Name of the service.', + 'default' => '', + 'example' => 'database', + ]) ->addRule('ping', [ 'type' => self::TYPE_INTEGER, 'description' => 'Duration in milliseconds how long the health check took.', diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 47a2268e21..96c9bde5c7 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -47,9 +47,9 @@ class HealthCustomServerTest extends Scope ], $this->getHeaders()), []); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals('pass', $response['body']['status']); - $this->assertIsInt($response['body']['ping']); - $this->assertLessThan(100, $response['body']['ping']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); /** * Test for FAILURE @@ -69,9 +69,53 @@ class HealthCustomServerTest extends Scope ], $this->getHeaders()), []); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals('pass', $response['body']['status']); - $this->assertIsInt($response['body']['ping']); - $this->assertLessThan(100, $response['body']['ping']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); + + /** + * Test for FAILURE + */ + + return []; + } + + public function testQueueSuccess(): array + { + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); + + /** + * Test for FAILURE + */ + + return []; + } + + public function testPubSubSuccess(): array + { + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/health/pubsub', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals('pass', $response['body']['statuses'][0]['status']); + $this->assertIsInt($response['body']['statuses'][0]['ping']); + $this->assertLessThan(100, $response['body']['statuses'][0]['ping']); /** * Test for FAILURE