From c6c7734c29e8a8bd0873054ac191cd2b5b77ec9a Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Wed, 11 Oct 2023 22:01:01 +0200 Subject: [PATCH] Merge pull request #6496 from appwrite/fix-readme-cn Update logo in README-CN.md ## What does this PR do? (Provide a description of what this PR does and why it's needed.) ## Test Plan (Write your test plan here. If you changed any code, please provide us with clear instructions on how you verified your changes work. Screenshots may also be helpful.) ## Related PRs and Issues - (Related PR or issue) ## Checklist - [ ] Have you read the [Contributing Guidelines on issues](https://github.com/appwrite/appwrite/blob/master/CONTRIBUTING.md)? - [ ] If the PR includes a change to an API's metadata (desc, label, params, etc.), does it also include updated API specs and example docs? --- app/http.php | 23 +++++- app/init.php | 76 +++++++++++-------- app/test | 184 ++++++++++++++++++++++++++++++++++++++++++++++ composer.lock | 22 +++--- package 2.json | 8 ++ package-lock.json | 10 +++ pnpm-lock.yaml | 5 ++ 7 files changed, 284 insertions(+), 44 deletions(-) create mode 100644 app/test create mode 100644 package 2.json create mode 100644 package-lock.json create mode 100644 pnpm-lock.yaml diff --git a/app/http.php b/app/http.php index fe1ed48724..72b89130b9 100644 --- a/app/http.php +++ b/app/http.php @@ -328,7 +328,28 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo $swooleResponse->end(\json_encode($output)); } finally { - $pools->reclaim(); + $connectionForConsole = $app->getResource('connectionForConsole'); + $connectionForProject = $app->getResource('connectionForProject'); + $connectionForQueue = $app->getResource('connectionForQueue'); + $connectionsForCache = $app->getResource('connectionsForCache'); + + if(!is_null($connectionForConsole)) { + $connectionForConsole->reclaim(); + } + + if(!is_null($connectionForProject)) { + $connectionForProject->reclaim(); + } + + if(!is_null($connectionForQueue)) { + $connectionForQueue->reclaim(); + } + + if(!empty($connectionsForCache)) { + foreach ($connectionsForCache as $connection) { + $connection->reclaim(); + } + } } }); diff --git a/app/init.php b/app/init.php index ba133ec7fb..7078f90870 100644 --- a/app/init.php +++ b/app/init.php @@ -893,12 +893,6 @@ App::setResource('mails', fn() => new Mail()); App::setResource('deletes', fn() => new Delete()); App::setResource('database', fn() => new EventDatabase()); App::setResource('messaging', fn() => new Phone()); -App::setResource('queue', function (Group $pools) { - return $pools->get('queue')->pop()->getResource(); -}, ['pools']); -App::setResource('queueForFunctions', function (Connection $queue) { - return new Func($queue); -}, ['queue']); App::setResource('usage', function ($register) { return new Stats($register->get('statsd')); }, ['register']); @@ -1091,17 +1085,25 @@ App::setResource('console', function () { ]); }, []); +App::setResource('connectionForProject', function () { return null; }, []); +App::setResource('connectionForConsole', function () { return null; }, []); +App::setResource('connectionForQueue', function () { return null; }, []); +App::setResource('connectionsForCache', function () { return []; }, []); + App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; } - $dbAdapter = $pools + $connection = $pools ->get($project->getAttribute('database')) ->pop() - ->getResource() ; + App::setResource('connectionForProject', function () use ($connection) { return $connection; }, []); + + $dbAdapter = $connection->getResource(); + $database = new Database($dbAdapter, $cache); $database->setNamespace('_' . $project->getInternalId()); @@ -1109,11 +1111,13 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForConsole, }, ['pools', 'dbForConsole', 'cache', 'project']); App::setResource('dbForConsole', function (Group $pools, Cache $cache) { - $dbAdapter = $pools + $connection = $pools ->get('console') - ->pop() - ->getResource() - ; + ->pop(); + + App::setResource('connectionForConsole', function () use ($connection) { return $connection; }, []); + + $dbAdapter = $connection->getResource(); $database = new Database($dbAdapter, $cache); @@ -1123,30 +1127,22 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) { }, ['pools', 'cache']); App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) { - $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - $getProjectDB = function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForConsole; } - - $databaseName = $project->getAttribute('database'); - - if (isset($databases[$databaseName])) { - $database = $databases[$databaseName]; - $database->setNamespace('_' . $project->getInternalId()); - return $database; - } - - $dbAdapter = $pools - ->get($databaseName) + + $connection = $pools + ->get($project->getAttribute('database')) ->pop() - ->getResource(); + ; + + $dbAdapter = $connection->getResource(); + + App::setResource('connectionForProject', function () use ($connection) { return $connection; }, []); $database = new Database($dbAdapter, $cache); - $databases[$databaseName] = $database; - $database->setNamespace('_' . $project->getInternalId()); return $database; @@ -1155,18 +1151,34 @@ App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, return $getProjectDB; }, ['pools', 'dbForConsole', 'cache']); +App::setResource('queue', function (Group $pools) { + $connection = $pools->get('queue')->pop(); + + App::setResource('connectionForQueue', function () use ($connection) { return $connection; }, []); + + return $connection->getResource(); +}, ['pools']); + +App::setResource('queueForFunctions', function (Connection $queue) { + return new Func($queue); +}, ['queue']); + App::setResource('cache', function (Group $pools) { $list = Config::getParam('pools-cache', []); $adapters = []; + $connections = []; foreach ($list as $value) { - $adapters[] = $pools + $connection = $pools ->get($value) - ->pop() - ->getResource() - ; + ->pop(); + + $connections[] = $connection; + $adapters[] = $connection->getResource(); } + App::setResource('connectionsForCache', function () use ($connections) { return $connections; }, []); + return new Cache(new Sharding($adapters)); }, ['pools']); diff --git a/app/test b/app/test new file mode 100644 index 0000000000..a4d28b1200 --- /dev/null +++ b/app/test @@ -0,0 +1,184 @@ +$register->set('pools', function () { + $group = new Group(); + + $fallbackForDB = 'db_main=' . AppwriteURL::unparse([ + 'scheme' => 'mariadb', + 'host' => App::getEnv('_APP_DB_HOST', 'mariadb'), + 'port' => App::getEnv('_APP_DB_PORT', '3306'), + 'user' => App::getEnv('_APP_DB_USER', ''), + 'pass' => App::getEnv('_APP_DB_PASS', ''), + 'path' => App::getEnv('_APP_DB_SCHEMA', ''), + ]); + $fallbackForRedis = 'redis_main=' . AppwriteURL::unparse([ + 'scheme' => 'redis', + 'host' => App::getEnv('_APP_REDIS_HOST', 'redis'), + 'port' => App::getEnv('_APP_REDIS_PORT', '6379'), + 'user' => App::getEnv('_APP_REDIS_USER', ''), + 'pass' => App::getEnv('_APP_REDIS_PASS', ''), + ]); + + $connections = [ + 'console' => [ + 'type' => 'database', + 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), + 'multiple' => false, + 'schemes' => ['mariadb', 'mysql'], + ], + 'database' => [ + 'type' => 'database', + 'dsns' => App::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), + 'multiple' => true, + 'schemes' => ['mariadb', 'mysql'], + ], + 'queue' => [ + 'type' => 'queue', + 'dsns' => App::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'pubsub' => [ + 'type' => 'pubsub', + 'dsns' => App::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), + 'multiple' => false, + 'schemes' => ['redis'], + ], + 'cache' => [ + 'type' => 'cache', + 'dsns' => App::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), + 'multiple' => true, + 'schemes' => ['redis'], + ], + ]; + + $maxConnections = App::getEnv('_APP_CONNECTIONS_MAX', 151); + $instanceConnections = $maxConnections / App::getEnv('_APP_POOL_CLIENTS', 14); + + $multiprocessing = App::getEnv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled'; + + if ($multiprocessing) { + $workerCount = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)); + } else { + $workerCount = 1; + } + + if ($workerCount > $instanceConnections) { + throw new \Exception('Pool size is too small. Increase the number of allowed database connections or decrease the number of workers.', 500); + } + + $poolSize = (int)($instanceConnections / $workerCount); + + foreach ($connections as $key => $connection) { + $type = $connection['type'] ?? ''; + $dsns = $connection['dsns'] ?? ''; + $multipe = $connection['multiple'] ?? false; + $schemes = $connection['schemes'] ?? []; + $config = []; + $dsns = explode(',', $connection['dsns'] ?? ''); + foreach ($dsns as &$dsn) { + $dsn = explode('=', $dsn); + $name = ($multipe) ? $key . '_' . $dsn[0] : $key; + $dsn = $dsn[1] ?? ''; + $config[] = $name; + if (empty($dsn)) { + //throw new Exception(Exception::GENERAL_SERVER_ERROR, "Missing value for DSN connection in {$key}"); + continue; + } + + $dsn = new DSN($dsn); + $dsnHost = $dsn->getHost(); + $dsnPort = $dsn->getPort(); + $dsnUser = $dsn->getUser(); + $dsnPass = $dsn->getPassword(); + $dsnScheme = $dsn->getScheme(); + $dsnDatabase = $dsn->getPath(); + + if (!in_array($dsnScheme, $schemes)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid console database scheme"); + } + + /** + * Get Resource + * + * Creation could be reused accross connection types like database, cache, queue, etc. + * + * Resource assignment to an adapter will happen below. + */ + switch ($dsnScheme) { + case 'mysql': + case 'mariadb': + $resource = function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDOProxy(function () use ($dsnHost, $dsnPort, $dsnUser, $dsnPass, $dsnDatabase) { + return new PDO("mysql:host={$dsnHost};port={$dsnPort};dbname={$dsnDatabase};charset=utf8mb4", $dsnUser, $dsnPass, array( + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => App::isDevelopment() ? PDO::ERRMODE_WARNING : PDO::ERRMODE_SILENT, // If in production mode, warnings are not displayed + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + )); + }); + }; + break; + case 'redis': + $resource = function () use ($dsnHost, $dsnPort, $dsnPass) { + $redis = new Redis(); + @$redis->pconnect($dsnHost, (int)$dsnPort); + if ($dsnPass) { + $redis->auth($dsnPass); + } + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + + return $redis; + }; + break; + + default: + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Invalid scheme"); + break; + } + + $pool = new Pool($name, $poolSize, function () use ($type, $resource, $dsn) { + // Get Adapter + $adapter = null; + switch ($type) { + case 'database': + $adapter = match ($dsn->getScheme()) { + 'mariadb' => new MariaDB($resource()), + 'mysql' => new MySQL($resource()), + default => null + }; + + $adapter->setDefaultDatabase($dsn->getPath()); + break; + case 'pubsub': + $adapter = $resource(); + break; + case 'queue': + $adapter = match ($dsn->getScheme()) { + 'redis' => new Queue\Connection\Redis($dsn->getHost(), $dsn->getPort()), + default => null + }; + break; + case 'cache': + $adapter = match ($dsn->getScheme()) { + 'redis' => new RedisCache($resource()), + default => null + }; + break; + + default: + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Server error: Missing adapter implementation."); + break; + } + + return $adapter; + }); + + $group->add($pool); + } + + Config::setParam('pools-' . $key, $config); + } + + return $group; +}); diff --git a/composer.lock b/composer.lock index 288a17101f..27712b38ac 100644 --- a/composer.lock +++ b/composer.lock @@ -1050,16 +1050,16 @@ }, { "name": "matomo/device-detector", - "version": "6.1.5", + "version": "6.1.6", "source": { "type": "git", "url": "https://github.com/matomo-org/device-detector.git", - "reference": "40ca2990dba2c1719e5c62168e822e0b86c167d4" + "reference": "5cbea85106e561c7138d03603eb6e05128480409" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/matomo-org/device-detector/zipball/40ca2990dba2c1719e5c62168e822e0b86c167d4", - "reference": "40ca2990dba2c1719e5c62168e822e0b86c167d4", + "url": "https://api.github.com/repos/matomo-org/device-detector/zipball/5cbea85106e561c7138d03603eb6e05128480409", + "reference": "5cbea85106e561c7138d03603eb6e05128480409", "shasum": "" }, "require": { @@ -1115,7 +1115,7 @@ "source": "https://github.com/matomo-org/matomo", "wiki": "https://dev.matomo.org/" }, - "time": "2023-08-17T16:17:41+00:00" + "time": "2023-10-02T10:01:54+00:00" }, { "name": "mongodb/mongodb", @@ -2152,16 +2152,16 @@ }, { "name": "utopia-php/database", - "version": "0.43.4", + "version": "0.43.5", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "cabdd02e8dc1732eb0b22007c511e7bb3caa5c8c" + "reference": "5f7b05189cfbcc0506090498c580c5765375a00a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/cabdd02e8dc1732eb0b22007c511e7bb3caa5c8c", - "reference": "cabdd02e8dc1732eb0b22007c511e7bb3caa5c8c", + "url": "https://api.github.com/repos/utopia-php/database/zipball/5f7b05189cfbcc0506090498c580c5765375a00a", + "reference": "5f7b05189cfbcc0506090498c580c5765375a00a", "shasum": "" }, "require": { @@ -2202,9 +2202,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.43.4" + "source": "https://github.com/utopia-php/database/tree/0.43.5" }, - "time": "2023-09-28T09:00:05+00:00" + "time": "2023-10-06T06:49:47+00:00" }, { "name": "utopia-php/domains", diff --git a/package 2.json b/package 2.json new file mode 100644 index 0000000000..6e32c7d515 --- /dev/null +++ b/package 2.json @@ -0,0 +1,8 @@ +{ + "private": true, + "name": "@appwrite.io/repo", + "repository": { + "type": "git", + "url": "git+https://github.com/appwrite/appwrite.git" + } +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000000..6ab33b14fe --- /dev/null +++ b/package-lock.json @@ -0,0 +1,10 @@ +{ + "name": "@appwrite.io/repo", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@appwrite.io/repo" + } + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000000..2b9f1883a1 --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,5 @@ +lockfileVersion: '6.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false