From 3f620bbc449d3b9683e17feea2ec0a94c3cb574a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 13 Jan 2025 18:26:54 +1300 Subject: [PATCH] Merge pull request #9213 from appwrite/fix-dead-connections Fix dead connections --- app/controllers/general.php | 14 ++++++++++- app/http.php | 11 ++++++++ app/init.php | 31 +++++++++++++++++------ app/worker.php | 50 +++++++++++++++++++++++-------------- 4 files changed, 78 insertions(+), 28 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 6f8566f69b..4467bd1b47 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -21,6 +21,7 @@ use Appwrite\Utopia\Response\Filters\V18 as ResponseV18; use Appwrite\Utopia\View; use Executor\Executor; use MaxMind\Db\Reader; +use Swoole\Database\DetectsLostConnections; use Swoole\Http\Request as SwooleRequest; use Utopia\App; use Utopia\CLI\Console; @@ -28,6 +29,7 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; @@ -38,6 +40,7 @@ use Utopia\Logger\Adapter\Sentry; use Utopia\Logger\Log; use Utopia\Logger\Log\User; use Utopia\Logger\Logger; +use Utopia\Pools\Connection; use Utopia\System\System; use Utopia\Validator\Hostname; use Utopia\Validator\Text; @@ -746,7 +749,16 @@ App::error() ->inject('logger') ->inject('log') ->inject('queueForUsage') - ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage) { + ->inject('connectionForProject') + ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage, Connection $connectionForProject) { + if ( + ($error instanceof PDOException || $error instanceof DatabaseException) + && DetectsLostConnections::causedByLostConnection($error) + ) { + // Mark connection as unhealthy so it will be recycled on next reclaim. + $connectionForProject->setHealthy(false); + } + $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); $class = \get_class($error); diff --git a/app/http.php b/app/http.php index 3a7562ffd1..b988c7d8ac 100644 --- a/app/http.php +++ b/app/http.php @@ -5,6 +5,7 @@ require_once __DIR__ . '/../vendor/autoload.php'; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Swoole\Constant; +use Swoole\Database\DetectsLostConnections; use Swoole\Http\Request as SwooleRequest; use Swoole\Http\Response as SwooleResponse; use Swoole\Http\Server; @@ -17,6 +18,7 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Duplicate; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -346,6 +348,15 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool $app->run($request, $response); } catch (\Throwable $th) { + if ( + ($th instanceof PDOException || $th instanceof DatabaseException) + && DetectsLostConnections::causedByLostConnection($th) + ) { + // Mark connection as unhealthy so it will be recycled on next reclaim. + $connectionForProject = $app->getResource('connectionForProject'); + $connectionForProject->setHealthy(false); + } + $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $logger = $app->getResource("logger"); diff --git a/app/init.php b/app/init.php index df2a66aa00..7f50e5d2c3 100644 --- a/app/init.php +++ b/app/init.php @@ -74,6 +74,7 @@ use Utopia\Logger\Adapter\Raygun; use Utopia\Logger\Adapter\Sentry; use Utopia\Logger\Log; use Utopia\Logger\Logger; +use Utopia\Pools\Connection as PoolConnection; use Utopia\Pools\Group; use Utopia\Pools\Pool; use Utopia\Queue; @@ -1418,7 +1419,26 @@ App::setResource('console', function () { ]); }, []); -App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project) { +App::setResource('connectionForProject', function (Group $pools, Document $project) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $pools + ->get('console') + ->pop(); + } + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + return $pools + ->get($dsn->getHost()) + ->pop(); +}, ['pools', 'project']); + +App::setResource('dbForProject', function (Group $pools, PoolConnection $connectionForProject, Database $dbForPlatform, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -1430,12 +1450,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform $dsn = new DSN('mysql://' . $project->getAttribute('database')); } - $dbAdapter = $pools - ->get($dsn->getHost()) - ->pop() - ->getResource(); - - $database = new Database($dbAdapter, $cache); + $database = new Database($connectionForProject->getResource(), $cache); $database ->setMetadata('host', \gethostname()) @@ -1458,7 +1473,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform } return $database; -}, ['pools', 'dbForPlatform', 'cache', 'project']); +}, ['pools', 'connectionForProject', 'dbForPlatform', 'cache', 'project']); App::setResource('dbForPlatform', function (Group $pools, Cache $cache) { $dbAdapter = $pools diff --git a/app/worker.php b/app/worker.php index 6eb1363e9b..62882b32b1 100644 --- a/app/worker.php +++ b/app/worker.php @@ -16,6 +16,7 @@ use Appwrite\Event\Migration; use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; +use Swoole\Database\DetectsLostConnections; use Swoole\Runtime; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\Cache\Adapter\Sharding; @@ -25,11 +26,13 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Validator\Authorization; use Utopia\DSN\DSN; use Utopia\Logger\Log; use Utopia\Logger\Logger; use Utopia\Platform\Service; +use Utopia\Pools\Connection as PoolConnection; use Utopia\Pools\Group; use Utopia\Queue\Connection; use Utopia\Queue\Message; @@ -66,13 +69,13 @@ Server::setResource('project', function (Message $message, Database $dbForPlatfo return $dbForPlatform->getDocument('projects', $project->getId()); }, ['message', 'dbForPlatform']); -Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { +Server::setResource('connectionForProject', function (Group $pools, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForPlatform; + return $pools + ->get('console') + ->pop(); } - $pools = $register->get('pools'); - try { $dsn = new DSN($project->getAttribute('database')); } catch (\InvalidArgumentException) { @@ -80,12 +83,17 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, $dsn = new DSN('mysql://' . $project->getAttribute('database')); } - $adapter = $pools + return $pools ->get($dsn->getHost()) - ->pop() - ->getResource(); + ->pop(); +}, ['pools', 'project']); - $database = new Database($adapter, $cache); +Server::setResource('dbForProject', function (PoolConnection $connectionForProject, Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForPlatform; + } + + $database = new Database($connectionForProject->getResource(), $cache); try { $dsn = new DSN($project->getAttribute('database')); @@ -109,12 +117,12 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, } return $database; -}, ['cache', 'register', 'message', 'project', 'dbForPlatform']); +}, ['connectionForProject', 'cache', 'register', 'message', 'project', 'dbForPlatform']); -Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { +Server::setResource('getProjectDB', function (Group $pools, PoolConnection $connectionForProject, Database $dbForPlatform, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - return function (Document $project) use ($pools, $dbForPlatform, $cache, &$databases): Database { + return function (Document $project) use ($pools, $connectionForProject, $dbForPlatform, $cache, &$databases): Database { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -146,12 +154,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf return $database; } - $dbAdapter = $pools - ->get($dsn->getHost()) - ->pop() - ->getResource(); - - $database = new Database($dbAdapter, $cache); + $database = new Database($connectionForProject->getResource(), $cache); $databases[$dsn->getHost()] = $database; @@ -171,7 +174,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf return $database; }; -}, ['pools', 'dbForPlatform', 'cache']); +}, ['pools', 'connectionForProject', 'dbForPlatform', 'cache']); Server::setResource('abuseRetention', function () { return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); @@ -410,7 +413,16 @@ $worker ->inject('log') ->inject('pools') ->inject('project') - ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project) use ($queueName) { + ->inject('connectionForProject') + ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project, PoolConnection $connectionForProject) use ($queueName) { + if ( + ($error instanceof PDOException || $error instanceof DatabaseException) + && DetectsLostConnections::causedByLostConnection($error) + ) { + // Mark connection as unhealthy, it will be recycled on next reclaim. + $connectionForProject->setHealthy(false); + } + $pools->reclaim(); $version = System::getEnv('_APP_VERSION', 'UNKNOWN');