From ee05f3d3bc449e670f21fd9f28a0de9b7dc07596 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 17 Dec 2024 09:49:31 +0530 Subject: [PATCH] feat: migrate to redis abuse --- .editorconfig | 0 .env | 2 +- app/controllers/api/users.php | 1 + app/controllers/shared/api.php | 6 ++--- app/init.php | 32 +++++++++++------------ app/realtime.php | 27 ++++++++++++++++--- src/Appwrite/Platform/Workers/Deletes.php | 14 +++++----- 7 files changed, 52 insertions(+), 30 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.env b/.env index 2470c786b1..6145411c94 100644 --- a/.env +++ b/.env @@ -15,7 +15,7 @@ _APP_SYSTEM_TEAM_EMAIL=team@appwrite.io _APP_EMAIL_SECURITY=security@appwrite.io _APP_EMAIL_CERTIFICATES=certificates@appwrite.io _APP_SYSTEM_RESPONSE_FORMAT= -_APP_OPTIONS_ABUSE=disabled +_APP_OPTIONS_ABUSE=enabled _APP_OPTIONS_ROUTER_PROTECTION=disabled _APP_OPTIONS_FORCE_HTTPS=disabled _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS=disabled diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index bdb24572eb..1e65c84d7a 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -188,6 +188,7 @@ App::post('/v1/users') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'create') + ->label('abuse-limit', 10) ->label('sdk.description', '/docs/references/users/create-user.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index ecac7507ba..c13d7e0684 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -422,7 +422,7 @@ App::init() ->inject('dbForProject') ->inject('redis') ->inject('mode') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, Redis $redis, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, \Redis $redis, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -487,8 +487,8 @@ App::init() if ( $enabled // Abuse is enabled - && !$isAppUser // User is not API key - && !$isPrivilegedUser // User is not an admin + // && !$isAppUser // User is not API key + // && !$isPrivilegedUser // User is not an admin && $abuse->check() // Route is rate-limited ) { throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED); diff --git a/app/init.php b/app/init.php index 8d478d1949..a9888e29f5 100644 --- a/app/init.php +++ b/app/init.php @@ -853,31 +853,31 @@ $register->set('pools', function () { $connections = [ 'console' => [ 'type' => 'database', - 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), + 'dsns' => $fallbackForDB, 'multiple' => false, 'schemes' => ['mariadb', 'mysql'], ], 'database' => [ 'type' => 'database', - 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), + 'dsns' => $fallbackForDB, 'multiple' => true, 'schemes' => ['mariadb', 'mysql'], ], 'queue' => [ 'type' => 'queue', - 'dsns' => System::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), + 'dsns' => $fallbackForRedis, 'multiple' => false, 'schemes' => ['redis'], ], 'pubsub' => [ 'type' => 'pubsub', - 'dsns' => System::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), + 'dsns' => $fallbackForRedis, 'multiple' => false, 'schemes' => ['redis'], ], 'cache' => [ 'type' => 'cache', - 'dsns' => System::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), + 'dsns' => $fallbackForRedis, 'multiple' => true, 'schemes' => ['redis'], ], @@ -949,12 +949,12 @@ $register->set('pools', function () { }); }, 'redis' => function () use ($dsnHost, $dsnPort, $dsnPass) { - $redis = new Redis(); + $redis = new \Redis(); @$redis->pconnect($dsnHost, (int)$dsnPort); if ($dsnPass) { $redis->auth($dsnPass); } - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); return $redis; }, @@ -1532,18 +1532,18 @@ App::setResource('cache', function (Group $pools) { }, ['pools']); App::setResource('redis', function (Group $pools) { - $list = Config::getParam('pools-cache', []); - $adapters = []; + $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); + $port = System::getEnv('_APP_REDIS_PORT', 6379); + $pass = System::getEnv('_APP_REDIS_PASS', ''); - foreach ($list as $value) { - $adapters[] = $pools - ->get($value) - ->pop() - ->getResource() - ; + $redis = new \Redis(); + @$redis->pconnect($host, (int)$port); + if ($pass) { + $redis->auth($pass); } + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); - return new Sharding($adapters); + return $redis; }, ['pools']); App::setResource('deviceForLocal', function () { diff --git a/app/realtime.php b/app/realtime.php index 97937c44fc..f55ae30f3e 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -138,6 +138,26 @@ if (!function_exists('getCache')) { } } +// Allows overriding +if (!function_exists('getRedis')) { + function getRedis(): \Redis + { + + $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); + $port = System::getEnv('_APP_REDIS_PORT', 6379); + $pass = System::getEnv('_APP_REDIS_PASS', ''); + + $redis = new \Redis(); + @$redis->pconnect($host, (int)$port); + if ($pass) { + $redis->auth($pass); + } + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); + + return $redis; + } +} + if (!function_exists('getRealtime')) { function getRealtime(): Realtime { @@ -481,7 +501,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new AppwriteException(AppwriteException::GENERAL_API_DISABLED); } - $cache = $app->getResource('cache'); + $redis = $app->getResource('redis'); $console = $app->getResource('console'); /** @var Document $console */ $user = $app->getResource('user'); /** @var Document $user */ @@ -490,7 +510,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, * * Abuse limits are connecting 128 times per minute and ip address. */ - $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $cache); + $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $redis); $timeLimit ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getURI()); @@ -593,7 +613,8 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re * * Abuse limits are sending 32 times per minute and connection. */ - $timeLimit = new TimeLimit('url:{url},connection:{connection}', 32, 60, $cache); + $redis = getRedis(); + $timeLimit = new TimeLimit('url:{url},connection:{connection}', 32, 60, $redis); $timeLimit ->setParam('{connection}', $connection) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index b1c8c90051..e217ebbf7b 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -47,6 +47,7 @@ class Deletes extends Action ->inject('message') ->inject('dbForConsole') ->inject('getProjectDB') + ->inject('redis') ->inject('deviceForFiles') ->inject('deviceForFunctions') ->inject('deviceForBuilds') @@ -57,8 +58,8 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) + fn ($message, $dbForConsole, callable $getProjectDB, \Redis $redis, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForConsole, $getProjectDB, $redis, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +67,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForConsole, callable $getProjectDB, \Redis $redis, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -126,7 +127,7 @@ class Deletes extends Action } break; case DELETE_TYPE_ABUSE: - $this->deleteAbuseLogs($project, $getProjectDB, $abuseRetention); + $this->deleteAbuseLogs($project, $redis, $abuseRetention); break; case DELETE_TYPE_REALTIME: $this->deleteRealtimeUsage($dbForConsole, $datetime); @@ -707,11 +708,10 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteAbuseLogs(Document $project, callable $getProjectDB, string $abuseRetention): void + private function deleteAbuseLogs(Document $project, \Redis $redis, string $abuseRetention): void { $projectId = $project->getId(); - $dbForProject = $getProjectDB($project); - $timeLimit = new TimeLimit("", 0, 1, $cache); + $timeLimit = new TimeLimit("", 0, 1, $redis); $abuse = new Abuse($timeLimit); try {