From a12b758658e32d01df7e1a2c5fa54b58b5855bce Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:11:22 +0000 Subject: [PATCH 01/41] feat: upgrade assistant to 0.6,0 --- app/controllers/api/console.php | 8 ++++++-- docker-compose.yml | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/console.php b/app/controllers/api/console.php index eeb823a3d3..b7c7c05641 100644 --- a/app/controllers/api/console.php +++ b/app/controllers/api/console.php @@ -69,11 +69,15 @@ App::post('/v1/console/assistant') ->label('abuse-limit', 15) ->label('abuse-key', 'userId:{userId}') ->param('prompt', '', new Text(2000), 'Prompt. A string containing questions asked to the AI assistant.') + ->param('systemPrompt', '', new Text(2000), 'System Prompt. A string containing context to help the AI assistant provide better responses.') ->inject('response') - ->action(function (string $prompt, Response $response) { + ->action(function (string $prompt, string $systemPrompt, Response $response) { $ch = curl_init('http://appwrite-assistant:3003/'); $responseHeaders = []; - $query = json_encode(['prompt' => $prompt]); + $query = json_encode([ + 'prompt' => $prompt, + 'systemPrompt' => $systemPrompt, + ]); $headers = ['accept: text/event-stream']; $handleEvent = function ($ch, $data) use ($response) { $response->chunk($data); diff --git a/docker-compose.yml b/docker-compose.yml index f9a79ee84d..9652767a1f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -868,7 +868,7 @@ services: appwrite-assistant: container_name: appwrite-assistant - image: appwrite/assistant:0.5.0 + image: appwrite/assistant:0.6.0 networks: - appwrite environment: From bc38fd26407931d0a6a34c394569039830cc718d Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:04:31 +0000 Subject: [PATCH 02/41] fix: hide system prompt --- app/controllers/api/console.php | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/console.php b/app/controllers/api/console.php index b7c7c05641..eeb823a3d3 100644 --- a/app/controllers/api/console.php +++ b/app/controllers/api/console.php @@ -69,15 +69,11 @@ App::post('/v1/console/assistant') ->label('abuse-limit', 15) ->label('abuse-key', 'userId:{userId}') ->param('prompt', '', new Text(2000), 'Prompt. A string containing questions asked to the AI assistant.') - ->param('systemPrompt', '', new Text(2000), 'System Prompt. A string containing context to help the AI assistant provide better responses.') ->inject('response') - ->action(function (string $prompt, string $systemPrompt, Response $response) { + ->action(function (string $prompt, Response $response) { $ch = curl_init('http://appwrite-assistant:3003/'); $responseHeaders = []; - $query = json_encode([ - 'prompt' => $prompt, - 'systemPrompt' => $systemPrompt, - ]); + $query = json_encode(['prompt' => $prompt]); $headers = ['accept: text/event-stream']; $handleEvent = function ($ch, $data) use ($response) { $response->chunk($data); From 515b5904b849136ff755a4c936cb328c1834e4f6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 10 Dec 2024 11:34:51 +0000 Subject: [PATCH 03/41] update project accessed at from router --- app/controllers/general.php | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/controllers/general.php b/app/controllers/general.php index 32cc68fecf..2c43116c10 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -91,6 +91,15 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw $project = Authorization::skip( fn () => $dbForPlatform->getDocument('projects', $projectId) ); + + if (!$project->isEmpty() && $project->getId() !== 'console') { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); + } + } + if (array_key_exists('proxy', $project->getAttribute('services', []))) { $status = $project->getAttribute('services', [])['proxy']; if (!$status) { From ab4a4441545ac30517610797e1adc0274103f195 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 10 Dec 2024 11:13:09 +0000 Subject: [PATCH 04/41] Fix: project accessed at on workers --- app/worker.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/app/worker.php b/app/worker.php index 2c7d8acb22..6fdb49d35a 100644 --- a/app/worker.php +++ b/app/worker.php @@ -374,6 +374,20 @@ try { $worker = $platform->getWorker(); +$worker + ->init() + ->inject('project') + ->inject('dbForConsole') + ->action(function (Document $project, Database $dbForConsole) { + if (!$project->isEmpty() && $project->getId() !== 'console') { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); + } + } + }); + $worker ->shutdown() ->inject('pools') From a24dc7903648671e0a3d515dde9bd9cec85e4581 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:36:02 +0000 Subject: [PATCH 05/41] chore: fix endpoint --- app/controllers/api/console.php | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/console.php b/app/controllers/api/console.php index eeb823a3d3..1b307b1d14 100644 --- a/app/controllers/api/console.php +++ b/app/controllers/api/console.php @@ -71,7 +71,7 @@ App::post('/v1/console/assistant') ->param('prompt', '', new Text(2000), 'Prompt. A string containing questions asked to the AI assistant.') ->inject('response') ->action(function (string $prompt, Response $response) { - $ch = curl_init('http://appwrite-assistant:3003/'); + $ch = curl_init('http://appwrite-assistant:3003/v1/models/assistant/prompt'); $responseHeaders = []; $query = json_encode(['prompt' => $prompt]); $headers = ['accept: text/event-stream']; diff --git a/docker-compose.yml b/docker-compose.yml index 3bdbd969b3..2fb19f7126 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -869,7 +869,7 @@ services: appwrite-assistant: container_name: appwrite-assistant - image: appwrite/assistant:0.6.0 + image: appwrite/assistant:0.7.0 networks: - appwrite environment: From 396faa56c2cc58b6aee2ffebb412eb25535da716 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 17 Dec 2024 08:21:38 +0530 Subject: [PATCH 06/41] chore: initial commit --- app/controllers/api/projects.php | 16 +--- app/controllers/shared/api.php | 7 +- app/http.php | 7 -- app/init.php | 15 ++++ app/realtime.php | 8 +- composer.lock | 90 +++++++++++------------ src/Appwrite/Platform/Workers/Deletes.php | 7 +- 7 files changed, 72 insertions(+), 78 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 92d3103293..06723f4583 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -16,7 +16,7 @@ use Appwrite\Utopia\Database\Validator\Queries\Projects; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use PHPMailer\PHPMailer\PHPMailer; -use Utopia\Abuse\Adapters\Database\TimeLimit; +use Utopia\Abuse\Adapters\Redis\TimeLimit; use Utopia\App; use Utopia\Audit\Audit; use Utopia\Cache\Cache; @@ -229,9 +229,6 @@ App::post('/v1/projects') if ($create || $projectTables) { $audit = new Audit($dbForProject); $audit->setup(); - - $abuse = new TimeLimit('', 0, 1, $dbForProject); - $abuse->setup(); } if (!$create && $sharedTablesV1) { @@ -245,17 +242,6 @@ App::post('/v1/projects') 'indexes' => $indexes, 'documentSecurity' => true ])); - - $attributes = \array_map(fn ($attribute) => new Document($attribute), TimeLimit::ATTRIBUTES); - $indexes = \array_map(fn (array $index) => new Document($index), TimeLimit::INDEXES); - $dbForProject->createDocument(Database::METADATA, new Document([ - '$id' => ID::custom('abuse'), - '$permissions' => [Permission::create(Role::any())], - 'name' => 'abuse', - 'attributes' => $attributes, - 'indexes' => $indexes, - 'documentSecurity' => true - ])); } if ($create || $sharedTablesV1) { diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 91883b0895..ecac7507ba 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -19,7 +19,7 @@ use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\Database\TimeLimit; +use Utopia\Abuse\Adapters\Redis\TimeLimit; use Utopia\App; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; @@ -420,8 +420,9 @@ App::init() ->inject('queueForBuilds') ->inject('queueForUsage') ->inject('dbForProject') + ->inject('redis') ->inject('mode') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, Redis $redis, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -444,7 +445,7 @@ App::init() foreach ($abuseKeyLabel as $abuseKey) { $start = $request->getContentRangeStart(); $end = $request->getContentRangeEnd(); - $timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $dbForProject); + $timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $redis); $timeLimit ->setParam('{projectId}', $project->getId()) ->setParam('{userId}', $user->getId()) diff --git a/app/http.php b/app/http.php index c4e48a7f69..22ac72227e 100644 --- a/app/http.php +++ b/app/http.php @@ -10,10 +10,8 @@ use Swoole\Http\Response as SwooleResponse; use Swoole\Http\Server; use Swoole\Process; use Swoole\Table; -use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\App; use Utopia\Audit\Audit; -use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; @@ -200,11 +198,6 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg $audit->setup(); } - if ($dbForConsole->getCollection(TimeLimit::COLLECTION)->isEmpty()) { - $adapter = new TimeLimit("", 0, 1, $dbForConsole); - $adapter->setup(); - } - /** @var array $collections */ $collections = Config::getParam('collections', []); $consoleCollections = $collections['console']; diff --git a/app/init.php b/app/init.php index 30ece74553..8d478d1949 100644 --- a/app/init.php +++ b/app/init.php @@ -1531,6 +1531,21 @@ App::setResource('cache', function (Group $pools) { return new Cache(new Sharding($adapters)); }, ['pools']); +App::setResource('redis', function (Group $pools) { + $list = Config::getParam('pools-cache', []); + $adapters = []; + + foreach ($list as $value) { + $adapters[] = $pools + ->get($value) + ->pop() + ->getResource() + ; + } + + return new Sharding($adapters); +}, ['pools']); + App::setResource('deviceForLocal', function () { return new Local(); }); diff --git a/app/realtime.php b/app/realtime.php index 03eaa53bcc..97937c44fc 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -13,7 +13,7 @@ use Swoole\Runtime; use Swoole\Table; use Swoole\Timer; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\Database\TimeLimit; +use Utopia\Abuse\Adapters\Redis\TimeLimit; use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; @@ -481,7 +481,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new AppwriteException(AppwriteException::GENERAL_API_DISABLED); } - $dbForProject = getProjectDB($project); + $cache = $app->getResource('cache'); $console = $app->getResource('console'); /** @var Document $console */ $user = $app->getResource('user'); /** @var Document $user */ @@ -490,7 +490,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, * * Abuse limits are connecting 128 times per minute and ip address. */ - $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $dbForProject); + $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $cache); $timeLimit ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getURI()); @@ -593,7 +593,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re * * Abuse limits are sending 32 times per minute and connection. */ - $timeLimit = new TimeLimit('url:{url},connection:{connection}', 32, 60, $database); + $timeLimit = new TimeLimit('url:{url},connection:{connection}', 32, 60, $cache); $timeLimit ->setParam('{connection}', $connection) diff --git a/composer.lock b/composer.lock index ee947d27f0..37a5c81436 100644 --- a/composer.lock +++ b/composer.lock @@ -1237,16 +1237,16 @@ }, { "name": "open-telemetry/api", - "version": "1.1.1", + "version": "1.1.2", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "542064815d38a6df55af7957cd6f1d7d967c99c6" + "reference": "04c85a1e41a3d59fa9bdc801a5de1df6624b95ed" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/542064815d38a6df55af7957cd6f1d7d967c99c6", - "reference": "542064815d38a6df55af7957cd6f1d7d967c99c6", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/04c85a1e41a3d59fa9bdc801a5de1df6624b95ed", + "reference": "04c85a1e41a3d59fa9bdc801a5de1df6624b95ed", "shasum": "" }, "require": { @@ -1260,13 +1260,13 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "1.1.x-dev" - }, "spi": { "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\HookManagerInterface": [ "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\ExtensionHookManager" ] + }, + "branch-alias": { + "dev-main": "1.1.x-dev" } }, "autoload": { @@ -1303,7 +1303,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2024-10-15T22:42:37+00:00" + "time": "2024-11-16T04:32:30+00:00" }, { "name": "open-telemetry/context", @@ -1530,13 +1530,13 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "1.0.x-dev" - }, "spi": { "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\HookManagerInterface": [ "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\ExtensionHookManager" ] + }, + "branch-alias": { + "dev-main": "1.0.x-dev" } }, "autoload": { @@ -2453,23 +2453,23 @@ }, { "name": "symfony/http-client", - "version": "v7.2.0", + "version": "v7.2.1", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "955e43336aff03df1e8a8e17daefabb0127a313b" + "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/955e43336aff03df1e8a8e17daefabb0127a313b", - "reference": "955e43336aff03df1e8a8e17daefabb0127a313b", + "url": "https://api.github.com/repos/symfony/http-client/zipball/ff4df2b68d1c67abb9fef146e6540ea16b58d99e", + "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e", "shasum": "" }, "require": { "php": ">=8.2", "psr/log": "^1|^2|^3", "symfony/deprecation-contracts": "^2.5|^3", - "symfony/http-client-contracts": "~3.4.3|^3.5.1", + "symfony/http-client-contracts": "~3.4.4|^3.5.2", "symfony/service-contracts": "^2.5|^3" }, "conflict": { @@ -2528,7 +2528,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.2.0" + "source": "https://github.com/symfony/http-client/tree/v7.2.1" }, "funding": [ { @@ -2544,20 +2544,20 @@ "type": "tidelift" } ], - "time": "2024-11-29T08:22:02+00:00" + "time": "2024-12-07T08:50:44+00:00" }, { "name": "symfony/http-client-contracts", - "version": "v3.5.1", + "version": "v3.5.2", "source": { "type": "git", "url": "https://github.com/symfony/http-client-contracts.git", - "reference": "c2f3ad828596624ca39ea40f83617ef51ca8bbf9" + "reference": "ee8d807ab20fcb51267fdace50fbe3494c31e645" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/c2f3ad828596624ca39ea40f83617ef51ca8bbf9", - "reference": "c2f3ad828596624ca39ea40f83617ef51ca8bbf9", + "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/ee8d807ab20fcb51267fdace50fbe3494c31e645", + "reference": "ee8d807ab20fcb51267fdace50fbe3494c31e645", "shasum": "" }, "require": { @@ -2606,7 +2606,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/http-client-contracts/tree/v3.5.1" + "source": "https://github.com/symfony/http-client-contracts/tree/v3.5.2" }, "funding": [ { @@ -2622,7 +2622,7 @@ "type": "tidelift" } ], - "time": "2024-11-25T12:02:18+00:00" + "time": "2024-12-07T08:49:48+00:00" }, { "name": "symfony/polyfill-mbstring", @@ -2650,8 +2650,8 @@ "type": "library", "extra": { "thanks": { - "name": "symfony/polyfill", - "url": "https://github.com/symfony/polyfill" + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" } }, "autoload": { @@ -3136,16 +3136,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.43.1", + "version": "0.43.2", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "e404c21e8dcf6a310bc83cf1d74e716b105598fa" + "reference": "374536b86d8d39066960a7da161d444a099bbc56" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/e404c21e8dcf6a310bc83cf1d74e716b105598fa", - "reference": "e404c21e8dcf6a310bc83cf1d74e716b105598fa", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/374536b86d8d39066960a7da161d444a099bbc56", + "reference": "374536b86d8d39066960a7da161d444a099bbc56", "shasum": "" }, "require": { @@ -3153,7 +3153,7 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/database": "0.53.*" + "utopia-php/database": "0.53.200" }, "require-dev": { "laravel/pint": "1.5.*", @@ -3181,9 +3181,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.43.1" + "source": "https://github.com/utopia-php/abuse/tree/0.43.2" }, - "time": "2024-10-23T04:29:12+00:00" + "time": "2024-12-12T19:43:24+00:00" }, { "name": "utopia-php/analytics", @@ -4807,16 +4807,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.39.25", + "version": "0.39.26", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "5b5323636a8d75a1c4faaae9728098dd6a6a47d1" + "reference": "39768deacb4913f93548c46fa0149c3fadc62d0c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/5b5323636a8d75a1c4faaae9728098dd6a6a47d1", - "reference": "5b5323636a8d75a1c4faaae9728098dd6a6a47d1", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/39768deacb4913f93548c46fa0149c3fadc62d0c", + "reference": "39768deacb4913f93548c46fa0149c3fadc62d0c", "shasum": "" }, "require": { @@ -4852,9 +4852,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.39.25" + "source": "https://github.com/appwrite/sdk-generator/tree/0.39.26" }, - "time": "2024-11-08T10:16:34+00:00" + "time": "2024-12-12T10:51:34+00:00" }, { "name": "doctrine/annotations", @@ -7576,16 +7576,16 @@ }, { "name": "symfony/console", - "version": "v7.2.0", + "version": "v7.2.1", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "23c8aae6d764e2bae02d2a99f7532a7f6ed619cf" + "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/23c8aae6d764e2bae02d2a99f7532a7f6ed619cf", - "reference": "23c8aae6d764e2bae02d2a99f7532a7f6ed619cf", + "url": "https://api.github.com/repos/symfony/console/zipball/fefcc18c0f5d0efe3ab3152f15857298868dc2c3", + "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3", "shasum": "" }, "require": { @@ -7649,7 +7649,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.2.0" + "source": "https://github.com/symfony/console/tree/v7.2.1" }, "funding": [ { @@ -7665,7 +7665,7 @@ "type": "tidelift" } ], - "time": "2024-11-06T14:24:19+00:00" + "time": "2024-12-11T03:49:26+00:00" }, { "name": "symfony/filesystem", diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index e6002bf75f..b1c8c90051 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -8,7 +8,7 @@ use Appwrite\Extend\Exception; use Executor\Executor; use Throwable; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\Database\TimeLimit; +use Utopia\Abuse\Adapters\Redis\TimeLimit; use Utopia\Audit\Audit; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; @@ -494,8 +494,7 @@ class Deletes extends Action $projectCollectionIds = [ ...\array_keys(Config::getParam('collections', [])['projects']), - Audit::COLLECTION, - TimeLimit::COLLECTION, + Audit::COLLECTION ]; $limit = \count($projectCollectionIds) + 25; @@ -712,7 +711,7 @@ class Deletes extends Action { $projectId = $project->getId(); $dbForProject = $getProjectDB($project); - $timeLimit = new TimeLimit("", 0, 1, $dbForProject); + $timeLimit = new TimeLimit("", 0, 1, $cache); $abuse = new Abuse($timeLimit); try { From ee05f3d3bc449e670f21fd9f28a0de9b7dc07596 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 17 Dec 2024 09:49:31 +0530 Subject: [PATCH 07/41] feat: migrate to redis abuse --- .editorconfig | 0 .env | 2 +- app/controllers/api/users.php | 1 + app/controllers/shared/api.php | 6 ++--- app/init.php | 32 +++++++++++------------ app/realtime.php | 27 ++++++++++++++++--- src/Appwrite/Platform/Workers/Deletes.php | 14 +++++----- 7 files changed, 52 insertions(+), 30 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.env b/.env index 2470c786b1..6145411c94 100644 --- a/.env +++ b/.env @@ -15,7 +15,7 @@ _APP_SYSTEM_TEAM_EMAIL=team@appwrite.io _APP_EMAIL_SECURITY=security@appwrite.io _APP_EMAIL_CERTIFICATES=certificates@appwrite.io _APP_SYSTEM_RESPONSE_FORMAT= -_APP_OPTIONS_ABUSE=disabled +_APP_OPTIONS_ABUSE=enabled _APP_OPTIONS_ROUTER_PROTECTION=disabled _APP_OPTIONS_FORCE_HTTPS=disabled _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS=disabled diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index bdb24572eb..1e65c84d7a 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -188,6 +188,7 @@ App::post('/v1/users') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'create') + ->label('abuse-limit', 10) ->label('sdk.description', '/docs/references/users/create-user.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index ecac7507ba..c13d7e0684 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -422,7 +422,7 @@ App::init() ->inject('dbForProject') ->inject('redis') ->inject('mode') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, Redis $redis, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, \Redis $redis, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -487,8 +487,8 @@ App::init() if ( $enabled // Abuse is enabled - && !$isAppUser // User is not API key - && !$isPrivilegedUser // User is not an admin + // && !$isAppUser // User is not API key + // && !$isPrivilegedUser // User is not an admin && $abuse->check() // Route is rate-limited ) { throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED); diff --git a/app/init.php b/app/init.php index 8d478d1949..a9888e29f5 100644 --- a/app/init.php +++ b/app/init.php @@ -853,31 +853,31 @@ $register->set('pools', function () { $connections = [ 'console' => [ 'type' => 'database', - 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), + 'dsns' => $fallbackForDB, 'multiple' => false, 'schemes' => ['mariadb', 'mysql'], ], 'database' => [ 'type' => 'database', - 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), + 'dsns' => $fallbackForDB, 'multiple' => true, 'schemes' => ['mariadb', 'mysql'], ], 'queue' => [ 'type' => 'queue', - 'dsns' => System::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), + 'dsns' => $fallbackForRedis, 'multiple' => false, 'schemes' => ['redis'], ], 'pubsub' => [ 'type' => 'pubsub', - 'dsns' => System::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), + 'dsns' => $fallbackForRedis, 'multiple' => false, 'schemes' => ['redis'], ], 'cache' => [ 'type' => 'cache', - 'dsns' => System::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), + 'dsns' => $fallbackForRedis, 'multiple' => true, 'schemes' => ['redis'], ], @@ -949,12 +949,12 @@ $register->set('pools', function () { }); }, 'redis' => function () use ($dsnHost, $dsnPort, $dsnPass) { - $redis = new Redis(); + $redis = new \Redis(); @$redis->pconnect($dsnHost, (int)$dsnPort); if ($dsnPass) { $redis->auth($dsnPass); } - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); return $redis; }, @@ -1532,18 +1532,18 @@ App::setResource('cache', function (Group $pools) { }, ['pools']); App::setResource('redis', function (Group $pools) { - $list = Config::getParam('pools-cache', []); - $adapters = []; + $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); + $port = System::getEnv('_APP_REDIS_PORT', 6379); + $pass = System::getEnv('_APP_REDIS_PASS', ''); - foreach ($list as $value) { - $adapters[] = $pools - ->get($value) - ->pop() - ->getResource() - ; + $redis = new \Redis(); + @$redis->pconnect($host, (int)$port); + if ($pass) { + $redis->auth($pass); } + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); - return new Sharding($adapters); + return $redis; }, ['pools']); App::setResource('deviceForLocal', function () { diff --git a/app/realtime.php b/app/realtime.php index 97937c44fc..f55ae30f3e 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -138,6 +138,26 @@ if (!function_exists('getCache')) { } } +// Allows overriding +if (!function_exists('getRedis')) { + function getRedis(): \Redis + { + + $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); + $port = System::getEnv('_APP_REDIS_PORT', 6379); + $pass = System::getEnv('_APP_REDIS_PASS', ''); + + $redis = new \Redis(); + @$redis->pconnect($host, (int)$port); + if ($pass) { + $redis->auth($pass); + } + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); + + return $redis; + } +} + if (!function_exists('getRealtime')) { function getRealtime(): Realtime { @@ -481,7 +501,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new AppwriteException(AppwriteException::GENERAL_API_DISABLED); } - $cache = $app->getResource('cache'); + $redis = $app->getResource('redis'); $console = $app->getResource('console'); /** @var Document $console */ $user = $app->getResource('user'); /** @var Document $user */ @@ -490,7 +510,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, * * Abuse limits are connecting 128 times per minute and ip address. */ - $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $cache); + $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $redis); $timeLimit ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getURI()); @@ -593,7 +613,8 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re * * Abuse limits are sending 32 times per minute and connection. */ - $timeLimit = new TimeLimit('url:{url},connection:{connection}', 32, 60, $cache); + $redis = getRedis(); + $timeLimit = new TimeLimit('url:{url},connection:{connection}', 32, 60, $redis); $timeLimit ->setParam('{connection}', $connection) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index b1c8c90051..e217ebbf7b 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -47,6 +47,7 @@ class Deletes extends Action ->inject('message') ->inject('dbForConsole') ->inject('getProjectDB') + ->inject('redis') ->inject('deviceForFiles') ->inject('deviceForFunctions') ->inject('deviceForBuilds') @@ -57,8 +58,8 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) + fn ($message, $dbForConsole, callable $getProjectDB, \Redis $redis, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForConsole, $getProjectDB, $redis, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +67,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForConsole, callable $getProjectDB, \Redis $redis, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -126,7 +127,7 @@ class Deletes extends Action } break; case DELETE_TYPE_ABUSE: - $this->deleteAbuseLogs($project, $getProjectDB, $abuseRetention); + $this->deleteAbuseLogs($project, $redis, $abuseRetention); break; case DELETE_TYPE_REALTIME: $this->deleteRealtimeUsage($dbForConsole, $datetime); @@ -707,11 +708,10 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteAbuseLogs(Document $project, callable $getProjectDB, string $abuseRetention): void + private function deleteAbuseLogs(Document $project, \Redis $redis, string $abuseRetention): void { $projectId = $project->getId(); - $dbForProject = $getProjectDB($project); - $timeLimit = new TimeLimit("", 0, 1, $cache); + $timeLimit = new TimeLimit("", 0, 1, $redis); $abuse = new Abuse($timeLimit); try { From 06c09503df70bb2e5677cf4d390864fc4f718ce0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Dec 2024 04:27:28 +0000 Subject: [PATCH 08/41] update project rsource --- app/worker.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/worker.php b/app/worker.php index 6fdb49d35a..06dc4c44d8 100644 --- a/app/worker.php +++ b/app/worker.php @@ -62,7 +62,13 @@ Server::setResource('project', function (Message $message, Database $dbForPlatfo return $project; } - return $dbForPlatform->getDocument('projects', $project->getId()); + // NOT all workers need valid DB config so we return empty if this fails + try { + return $dbForPlatform->getDocument('projects', $project->getId()); + } catch (\Throwable $e) { + return new Document([]); + } + }, ['message', 'dbForPlatform']); Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { From e8cea7034693f9981af427a38c15876b43084035 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 17 Dec 2024 10:00:28 +0530 Subject: [PATCH 09/41] feat: cleanup --- .editorconfig | 0 app/controllers/api/users.php | 1 - 2 files changed, 1 deletion(-) delete mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 1e65c84d7a..bdb24572eb 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -188,7 +188,6 @@ App::post('/v1/users') ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'create') - ->label('abuse-limit', 10) ->label('sdk.description', '/docs/references/users/create-user.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) From 67996a8a14938b67c0a28f429abe0087d0773af3 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 17 Dec 2024 10:00:45 +0530 Subject: [PATCH 10/41] feat: cleanup --- app/controllers/shared/api.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index c13d7e0684..af2f6a6e7c 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -487,8 +487,8 @@ App::init() if ( $enabled // Abuse is enabled - // && !$isAppUser // User is not API key - // && !$isPrivilegedUser // User is not an admin + && !$isAppUser // User is not API key + && !$isPrivilegedUser // User is not an admin && $abuse->check() // Route is rate-limited ) { throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED); From 801f311faf57315ee5cdd310b5cb35c59f9503c9 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 17 Dec 2024 10:08:16 +0530 Subject: [PATCH 11/41] feat: cleanup --- app/realtime.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/realtime.php b/app/realtime.php index f55ae30f3e..b9ea7c4765 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -142,7 +142,6 @@ if (!function_exists('getCache')) { if (!function_exists('getRedis')) { function getRedis(): \Redis { - $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); $port = System::getEnv('_APP_REDIS_PORT', 6379); $pass = System::getEnv('_APP_REDIS_PASS', ''); From b55a01c5b339d8bf838391571e92e674d86c092c Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 17 Dec 2024 10:12:17 +0530 Subject: [PATCH 12/41] chore: linter --- app/controllers/api/projects.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 8411ac2a1f..cbda6ad544 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -16,7 +16,6 @@ use Appwrite\Utopia\Database\Validator\Queries\Projects; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use PHPMailer\PHPMailer\PHPMailer; -use Utopia\Abuse\Adapters\Redis\TimeLimit; use Utopia\App; use Utopia\Audit\Audit; use Utopia\Cache\Cache; From cc867e285971f40a996864832b4cfbac95f6f746 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 17 Dec 2024 10:23:09 +0530 Subject: [PATCH 13/41] chore: rename variable --- app/init.php | 4 ++-- src/Appwrite/Platform/Workers/Deletes.php | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/init.php b/app/init.php index 2d01597bc1..056e8ed3d5 100644 --- a/app/init.php +++ b/app/init.php @@ -1531,7 +1531,7 @@ App::setResource('cache', function (Group $pools) { return new Cache(new Sharding($adapters)); }, ['pools']); -App::setResource('redis', function (Group $pools) { +App::setResource('redis', function () { $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); $port = System::getEnv('_APP_REDIS_PORT', 6379); $pass = System::getEnv('_APP_REDIS_PASS', ''); @@ -1544,7 +1544,7 @@ App::setResource('redis', function (Group $pools) { $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); return $redis; -}, ['pools']); +}); App::setResource('deviceForLocal', function () { return new Local(); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index bbc04cec02..6fa65ec842 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -58,8 +58,8 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, $dbForConsole, callable $getProjectDB, \Redis $redis, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $dbForConsole, $getProjectDB, $redis, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) + fn ($message, $dbForPlatform, callable $getProjectDB, \Redis $redis, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForPlatform, $getProjectDB, $redis, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) ); } @@ -67,7 +67,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForConsole, callable $getProjectDB, \Redis $redis, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, \Redis $redis, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; From 52ac358f539757e8cc92ea3fe550a98bc0c9288e Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 17 Dec 2024 10:23:52 +0530 Subject: [PATCH 14/41] chore: rename variable --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index e998cabee1..8f7d7996e7 100644 --- a/.env +++ b/.env @@ -15,7 +15,7 @@ _APP_SYSTEM_TEAM_EMAIL=team@appwrite.io _APP_EMAIL_SECURITY=security@appwrite.io _APP_EMAIL_CERTIFICATES=certificates@appwrite.io _APP_SYSTEM_RESPONSE_FORMAT= -_APP_OPTIONS_ABUSE=enabled +_APP_OPTIONS_ABUSE=disabled _APP_OPTIONS_ROUTER_PROTECTION=disabled _APP_OPTIONS_FORCE_HTTPS=disabled _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS=disabled From c7713f58dd4b0fc33f4d083a183d5ea925ac1233 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Dec 2024 13:52:58 +0000 Subject: [PATCH 15/41] remove worker hook --- app/worker.php | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/app/worker.php b/app/worker.php index 06dc4c44d8..fd9681b37e 100644 --- a/app/worker.php +++ b/app/worker.php @@ -380,20 +380,6 @@ try { $worker = $platform->getWorker(); -$worker - ->init() - ->inject('project') - ->inject('dbForConsole') - ->action(function (Document $project, Database $dbForConsole) { - if (!$project->isEmpty() && $project->getId() !== 'console') { - $accessedAt = $project->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $project->setAttribute('accessedAt', DateTime::now()); - Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); - } - } - }); - $worker ->shutdown() ->inject('pools') From 0cdb2a98be906e86e6248ef78394c32ae9dcf48a Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Dec 2024 14:05:37 +0000 Subject: [PATCH 16/41] fix db name --- app/controllers/general.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 2c43116c10..bb60b01ddd 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -96,7 +96,7 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw $accessedAt = $project->getAttribute('accessedAt', ''); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { $project->setAttribute('accessedAt', DateTime::now()); - Authorization::skip(fn () => $dbForConsole->updateDocument('projects', $project->getId(), $project)); + Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); } } From 2fb9ebe71df40a258f6c89d8d16a0ba8fc99f331 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Dec 2024 14:05:49 +0000 Subject: [PATCH 17/41] update project accessed at in schedules --- .../Platform/Tasks/ScheduleExecutions.php | 12 ++++++++++++ src/Appwrite/Platform/Tasks/ScheduleFunctions.php | 15 +++++++++++++-- src/Appwrite/Platform/Tasks/ScheduleMessages.php | 14 +++++++++++++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index 50beb48e9d..9be7950fd6 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -5,6 +5,8 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Func; use Swoole\Coroutine as Co; use Utopia\Database\Database; +use Utopia\Database\DateTime; +use Utopia\Database\Validator\Authorization; use Utopia\Pools\Group; class ScheduleExecutions extends ScheduleBase @@ -57,6 +59,16 @@ class ScheduleExecutions extends ScheduleBase $delay = $scheduledAt->getTimestamp() - (new \DateTime())->getTimestamp(); + $project = $schedule['project']; + + if (!$project->isEmpty() && $project->getId() !== 'console') { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); + } + } + \go(function () use ($queueForFunctions, $schedule, $delay, $data) { Co::sleep($delay); diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index 156ff1e31d..d1a9662967 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -7,6 +7,7 @@ use Cron\CronExpression; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; +use Utopia\Database\Validator\Authorization; use Utopia\Pools\Group; class ScheduleFunctions extends ScheduleBase @@ -70,7 +71,7 @@ class ScheduleFunctions extends ScheduleBase } foreach ($delayedExecutions as $delay => $scheduleKeys) { - \go(function () use ($delay, $scheduleKeys, $pools) { + \go(function () use ($delay, $scheduleKeys, $pools, $dbForPlatform) { \sleep($delay); // in seconds $queue = $pools->get('queue')->pop(); @@ -84,6 +85,16 @@ class ScheduleFunctions extends ScheduleBase $schedule = $this->schedules[$scheduleKey]; + $project = $schedule['project']; + + if (!$project->isEmpty() && $project->getId() !== 'console') { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); + } + } + $queueForFunctions = new Func($connection); $queueForFunctions @@ -91,7 +102,7 @@ class ScheduleFunctions extends ScheduleBase ->setFunction($schedule['resource']) ->setMethod('POST') ->setPath('/') - ->setProject($schedule['project']) + ->setProject($project) ->trigger(); } diff --git a/src/Appwrite/Platform/Tasks/ScheduleMessages.php b/src/Appwrite/Platform/Tasks/ScheduleMessages.php index 2136e62f08..3beb8c48fb 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleMessages.php +++ b/src/Appwrite/Platform/Tasks/ScheduleMessages.php @@ -4,6 +4,8 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Messaging; use Utopia\Database\Database; +use Utopia\Database\DateTime; +use Utopia\Database\Validator\Authorization; use Utopia\Pools\Group; class ScheduleMessages extends ScheduleBase @@ -45,10 +47,20 @@ class ScheduleMessages extends ScheduleBase $connection = $queue->getResource(); $queueForMessaging = new Messaging($connection); + $project = $schedule['project']; + + if (!$project->isEmpty() && $project->getId() !== 'console') { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); + } + } + $queueForMessaging ->setType(MESSAGE_SEND_TYPE_EXTERNAL) ->setMessageId($schedule['resourceId']) - ->setProject($schedule['project']) + ->setProject($project) ->trigger(); $dbForPlatform->deleteDocument( From d7d69b63a8f95eefc895a532079fbc56ea741045 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Dec 2024 14:15:27 +0000 Subject: [PATCH 18/41] reset worker --- app/worker.php | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/app/worker.php b/app/worker.php index fd9681b37e..563a801b6b 100644 --- a/app/worker.php +++ b/app/worker.php @@ -62,13 +62,7 @@ Server::setResource('project', function (Message $message, Database $dbForPlatfo return $project; } - // NOT all workers need valid DB config so we return empty if this fails - try { - return $dbForPlatform->getDocument('projects', $project->getId()); - } catch (\Throwable $e) { - return new Document([]); - } - + return $dbForPlatform->getDocument('projects', $project->getId()); }, ['message', 'dbForPlatform']); Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { @@ -435,4 +429,4 @@ $worker->workerStart() Console::info("Worker $workerName started"); }); -$worker->start(); +$worker->start(); \ No newline at end of file From e24cce4e48aee24b6fc8e6713565e82bf1b340d1 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Dec 2024 14:19:26 +0000 Subject: [PATCH 19/41] refactor --- app/worker.php | 2 +- src/Appwrite/Platform/Tasks/ScheduleBase.php | 12 ++++++++++++ src/Appwrite/Platform/Tasks/ScheduleExecutions.php | 12 +----------- src/Appwrite/Platform/Tasks/ScheduleFunctions.php | 13 ++----------- src/Appwrite/Platform/Tasks/ScheduleMessages.php | 14 ++------------ 5 files changed, 18 insertions(+), 35 deletions(-) diff --git a/app/worker.php b/app/worker.php index 563a801b6b..2c7d8acb22 100644 --- a/app/worker.php +++ b/app/worker.php @@ -429,4 +429,4 @@ $worker->workerStart() Console::info("Worker $workerName started"); }); -$worker->start(); \ No newline at end of file +$worker->start(); diff --git a/src/Appwrite/Platform/Tasks/ScheduleBase.php b/src/Appwrite/Platform/Tasks/ScheduleBase.php index 10623e7fa5..dad2db0d9a 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleBase.php +++ b/src/Appwrite/Platform/Tasks/ScheduleBase.php @@ -9,6 +9,7 @@ use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception; use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; use Utopia\Pools\Group; use Utopia\System\System; @@ -39,6 +40,17 @@ abstract class ScheduleBase extends Action ->callback(fn (Group $pools, Database $dbForPlatform, callable $getProjectDB) => $this->action($pools, $dbForPlatform, $getProjectDB)); } + protected function updateProjectAccess(Document $project, Database $dbForPlatform): void + { + if (!$project->isEmpty() && $project->getId() !== 'console') { + $accessedAt = $project->getAttribute('accessedAt', ''); + if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { + $project->setAttribute('accessedAt', DateTime::now()); + Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); + } + } + } + /** * 1. Load all documents from 'schedules' collection to create local copy * 2. Create timer that sync all changes from 'schedules' collection to local copy. Only reading changes thanks to 'resourceUpdatedAt' attribute diff --git a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php index 9be7950fd6..086bad513e 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleExecutions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleExecutions.php @@ -5,8 +5,6 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Func; use Swoole\Coroutine as Co; use Utopia\Database\Database; -use Utopia\Database\DateTime; -use Utopia\Database\Validator\Authorization; use Utopia\Pools\Group; class ScheduleExecutions extends ScheduleBase @@ -59,15 +57,7 @@ class ScheduleExecutions extends ScheduleBase $delay = $scheduledAt->getTimestamp() - (new \DateTime())->getTimestamp(); - $project = $schedule['project']; - - if (!$project->isEmpty() && $project->getId() !== 'console') { - $accessedAt = $project->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $project->setAttribute('accessedAt', DateTime::now()); - Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); - } - } + $this->updateProjectAccess($schedule['project'], $dbForPlatform); \go(function () use ($queueForFunctions, $schedule, $delay, $data) { Co::sleep($delay); diff --git a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php index d1a9662967..c443bb6c2d 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleFunctions.php +++ b/src/Appwrite/Platform/Tasks/ScheduleFunctions.php @@ -7,7 +7,6 @@ use Cron\CronExpression; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\DateTime; -use Utopia\Database\Validator\Authorization; use Utopia\Pools\Group; class ScheduleFunctions extends ScheduleBase @@ -85,15 +84,7 @@ class ScheduleFunctions extends ScheduleBase $schedule = $this->schedules[$scheduleKey]; - $project = $schedule['project']; - - if (!$project->isEmpty() && $project->getId() !== 'console') { - $accessedAt = $project->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $project->setAttribute('accessedAt', DateTime::now()); - Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); - } - } + $this->updateProjectAccess($schedule['project'], $dbForPlatform); $queueForFunctions = new Func($connection); @@ -102,7 +93,7 @@ class ScheduleFunctions extends ScheduleBase ->setFunction($schedule['resource']) ->setMethod('POST') ->setPath('/') - ->setProject($project) + ->setProject($schedule['project']) ->trigger(); } diff --git a/src/Appwrite/Platform/Tasks/ScheduleMessages.php b/src/Appwrite/Platform/Tasks/ScheduleMessages.php index 3beb8c48fb..5d997fc5bb 100644 --- a/src/Appwrite/Platform/Tasks/ScheduleMessages.php +++ b/src/Appwrite/Platform/Tasks/ScheduleMessages.php @@ -4,8 +4,6 @@ namespace Appwrite\Platform\Tasks; use Appwrite\Event\Messaging; use Utopia\Database\Database; -use Utopia\Database\DateTime; -use Utopia\Database\Validator\Authorization; use Utopia\Pools\Group; class ScheduleMessages extends ScheduleBase @@ -47,20 +45,12 @@ class ScheduleMessages extends ScheduleBase $connection = $queue->getResource(); $queueForMessaging = new Messaging($connection); - $project = $schedule['project']; - - if (!$project->isEmpty() && $project->getId() !== 'console') { - $accessedAt = $project->getAttribute('accessedAt', ''); - if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { - $project->setAttribute('accessedAt', DateTime::now()); - Authorization::skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), $project)); - } - } + $this->updateProjectAccess($schedule['project'], $dbForPlatform); $queueForMessaging ->setType(MESSAGE_SEND_TYPE_EXTERNAL) ->setMessageId($schedule['resourceId']) - ->setProject($project) + ->setProject($schedule['project']) ->trigger(); $dbForPlatform->deleteDocument( From 01393faf0bce3a2bca29f72b9596e4d9d448148d Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 18 Dec 2024 11:58:34 +0530 Subject: [PATCH 20/41] feat: update abuse library --- app/controllers/api/functions.php | 9 +++--- app/controllers/shared/api.php | 8 ++--- app/init.php | 8 +++++ app/realtime.php | 13 ++++++-- composer.json | 2 +- composer.lock | 39 ++++++++++++++--------- src/Appwrite/Platform/Workers/Deletes.php | 16 +++++----- 7 files changed, 60 insertions(+), 35 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 5934258037..b997d79733 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -24,7 +24,7 @@ use Appwrite\Utopia\Response\Model\Rule; use Executor\Executor; use MaxMind\Db\Reader; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\Database\TimeLimit; +use Utopia\Abuse\Adapters\TimeLimit; use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -180,22 +180,23 @@ App::post('/v1/functions') ->inject('request') ->inject('response') ->inject('dbForProject') + ->inject('adapterForAbuse') ->inject('project') ->inject('user') ->inject('queueForEvents') ->inject('queueForBuilds') ->inject('dbForPlatform') ->inject('gitHub') - ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { + ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, TimeLimit $adapterForAbuse, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; // Temporary abuse check - $abuseCheck = function () use ($project, $dbForProject, $response) { + $abuseCheck = function () use ($project, $adapterForAbuse, $response) { $abuseKey = "projectId:{projectId},url:{url}"; $abuseLimit = App::getEnv('_APP_FUNCTIONS_CREATION_ABUSE_LIMIT', 50); $abuseTime = 86400; // 1 day - $timeLimit = new TimeLimit($abuseKey, $abuseLimit, $abuseTime, $dbForProject); + $timeLimit = $adapterForAbuse($abuseKey, $abuseLimit, $abuseTime); $timeLimit ->setParam('{projectId}', $project->getId()) ->setParam('{url}', '/v1/functions'); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 9a15467ec0..59e5012fe9 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -19,7 +19,7 @@ use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\Redis\TimeLimit; +use Utopia\Abuse\Adapters\TimeLimit; use Utopia\App; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; @@ -420,9 +420,9 @@ App::init() ->inject('queueForBuilds') ->inject('queueForUsage') ->inject('dbForProject') - ->inject('redis') + ->inject('adapterForAbuse') ->inject('mode') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, \Redis $redis, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, TimeLimit $adapterForAbuse, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -445,7 +445,7 @@ App::init() foreach ($abuseKeyLabel as $abuseKey) { $start = $request->getContentRangeStart(); $end = $request->getContentRangeEnd(); - $timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $redis); + $timeLimit = $adapterForAbuse($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600)); $timeLimit ->setParam('{projectId}', $project->getId()) ->setParam('{userId}', $user->getId()) diff --git a/app/init.php b/app/init.php index 056e8ed3d5..0c6ff1f57b 100644 --- a/app/init.php +++ b/app/init.php @@ -48,6 +48,7 @@ use Appwrite\Utopia\Request; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; +use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Cache\Adapter\Sharding; @@ -93,6 +94,7 @@ use Utopia\Validator\Range; use Utopia\Validator\URL; use Utopia\Validator\WhiteList; use Utopia\VCS\Adapter\Git\GitHub as VcsGitHub; +use Redis; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; @@ -1546,6 +1548,12 @@ App::setResource('redis', function () { return $redis; }); +App::setResource('adapterForAbuse', function (\Redis $redis) { + return function (string $key, int $limit, int $time) use ($redis) { + return new TimeLimitRedis($key, $limit, $time, $redis); + }; +}, ['redis']); + App::setResource('deviceForLocal', function () { return new Local(); }); diff --git a/app/realtime.php b/app/realtime.php index 2df8abfbfa..59212e14f1 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -13,7 +13,7 @@ use Swoole\Runtime; use Swoole\Table; use Swoole\Timer; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\Redis\TimeLimit; +use Utopia\Abuse\Adapters\TimeLimit; use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; @@ -157,6 +157,13 @@ if (!function_exists('getRedis')) { } } +if (!function_exists('getAdapterForAbuse')) { + function getAdapterForAbuse(): TimeLimit + { + return new TimeLimit("", 0, 1, getRedis()); + } +} + if (!function_exists('getRealtime')) { function getRealtime(): Realtime { @@ -500,7 +507,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new AppwriteException(AppwriteException::GENERAL_API_DISABLED); } - $redis = $app->getResource('redis'); + $adapterForAbuse = $app->getResource('adapterForAbuse'); $console = $app->getResource('console'); /** @var Document $console */ $user = $app->getResource('user'); /** @var Document $user */ @@ -509,7 +516,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, * * Abuse limits are connecting 128 times per minute and ip address. */ - $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $redis); + $timeLimit = $adapterForAbuse('url:{url},ip:{ip}', 128, 60); $timeLimit ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getURI()); diff --git a/composer.json b/composer.json index 512e203a5e..c4d4bfeb75 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.16.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "0.43.*", + "utopia-php/abuse": "dev-add-support-for-sharding-adapter as 0.43.1000", "utopia-php/analytics": "0.10.*", "utopia-php/audit": "0.43.*", "utopia-php/cache": "0.11.*", diff --git a/composer.lock b/composer.lock index 37a5c81436..14135bc27f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fae350df93342992edd8f639948e1570", + "content-hash": "2dd50e9cf2c2b74a56ea42d8593ffc0a", "packages": [ { "name": "adhocore/jwt", @@ -3136,16 +3136,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.43.2", + "version": "dev-add-support-for-sharding-adapter", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "374536b86d8d39066960a7da161d444a099bbc56" + "reference": "627d1ba56451dd70ca26a1253c42e039b4bfc1a7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/374536b86d8d39066960a7da161d444a099bbc56", - "reference": "374536b86d8d39066960a7da161d444a099bbc56", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/627d1ba56451dd70ca26a1253c42e039b4bfc1a7", + "reference": "627d1ba56451dd70ca26a1253c42e039b4bfc1a7", "shasum": "" }, "require": { @@ -3181,9 +3181,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.43.2" + "source": "https://github.com/utopia-php/abuse/tree/add-support-for-sharding-adapter" }, - "time": "2024-12-12T19:43:24+00:00" + "time": "2024-12-18T06:08:34+00:00" }, { "name": "utopia-php/analytics", @@ -4807,16 +4807,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.39.26", + "version": "0.39.27", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "39768deacb4913f93548c46fa0149c3fadc62d0c" + "reference": "27d8ecde30e40cbfe1124cc0430c406d3e144849" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/39768deacb4913f93548c46fa0149c3fadc62d0c", - "reference": "39768deacb4913f93548c46fa0149c3fadc62d0c", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/27d8ecde30e40cbfe1124cc0430c406d3e144849", + "reference": "27d8ecde30e40cbfe1124cc0430c406d3e144849", "shasum": "" }, "require": { @@ -4852,9 +4852,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.39.26" + "source": "https://github.com/appwrite/sdk-generator/tree/0.39.27" }, - "time": "2024-12-12T10:51:34+00:00" + "time": "2024-12-16T11:32:02+00:00" }, { "name": "doctrine/annotations", @@ -8554,9 +8554,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/abuse", + "version": "dev-add-support-for-sharding-adapter", + "alias": "0.43.1000", + "alias_normalized": "0.43.1000.0" + } + ], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/abuse": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 6fa65ec842..d6bba4b88a 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -8,7 +8,7 @@ use Appwrite\Extend\Exception; use Executor\Executor; use Throwable; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\Redis\TimeLimit; +use Utopia\Abuse\Adapters\TimeLimit; use Utopia\Audit\Audit; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; @@ -47,7 +47,7 @@ class Deletes extends Action ->inject('message') ->inject('dbForPlatform') ->inject('getProjectDB') - ->inject('redis') + ->inject('adapterForAbuse') ->inject('deviceForFiles') ->inject('deviceForFunctions') ->inject('deviceForBuilds') @@ -58,8 +58,8 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, $dbForPlatform, callable $getProjectDB, \Redis $redis, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $dbForPlatform, $getProjectDB, $redis, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) + fn ($message, $dbForPlatform, callable $getProjectDB, TimeLimit $adapterForAbuse, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForPlatform, $getProjectDB, $adapterForAbuse, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) ); } @@ -67,7 +67,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, \Redis $redis, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, TimeLimit $adapterForAbuse, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -127,7 +127,7 @@ class Deletes extends Action } break; case DELETE_TYPE_ABUSE: - $this->deleteAbuseLogs($project, $redis, $abuseRetention); + $this->deleteAbuseLogs($project, $adapterForAbuse, $abuseRetention); break; case DELETE_TYPE_REALTIME: $this->deleteRealtimeUsage($dbForPlatform, $datetime); @@ -708,10 +708,10 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteAbuseLogs(Document $project, \Redis $redis, string $abuseRetention): void + private function deleteAbuseLogs(Document $project, TimeLimit $adapterForAbuse, string $abuseRetention): void { $projectId = $project->getId(); - $timeLimit = new TimeLimit("", 0, 1, $redis); + $timeLimit = $adapterForAbuse("", 0, 1); $abuse = new Abuse($timeLimit); try { From f1a089f8c6386e78abb023de3ace4adb2dd16247 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 18 Dec 2024 12:57:52 +0530 Subject: [PATCH 21/41] feat: fix tests --- app/controllers/api/functions.php | 2 +- app/controllers/shared/api.php | 2 +- src/Appwrite/Platform/Workers/Deletes.php | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index b997d79733..bdcb6b3286 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -187,7 +187,7 @@ App::post('/v1/functions') ->inject('queueForBuilds') ->inject('dbForPlatform') ->inject('gitHub') - ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, TimeLimit $adapterForAbuse, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { + ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, callable $adapterForAbuse, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; // Temporary abuse check diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 59e5012fe9..148c7282e7 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -422,7 +422,7 @@ App::init() ->inject('dbForProject') ->inject('adapterForAbuse') ->inject('mode') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, TimeLimit $adapterForAbuse, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, callable $adapterForAbuse, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index d6bba4b88a..dae5f9fe58 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -58,7 +58,7 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, $dbForPlatform, callable $getProjectDB, TimeLimit $adapterForAbuse, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + fn ($message, $dbForPlatform, callable $getProjectDB, callable $adapterForAbuse, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForPlatform, $getProjectDB, $adapterForAbuse, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) ); } @@ -67,7 +67,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, TimeLimit $adapterForAbuse, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, callable $adapterForAbuse, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -708,7 +708,7 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteAbuseLogs(Document $project, TimeLimit $adapterForAbuse, string $abuseRetention): void + private function deleteAbuseLogs(Document $project, callable $adapterForAbuse, string $abuseRetention): void { $projectId = $project->getId(); $timeLimit = $adapterForAbuse("", 0, 1); From 911edd9457d6d46560e7ca5e4367b10add5f995d Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 18 Dec 2024 13:08:17 +0530 Subject: [PATCH 22/41] chore: linter --- app/controllers/api/functions.php | 1 - app/controllers/shared/api.php | 1 - app/init.php | 1 - composer.json | 2 +- composer.lock | 11 ++--------- src/Appwrite/Platform/Workers/Deletes.php | 1 - 6 files changed, 3 insertions(+), 14 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index bdcb6b3286..219832077f 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -24,7 +24,6 @@ use Appwrite\Utopia\Response\Model\Rule; use Executor\Executor; use MaxMind\Db\Reader; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\TimeLimit; use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 148c7282e7..50a4a1e02e 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -19,7 +19,6 @@ use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\TimeLimit; use Utopia\App; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; diff --git a/app/init.php b/app/init.php index 0c6ff1f57b..5282c7f70d 100644 --- a/app/init.php +++ b/app/init.php @@ -94,7 +94,6 @@ use Utopia\Validator\Range; use Utopia\Validator\URL; use Utopia\Validator\WhiteList; use Utopia\VCS\Adapter\Git\GitHub as VcsGitHub; -use Redis; const APP_NAME = 'Appwrite'; const APP_DOMAIN = 'appwrite.io'; diff --git a/composer.json b/composer.json index c4d4bfeb75..5f81fd7752 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.16.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "dev-add-support-for-sharding-adapter as 0.43.1000", + "utopia-php/abuse": "dev-add-support-for-sharding-adapter", "utopia-php/analytics": "0.10.*", "utopia-php/audit": "0.43.*", "utopia-php/cache": "0.11.*", diff --git a/composer.lock b/composer.lock index 14135bc27f..81690cc718 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2dd50e9cf2c2b74a56ea42d8593ffc0a", + "content-hash": "d2345c93c578d7ac06f364fc5a175810", "packages": [ { "name": "adhocore/jwt", @@ -8554,14 +8554,7 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/abuse", - "version": "dev-add-support-for-sharding-adapter", - "alias": "0.43.1000", - "alias_normalized": "0.43.1000.0" - } - ], + "aliases": [], "minimum-stability": "stable", "stability-flags": { "utopia-php/abuse": 20 diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index dae5f9fe58..bd9a8f976b 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -8,7 +8,6 @@ use Appwrite\Extend\Exception; use Executor\Executor; use Throwable; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\TimeLimit; use Utopia\Audit\Audit; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; From 77a8c271531ff78f85c56dc7dc5d5c60c5cb3100 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 18 Dec 2024 13:26:31 +0530 Subject: [PATCH 23/41] chore: fix realtime tests --- app/realtime.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 59212e14f1..78b17a3a78 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -619,8 +619,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re * * Abuse limits are sending 32 times per minute and connection. */ - $redis = getRedis(); - $timeLimit = new TimeLimit('url:{url},connection:{connection}', 32, 60, $redis); + $timeLimit = getAdapterForAbuse('url:{url},connection:{connection}', 32, 60); $timeLimit ->setParam('{connection}', $connection) From 599cc5e19afffe8bef10c4c720883a7ecb32b3dd Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 18 Dec 2024 15:57:04 +0530 Subject: [PATCH 24/41] chore: review comments --- app/controllers/api/functions.php | 8 ++++---- app/controllers/shared/api.php | 6 +++--- app/init.php | 2 +- app/realtime.php | 14 +++++++------- src/Appwrite/Platform/Workers/Deletes.php | 14 +++++++------- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 219832077f..86fea8a60d 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -179,23 +179,23 @@ App::post('/v1/functions') ->inject('request') ->inject('response') ->inject('dbForProject') - ->inject('adapterForAbuse') + ->inject('timelimit') ->inject('project') ->inject('user') ->inject('queueForEvents') ->inject('queueForBuilds') ->inject('dbForPlatform') ->inject('gitHub') - ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, callable $adapterForAbuse, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { + ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, callable $timelimit, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; // Temporary abuse check - $abuseCheck = function () use ($project, $adapterForAbuse, $response) { + $abuseCheck = function () use ($project, $timelimit, $response) { $abuseKey = "projectId:{projectId},url:{url}"; $abuseLimit = App::getEnv('_APP_FUNCTIONS_CREATION_ABUSE_LIMIT', 50); $abuseTime = 86400; // 1 day - $timeLimit = $adapterForAbuse($abuseKey, $abuseLimit, $abuseTime); + $timeLimit = $timelimit($abuseKey, $abuseLimit, $abuseTime); $timeLimit ->setParam('{projectId}', $project->getId()) ->setParam('{url}', '/v1/functions'); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 50a4a1e02e..7752f2f8a9 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -419,9 +419,9 @@ App::init() ->inject('queueForBuilds') ->inject('queueForUsage') ->inject('dbForProject') - ->inject('adapterForAbuse') + ->inject('timelimit') ->inject('mode') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, callable $adapterForAbuse, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, callable $timelimit, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -444,7 +444,7 @@ App::init() foreach ($abuseKeyLabel as $abuseKey) { $start = $request->getContentRangeStart(); $end = $request->getContentRangeEnd(); - $timeLimit = $adapterForAbuse($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600)); + $timeLimit = $timelimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600)); $timeLimit ->setParam('{projectId}', $project->getId()) ->setParam('{userId}', $user->getId()) diff --git a/app/init.php b/app/init.php index 5282c7f70d..a738a44577 100644 --- a/app/init.php +++ b/app/init.php @@ -1547,7 +1547,7 @@ App::setResource('redis', function () { return $redis; }); -App::setResource('adapterForAbuse', function (\Redis $redis) { +App::setResource('timelimit', function (\Redis $redis) { return function (string $key, int $limit, int $time) use ($redis) { return new TimeLimitRedis($key, $limit, $time, $redis); }; diff --git a/app/realtime.php b/app/realtime.php index 78b17a3a78..d47098340e 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -157,8 +157,8 @@ if (!function_exists('getRedis')) { } } -if (!function_exists('getAdapterForAbuse')) { - function getAdapterForAbuse(): TimeLimit +if (!function_exists('getTimelimit')) { + function getTimelimit(): TimeLimit { return new TimeLimit("", 0, 1, getRedis()); } @@ -507,7 +507,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new AppwriteException(AppwriteException::GENERAL_API_DISABLED); } - $adapterForAbuse = $app->getResource('adapterForAbuse'); + $timelimit = $app->getResource('timelimit'); $console = $app->getResource('console'); /** @var Document $console */ $user = $app->getResource('user'); /** @var Document $user */ @@ -516,12 +516,12 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, * * Abuse limits are connecting 128 times per minute and ip address. */ - $timeLimit = $adapterForAbuse('url:{url},ip:{ip}', 128, 60); - $timeLimit + $timelimit = $timelimit('url:{url},ip:{ip}', 128, 60); + $timelimit ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getURI()); - $abuse = new Abuse($timeLimit); + $abuse = new Abuse($timelimit); if (System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled' && $abuse->check()) { throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many requests'); @@ -619,7 +619,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re * * Abuse limits are sending 32 times per minute and connection. */ - $timeLimit = getAdapterForAbuse('url:{url},connection:{connection}', 32, 60); + $timeLimit = getTimelimit('url:{url},connection:{connection}', 32, 60); $timeLimit ->setParam('{connection}', $connection) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index bd9a8f976b..bbc808e248 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -46,7 +46,7 @@ class Deletes extends Action ->inject('message') ->inject('dbForPlatform') ->inject('getProjectDB') - ->inject('adapterForAbuse') + ->inject('timelimit') ->inject('deviceForFiles') ->inject('deviceForFunctions') ->inject('deviceForBuilds') @@ -57,8 +57,8 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, $dbForPlatform, callable $getProjectDB, callable $adapterForAbuse, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $dbForPlatform, $getProjectDB, $adapterForAbuse, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) + fn ($message, $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForPlatform, $getProjectDB, $timelimit, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, callable $adapterForAbuse, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -126,7 +126,7 @@ class Deletes extends Action } break; case DELETE_TYPE_ABUSE: - $this->deleteAbuseLogs($project, $adapterForAbuse, $abuseRetention); + $this->deleteAbuseLogs($project, $timelimit, $abuseRetention); break; case DELETE_TYPE_REALTIME: $this->deleteRealtimeUsage($dbForPlatform, $datetime); @@ -707,10 +707,10 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteAbuseLogs(Document $project, callable $adapterForAbuse, string $abuseRetention): void + private function deleteAbuseLogs(Document $project, callable $timelimit, string $abuseRetention): void { $projectId = $project->getId(); - $timeLimit = $adapterForAbuse("", 0, 1); + $timeLimit = $timelimit("", 0, 1); $abuse = new Abuse($timeLimit); try { From d0f04126f3d4795abf8b8a6f9f7d978a557c2339 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 18 Dec 2024 19:59:49 +0530 Subject: [PATCH 25/41] chore: add missing resources to workers --- app/worker.php | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/app/worker.php b/app/worker.php index 2c7d8acb22..7539a26879 100644 --- a/app/worker.php +++ b/app/worker.php @@ -17,6 +17,7 @@ use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; +use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -200,6 +201,27 @@ Server::setResource('cache', function (Registry $register) { return new Cache(new Sharding($adapters)); }, ['register']); +Server::setResource('redis', function () { + $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); + $port = System::getEnv('_APP_REDIS_PORT', 6379); + $pass = System::getEnv('_APP_REDIS_PASS', ''); + + $redis = new \Redis(); + @$redis->pconnect($host, (int)$port); + if ($pass) { + $redis->auth($pass); + } + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); + + return $redis; +}); + +Server::setResource('timelimit', function (\Redis $redis) { + return function (string $key, int $limit, int $time) use ($redis) { + return new TimeLimitRedis($key, $limit, $time, $redis); + }; +}, ['redis']); + Server::setResource('log', fn () => new Log()); Server::setResource('queueForUsage', function (Connection $queue) { From 5a9dbe9c80b7d1fa121ee0be1ea724af1193a525 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 18 Dec 2024 20:06:24 +0530 Subject: [PATCH 26/41] chore: fix realtime test --- app/realtime.php | 6 +++--- phpunit.xml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index d47098340e..4f87e4dea1 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -13,7 +13,7 @@ use Swoole\Runtime; use Swoole\Table; use Swoole\Timer; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\TimeLimit; +use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; @@ -158,9 +158,9 @@ if (!function_exists('getRedis')) { } if (!function_exists('getTimelimit')) { - function getTimelimit(): TimeLimit + function getTimelimit(): TimeLimitRedis { - return new TimeLimit("", 0, 1, getRedis()); + return new TimeLimitRedis("", 0, 1, getRedis()); } } diff --git a/phpunit.xml b/phpunit.xml index 4c4e55ea4e..598b730908 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -6,7 +6,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false" + stopOnFailure="true" > From 12cb3b10bd6283fb00dc0b46b590bf2f228b145f Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 18 Dec 2024 20:42:29 +0530 Subject: [PATCH 27/41] chore: update abuse library --- composer.json | 2 +- composer.lock | 30 ++++++++++++++---------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/composer.json b/composer.json index 5f81fd7752..a8a54c75ab 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.16.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "dev-add-support-for-sharding-adapter", + "utopia-php/abuse": "0.44.*", "utopia-php/analytics": "0.10.*", "utopia-php/audit": "0.43.*", "utopia-php/cache": "0.11.*", diff --git a/composer.lock b/composer.lock index 81690cc718..09f9b3aed9 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d2345c93c578d7ac06f364fc5a175810", + "content-hash": "fd1cf738be4e037abdc1a0fa4cb27a3c", "packages": [ { "name": "adhocore/jwt", @@ -709,16 +709,16 @@ }, { "name": "google/protobuf", - "version": "v4.29.1", + "version": "v4.29.2", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "6042b5483f8029e42473faeb8ef75ba266278381" + "reference": "79aa5014efeeec3d137df5cdb0ae2fc163953945" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/6042b5483f8029e42473faeb8ef75ba266278381", - "reference": "6042b5483f8029e42473faeb8ef75ba266278381", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/79aa5014efeeec3d137df5cdb0ae2fc163953945", + "reference": "79aa5014efeeec3d137df5cdb0ae2fc163953945", "shasum": "" }, "require": { @@ -747,9 +747,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.1" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.2" }, - "time": "2024-12-03T22:07:45+00:00" + "time": "2024-12-18T14:11:12+00:00" }, { "name": "jean85/pretty-package-versions", @@ -3136,16 +3136,16 @@ }, { "name": "utopia-php/abuse", - "version": "dev-add-support-for-sharding-adapter", + "version": "0.44.0", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "627d1ba56451dd70ca26a1253c42e039b4bfc1a7" + "reference": "25b9654ddfb1a17161debf1d26f2915e0128d9fd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/627d1ba56451dd70ca26a1253c42e039b4bfc1a7", - "reference": "627d1ba56451dd70ca26a1253c42e039b4bfc1a7", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/25b9654ddfb1a17161debf1d26f2915e0128d9fd", + "reference": "25b9654ddfb1a17161debf1d26f2915e0128d9fd", "shasum": "" }, "require": { @@ -3181,9 +3181,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/add-support-for-sharding-adapter" + "source": "https://github.com/utopia-php/abuse/tree/0.44.0" }, - "time": "2024-12-18T06:08:34+00:00" + "time": "2024-12-18T15:09:47+00:00" }, { "name": "utopia-php/analytics", @@ -8556,9 +8556,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/abuse": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From 53d8e2b0c3e07598a5d2c22f30f71ed16abed1bb Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 19 Dec 2024 16:43:05 +0530 Subject: [PATCH 28/41] chore: optimise webhooks payload --- src/Appwrite/Event/Webhook.php | 12 ++++++++++++ src/Appwrite/Platform/Workers/Webhooks.php | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Event/Webhook.php b/src/Appwrite/Event/Webhook.php index 125c9a78d5..6457c53e68 100644 --- a/src/Appwrite/Event/Webhook.php +++ b/src/Appwrite/Event/Webhook.php @@ -14,4 +14,16 @@ class Webhook extends Event ->setQueue(Event::WEBHOOK_QUEUE_NAME) ->setClass(Event::WEBHOOK_CLASS_NAME); } + + public function trigger(): string|bool + { + /** Filter out context and trim project to keep the payload small */ + $this->context = []; + $this->project = [ + '$id' => $this->project->getId(), + '$internalId' => $this->project->getAttribute('internalId'), + ]; + + return parent::trigger(); + } } diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php index ba0bdaa6a3..271c4c00f0 100644 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -56,9 +56,10 @@ class Webhooks extends Action $events = $payload['events']; $webhookPayload = json_encode($payload['payload']); - $project = new Document($payload['project']); $user = new Document($payload['user'] ?? []); + $project = new Document($payload['project']); + $project = $dbForPlatform->getDocument('projects', $project->getId()); $log->addTag('projectId', $project->getId()); foreach ($project->getAttribute('webhooks', []) as $webhook) { From d03d3c05b4d1c9331029981e345912d81504875e Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 19 Dec 2024 16:52:10 +0530 Subject: [PATCH 29/41] chore: optimise webhooks payload --- src/Appwrite/Event/Webhook.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Event/Webhook.php b/src/Appwrite/Event/Webhook.php index 6457c53e68..e4c4f5e210 100644 --- a/src/Appwrite/Event/Webhook.php +++ b/src/Appwrite/Event/Webhook.php @@ -3,6 +3,7 @@ namespace Appwrite\Event; use Utopia\Queue\Connection; +use Utopia\Database\Document; class Webhook extends Event { @@ -19,10 +20,10 @@ class Webhook extends Event { /** Filter out context and trim project to keep the payload small */ $this->context = []; - $this->project = [ + $this->project = new Document([ '$id' => $this->project->getId(), '$internalId' => $this->project->getAttribute('internalId'), - ]; + ]); return parent::trigger(); } From 6ec640cfd5905e42455f0468eec477a1c53d50ec Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 19 Dec 2024 16:56:17 +0530 Subject: [PATCH 30/41] chore: linter --- src/Appwrite/Event/Webhook.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Event/Webhook.php b/src/Appwrite/Event/Webhook.php index e4c4f5e210..954e8df1fd 100644 --- a/src/Appwrite/Event/Webhook.php +++ b/src/Appwrite/Event/Webhook.php @@ -2,8 +2,8 @@ namespace Appwrite\Event; -use Utopia\Queue\Connection; use Utopia\Database\Document; +use Utopia\Queue\Connection; class Webhook extends Event { From b97dea7996f7fd217562cff675d7fb996605fe90 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 19 Dec 2024 20:27:54 +0530 Subject: [PATCH 31/41] Revert "chore: initial commit" --- app/controllers/api/functions.php | 8 +- app/controllers/api/projects.php | 15 ++++ app/controllers/shared/api.php | 6 +- app/http.php | 7 ++ app/init.php | 36 ++------ app/realtime.php | 38 ++------ app/worker.php | 22 ----- composer.json | 2 +- composer.lock | 102 +++++++++++----------- phpunit.xml | 2 +- src/Appwrite/Platform/Workers/Deletes.php | 18 ++-- 11 files changed, 105 insertions(+), 151 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 86fea8a60d..5934258037 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -24,6 +24,7 @@ use Appwrite\Utopia\Response\Model\Rule; use Executor\Executor; use MaxMind\Db\Reader; use Utopia\Abuse\Abuse; +use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -179,23 +180,22 @@ App::post('/v1/functions') ->inject('request') ->inject('response') ->inject('dbForProject') - ->inject('timelimit') ->inject('project') ->inject('user') ->inject('queueForEvents') ->inject('queueForBuilds') ->inject('dbForPlatform') ->inject('gitHub') - ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, callable $timelimit, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { + ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; // Temporary abuse check - $abuseCheck = function () use ($project, $timelimit, $response) { + $abuseCheck = function () use ($project, $dbForProject, $response) { $abuseKey = "projectId:{projectId},url:{url}"; $abuseLimit = App::getEnv('_APP_FUNCTIONS_CREATION_ABUSE_LIMIT', 50); $abuseTime = 86400; // 1 day - $timeLimit = $timelimit($abuseKey, $abuseLimit, $abuseTime); + $timeLimit = new TimeLimit($abuseKey, $abuseLimit, $abuseTime, $dbForProject); $timeLimit ->setParam('{projectId}', $project->getId()) ->setParam('{url}', '/v1/functions'); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index cbda6ad544..388c9e22ea 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -16,6 +16,7 @@ use Appwrite\Utopia\Database\Validator\Queries\Projects; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use PHPMailer\PHPMailer\PHPMailer; +use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\App; use Utopia\Audit\Audit; use Utopia\Cache\Cache; @@ -228,6 +229,9 @@ App::post('/v1/projects') if ($create || $projectTables) { $audit = new Audit($dbForProject); $audit->setup(); + + $abuse = new TimeLimit('', 0, 1, $dbForProject); + $abuse->setup(); } if (!$create && $sharedTablesV1) { @@ -241,6 +245,17 @@ App::post('/v1/projects') 'indexes' => $indexes, 'documentSecurity' => true ])); + + $attributes = \array_map(fn ($attribute) => new Document($attribute), TimeLimit::ATTRIBUTES); + $indexes = \array_map(fn (array $index) => new Document($index), TimeLimit::INDEXES); + $dbForProject->createDocument(Database::METADATA, new Document([ + '$id' => ID::custom('abuse'), + '$permissions' => [Permission::create(Role::any())], + 'name' => 'abuse', + 'attributes' => $attributes, + 'indexes' => $indexes, + 'documentSecurity' => true + ])); } if ($create || $sharedTablesV1) { diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 7752f2f8a9..a0f65eb484 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -19,6 +19,7 @@ use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Utopia\Abuse\Abuse; +use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\App; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; @@ -419,9 +420,8 @@ App::init() ->inject('queueForBuilds') ->inject('queueForUsage') ->inject('dbForProject') - ->inject('timelimit') ->inject('mode') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, callable $timelimit, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -444,7 +444,7 @@ App::init() foreach ($abuseKeyLabel as $abuseKey) { $start = $request->getContentRangeStart(); $end = $request->getContentRangeEnd(); - $timeLimit = $timelimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600)); + $timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $dbForProject); $timeLimit ->setParam('{projectId}', $project->getId()) ->setParam('{userId}', $user->getId()) diff --git a/app/http.php b/app/http.php index 61afce3eae..74b829c384 100644 --- a/app/http.php +++ b/app/http.php @@ -10,8 +10,10 @@ use Swoole\Http\Response as SwooleResponse; use Swoole\Http\Server; use Swoole\Process; use Swoole\Table; +use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\App; use Utopia\Audit\Audit; +use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; @@ -198,6 +200,11 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg $audit->setup(); } + if ($dbForPlatform->getCollection(TimeLimit::COLLECTION)->isEmpty()) { + $adapter = new TimeLimit("", 0, 1, $dbForPlatform); + $adapter->setup(); + } + /** @var array $collections */ $collections = Config::getParam('collections', []); $consoleCollections = $collections['console']; diff --git a/app/init.php b/app/init.php index a738a44577..109849fdce 100644 --- a/app/init.php +++ b/app/init.php @@ -48,7 +48,6 @@ use Appwrite\Utopia\Request; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; -use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Cache\Adapter\Sharding; @@ -854,31 +853,31 @@ $register->set('pools', function () { $connections = [ 'console' => [ 'type' => 'database', - 'dsns' => $fallbackForDB, + 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), 'multiple' => false, 'schemes' => ['mariadb', 'mysql'], ], 'database' => [ 'type' => 'database', - 'dsns' => $fallbackForDB, + 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), 'multiple' => true, 'schemes' => ['mariadb', 'mysql'], ], 'queue' => [ 'type' => 'queue', - 'dsns' => $fallbackForRedis, + 'dsns' => System::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), 'multiple' => false, 'schemes' => ['redis'], ], 'pubsub' => [ 'type' => 'pubsub', - 'dsns' => $fallbackForRedis, + 'dsns' => System::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), 'multiple' => false, 'schemes' => ['redis'], ], 'cache' => [ 'type' => 'cache', - 'dsns' => $fallbackForRedis, + 'dsns' => System::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), 'multiple' => true, 'schemes' => ['redis'], ], @@ -950,12 +949,12 @@ $register->set('pools', function () { }); }, 'redis' => function () use ($dsnHost, $dsnPort, $dsnPass) { - $redis = new \Redis(); + $redis = new Redis(); @$redis->pconnect($dsnHost, (int)$dsnPort); if ($dsnPass) { $redis->auth($dsnPass); } - $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); return $redis; }, @@ -1532,27 +1531,6 @@ App::setResource('cache', function (Group $pools) { return new Cache(new Sharding($adapters)); }, ['pools']); -App::setResource('redis', function () { - $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); - $port = System::getEnv('_APP_REDIS_PORT', 6379); - $pass = System::getEnv('_APP_REDIS_PASS', ''); - - $redis = new \Redis(); - @$redis->pconnect($host, (int)$port); - if ($pass) { - $redis->auth($pass); - } - $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); - - return $redis; -}); - -App::setResource('timelimit', function (\Redis $redis) { - return function (string $key, int $limit, int $time) use ($redis) { - return new TimeLimitRedis($key, $limit, $time, $redis); - }; -}, ['redis']); - App::setResource('deviceForLocal', function () { return new Local(); }); diff --git a/app/realtime.php b/app/realtime.php index 4f87e4dea1..54fd1e05f7 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -13,7 +13,7 @@ use Swoole\Runtime; use Swoole\Table; use Swoole\Timer; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; +use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; @@ -138,32 +138,6 @@ if (!function_exists('getCache')) { } } -// Allows overriding -if (!function_exists('getRedis')) { - function getRedis(): \Redis - { - $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); - $port = System::getEnv('_APP_REDIS_PORT', 6379); - $pass = System::getEnv('_APP_REDIS_PASS', ''); - - $redis = new \Redis(); - @$redis->pconnect($host, (int)$port); - if ($pass) { - $redis->auth($pass); - } - $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); - - return $redis; - } -} - -if (!function_exists('getTimelimit')) { - function getTimelimit(): TimeLimitRedis - { - return new TimeLimitRedis("", 0, 1, getRedis()); - } -} - if (!function_exists('getRealtime')) { function getRealtime(): Realtime { @@ -507,7 +481,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new AppwriteException(AppwriteException::GENERAL_API_DISABLED); } - $timelimit = $app->getResource('timelimit'); + $dbForProject = getProjectDB($project); $console = $app->getResource('console'); /** @var Document $console */ $user = $app->getResource('user'); /** @var Document $user */ @@ -516,12 +490,12 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, * * Abuse limits are connecting 128 times per minute and ip address. */ - $timelimit = $timelimit('url:{url},ip:{ip}', 128, 60); - $timelimit + $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $dbForProject); + $timeLimit ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getURI()); - $abuse = new Abuse($timelimit); + $abuse = new Abuse($timeLimit); if (System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled' && $abuse->check()) { throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many requests'); @@ -619,7 +593,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re * * Abuse limits are sending 32 times per minute and connection. */ - $timeLimit = getTimelimit('url:{url},connection:{connection}', 32, 60); + $timeLimit = new TimeLimit('url:{url},connection:{connection}', 32, 60, $database); $timeLimit ->setParam('{connection}', $connection) diff --git a/app/worker.php b/app/worker.php index 7539a26879..2c7d8acb22 100644 --- a/app/worker.php +++ b/app/worker.php @@ -17,7 +17,6 @@ use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; -use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -201,27 +200,6 @@ Server::setResource('cache', function (Registry $register) { return new Cache(new Sharding($adapters)); }, ['register']); -Server::setResource('redis', function () { - $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); - $port = System::getEnv('_APP_REDIS_PORT', 6379); - $pass = System::getEnv('_APP_REDIS_PASS', ''); - - $redis = new \Redis(); - @$redis->pconnect($host, (int)$port); - if ($pass) { - $redis->auth($pass); - } - $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); - - return $redis; -}); - -Server::setResource('timelimit', function (\Redis $redis) { - return function (string $key, int $limit, int $time) use ($redis) { - return new TimeLimitRedis($key, $limit, $time, $redis); - }; -}, ['redis']); - Server::setResource('log', fn () => new Log()); Server::setResource('queueForUsage', function (Connection $queue) { diff --git a/composer.json b/composer.json index a8a54c75ab..512e203a5e 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.16.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "0.44.*", + "utopia-php/abuse": "0.43.*", "utopia-php/analytics": "0.10.*", "utopia-php/audit": "0.43.*", "utopia-php/cache": "0.11.*", diff --git a/composer.lock b/composer.lock index 09f9b3aed9..732bafd219 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fd1cf738be4e037abdc1a0fa4cb27a3c", + "content-hash": "fae350df93342992edd8f639948e1570", "packages": [ { "name": "adhocore/jwt", @@ -709,16 +709,16 @@ }, { "name": "google/protobuf", - "version": "v4.29.2", + "version": "v4.29.1", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "79aa5014efeeec3d137df5cdb0ae2fc163953945" + "reference": "6042b5483f8029e42473faeb8ef75ba266278381" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/79aa5014efeeec3d137df5cdb0ae2fc163953945", - "reference": "79aa5014efeeec3d137df5cdb0ae2fc163953945", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/6042b5483f8029e42473faeb8ef75ba266278381", + "reference": "6042b5483f8029e42473faeb8ef75ba266278381", "shasum": "" }, "require": { @@ -747,9 +747,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.2" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.1" }, - "time": "2024-12-18T14:11:12+00:00" + "time": "2024-12-03T22:07:45+00:00" }, { "name": "jean85/pretty-package-versions", @@ -1237,16 +1237,16 @@ }, { "name": "open-telemetry/api", - "version": "1.1.2", + "version": "1.1.1", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "04c85a1e41a3d59fa9bdc801a5de1df6624b95ed" + "reference": "542064815d38a6df55af7957cd6f1d7d967c99c6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/04c85a1e41a3d59fa9bdc801a5de1df6624b95ed", - "reference": "04c85a1e41a3d59fa9bdc801a5de1df6624b95ed", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/542064815d38a6df55af7957cd6f1d7d967c99c6", + "reference": "542064815d38a6df55af7957cd6f1d7d967c99c6", "shasum": "" }, "require": { @@ -1260,13 +1260,13 @@ }, "type": "library", "extra": { + "branch-alias": { + "dev-main": "1.1.x-dev" + }, "spi": { "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\HookManagerInterface": [ "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\ExtensionHookManager" ] - }, - "branch-alias": { - "dev-main": "1.1.x-dev" } }, "autoload": { @@ -1303,7 +1303,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2024-11-16T04:32:30+00:00" + "time": "2024-10-15T22:42:37+00:00" }, { "name": "open-telemetry/context", @@ -1530,13 +1530,13 @@ }, "type": "library", "extra": { + "branch-alias": { + "dev-main": "1.0.x-dev" + }, "spi": { "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\HookManagerInterface": [ "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\ExtensionHookManager" ] - }, - "branch-alias": { - "dev-main": "1.0.x-dev" } }, "autoload": { @@ -2453,23 +2453,23 @@ }, { "name": "symfony/http-client", - "version": "v7.2.1", + "version": "v7.2.0", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e" + "reference": "955e43336aff03df1e8a8e17daefabb0127a313b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/ff4df2b68d1c67abb9fef146e6540ea16b58d99e", - "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e", + "url": "https://api.github.com/repos/symfony/http-client/zipball/955e43336aff03df1e8a8e17daefabb0127a313b", + "reference": "955e43336aff03df1e8a8e17daefabb0127a313b", "shasum": "" }, "require": { "php": ">=8.2", "psr/log": "^1|^2|^3", "symfony/deprecation-contracts": "^2.5|^3", - "symfony/http-client-contracts": "~3.4.4|^3.5.2", + "symfony/http-client-contracts": "~3.4.3|^3.5.1", "symfony/service-contracts": "^2.5|^3" }, "conflict": { @@ -2528,7 +2528,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.2.1" + "source": "https://github.com/symfony/http-client/tree/v7.2.0" }, "funding": [ { @@ -2544,20 +2544,20 @@ "type": "tidelift" } ], - "time": "2024-12-07T08:50:44+00:00" + "time": "2024-11-29T08:22:02+00:00" }, { "name": "symfony/http-client-contracts", - "version": "v3.5.2", + "version": "v3.5.1", "source": { "type": "git", "url": "https://github.com/symfony/http-client-contracts.git", - "reference": "ee8d807ab20fcb51267fdace50fbe3494c31e645" + "reference": "c2f3ad828596624ca39ea40f83617ef51ca8bbf9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/ee8d807ab20fcb51267fdace50fbe3494c31e645", - "reference": "ee8d807ab20fcb51267fdace50fbe3494c31e645", + "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/c2f3ad828596624ca39ea40f83617ef51ca8bbf9", + "reference": "c2f3ad828596624ca39ea40f83617ef51ca8bbf9", "shasum": "" }, "require": { @@ -2606,7 +2606,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/http-client-contracts/tree/v3.5.2" + "source": "https://github.com/symfony/http-client-contracts/tree/v3.5.1" }, "funding": [ { @@ -2622,7 +2622,7 @@ "type": "tidelift" } ], - "time": "2024-12-07T08:49:48+00:00" + "time": "2024-11-25T12:02:18+00:00" }, { "name": "symfony/polyfill-mbstring", @@ -2650,8 +2650,8 @@ "type": "library", "extra": { "thanks": { - "url": "https://github.com/symfony/polyfill", - "name": "symfony/polyfill" + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" } }, "autoload": { @@ -3136,16 +3136,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.44.0", + "version": "0.43.2", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "25b9654ddfb1a17161debf1d26f2915e0128d9fd" + "reference": "374536b86d8d39066960a7da161d444a099bbc56" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/25b9654ddfb1a17161debf1d26f2915e0128d9fd", - "reference": "25b9654ddfb1a17161debf1d26f2915e0128d9fd", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/374536b86d8d39066960a7da161d444a099bbc56", + "reference": "374536b86d8d39066960a7da161d444a099bbc56", "shasum": "" }, "require": { @@ -3181,9 +3181,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.44.0" + "source": "https://github.com/utopia-php/abuse/tree/0.43.2" }, - "time": "2024-12-18T15:09:47+00:00" + "time": "2024-12-12T19:43:24+00:00" }, { "name": "utopia-php/analytics", @@ -4807,16 +4807,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.39.27", + "version": "0.39.25", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "27d8ecde30e40cbfe1124cc0430c406d3e144849" + "reference": "5b5323636a8d75a1c4faaae9728098dd6a6a47d1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/27d8ecde30e40cbfe1124cc0430c406d3e144849", - "reference": "27d8ecde30e40cbfe1124cc0430c406d3e144849", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/5b5323636a8d75a1c4faaae9728098dd6a6a47d1", + "reference": "5b5323636a8d75a1c4faaae9728098dd6a6a47d1", "shasum": "" }, "require": { @@ -4852,9 +4852,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.39.27" + "source": "https://github.com/appwrite/sdk-generator/tree/0.39.25" }, - "time": "2024-12-16T11:32:02+00:00" + "time": "2024-11-08T10:16:34+00:00" }, { "name": "doctrine/annotations", @@ -7576,16 +7576,16 @@ }, { "name": "symfony/console", - "version": "v7.2.1", + "version": "v7.2.0", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3" + "reference": "23c8aae6d764e2bae02d2a99f7532a7f6ed619cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/fefcc18c0f5d0efe3ab3152f15857298868dc2c3", - "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3", + "url": "https://api.github.com/repos/symfony/console/zipball/23c8aae6d764e2bae02d2a99f7532a7f6ed619cf", + "reference": "23c8aae6d764e2bae02d2a99f7532a7f6ed619cf", "shasum": "" }, "require": { @@ -7649,7 +7649,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.2.1" + "source": "https://github.com/symfony/console/tree/v7.2.0" }, "funding": [ { @@ -7665,7 +7665,7 @@ "type": "tidelift" } ], - "time": "2024-12-11T03:49:26+00:00" + "time": "2024-11-06T14:24:19+00:00" }, { "name": "symfony/filesystem", diff --git a/phpunit.xml b/phpunit.xml index 598b730908..4c4e55ea4e 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -6,7 +6,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="true" + stopOnFailure="false" > diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index bbc808e248..f286ed4c4c 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -8,6 +8,7 @@ use Appwrite\Extend\Exception; use Executor\Executor; use Throwable; use Utopia\Abuse\Abuse; +use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\Audit\Audit; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; @@ -46,7 +47,6 @@ class Deletes extends Action ->inject('message') ->inject('dbForPlatform') ->inject('getProjectDB') - ->inject('timelimit') ->inject('deviceForFiles') ->inject('deviceForFunctions') ->inject('deviceForBuilds') @@ -57,8 +57,8 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $dbForPlatform, $getProjectDB, $timelimit, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) + fn ($message, $dbForPlatform, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForPlatform, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -126,7 +126,7 @@ class Deletes extends Action } break; case DELETE_TYPE_ABUSE: - $this->deleteAbuseLogs($project, $timelimit, $abuseRetention); + $this->deleteAbuseLogs($project, $getProjectDB, $abuseRetention); break; case DELETE_TYPE_REALTIME: $this->deleteRealtimeUsage($dbForPlatform, $datetime); @@ -494,7 +494,8 @@ class Deletes extends Action $projectCollectionIds = [ ...\array_keys(Config::getParam('collections', [])['projects']), - Audit::COLLECTION + Audit::COLLECTION, + TimeLimit::COLLECTION, ]; $limit = \count($projectCollectionIds) + 25; @@ -707,10 +708,11 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteAbuseLogs(Document $project, callable $timelimit, string $abuseRetention): void + private function deleteAbuseLogs(Document $project, callable $getProjectDB, string $abuseRetention): void { $projectId = $project->getId(); - $timeLimit = $timelimit("", 0, 1); + $dbForProject = $getProjectDB($project); + $timeLimit = new TimeLimit("", 0, 1, $dbForProject); $abuse = new Abuse($timeLimit); try { From 718edd37f7cec2190f440491474250337478a4e4 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 19 Dec 2024 21:15:13 +0530 Subject: [PATCH 32/41] chore: fix attribute name --- src/Appwrite/Event/Webhook.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Event/Webhook.php b/src/Appwrite/Event/Webhook.php index 954e8df1fd..3b88fd4c54 100644 --- a/src/Appwrite/Event/Webhook.php +++ b/src/Appwrite/Event/Webhook.php @@ -22,7 +22,7 @@ class Webhook extends Event $this->context = []; $this->project = new Document([ '$id' => $this->project->getId(), - '$internalId' => $this->project->getAttribute('internalId'), + '$internalId' => $this->project->getAttribute('$internalId'), ]); return parent::trigger(); From e2bdb46d4836964cb8c0bf94e5c2c59c1acff505 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 19 Dec 2024 21:17:27 +0530 Subject: [PATCH 33/41] chore: fix attribute name --- src/Appwrite/Event/Webhook.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Event/Webhook.php b/src/Appwrite/Event/Webhook.php index 3b88fd4c54..36c6923cae 100644 --- a/src/Appwrite/Event/Webhook.php +++ b/src/Appwrite/Event/Webhook.php @@ -22,7 +22,7 @@ class Webhook extends Event $this->context = []; $this->project = new Document([ '$id' => $this->project->getId(), - '$internalId' => $this->project->getAttribute('$internalId'), + '$internalId' => $this->project->getInternalId(), ]); return parent::trigger(); From 50601bd454d177e1c9ff714f78f7ddd2928fbdcb Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Dec 2024 20:14:50 +0530 Subject: [PATCH 34/41] chore: use redis adapter for abuse --- app/controllers/api/functions.php | 10 +- app/controllers/api/projects.php | 15 --- app/controllers/shared/api.php | 8 +- app/http.php | 7 -- app/init.php | 36 ++++++-- app/realtime.php | 38 ++++++-- app/worker.php | 22 +++++ composer.json | 2 +- composer.lock | 106 +++++++++++----------- src/Appwrite/Platform/Workers/Deletes.php | 18 ++-- 10 files changed, 155 insertions(+), 107 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 5934258037..d4d5fc64cf 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -24,7 +24,6 @@ use Appwrite\Utopia\Response\Model\Rule; use Executor\Executor; use MaxMind\Db\Reader; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\App; use Utopia\CLI\Console; use Utopia\Config\Config; @@ -180,22 +179,23 @@ App::post('/v1/functions') ->inject('request') ->inject('response') ->inject('dbForProject') + ->inject('timelimit') ->inject('project') ->inject('user') ->inject('queueForEvents') ->inject('queueForBuilds') ->inject('dbForPlatform') ->inject('gitHub') - ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { + ->action(function (string $functionId, string $name, string $runtime, array $execute, array $events, string $schedule, int $timeout, bool $enabled, bool $logging, string $entrypoint, string $commands, array $scopes, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, callable $timelimit, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForPlatform, GitHub $github) use ($redeployVcs) { $functionId = ($functionId == 'unique()') ? ID::unique() : $functionId; // Temporary abuse check - $abuseCheck = function () use ($project, $dbForProject, $response) { + $abuseCheck = function () use ($project, $timelimit, $response) { $abuseKey = "projectId:{projectId},url:{url}"; $abuseLimit = App::getEnv('_APP_FUNCTIONS_CREATION_ABUSE_LIMIT', 50); $abuseTime = 86400; // 1 day - $timeLimit = new TimeLimit($abuseKey, $abuseLimit, $abuseTime, $dbForProject); + $timeLimit = $timelimit($abuseKey, $abuseLimit, $abuseTime); $timeLimit ->setParam('{projectId}', $project->getId()) ->setParam('{url}', '/v1/functions'); @@ -203,7 +203,7 @@ App::post('/v1/functions') $abuse = new Abuse($timeLimit); $remaining = $timeLimit->remaining(); $limit = $timeLimit->limit(); - $time = (new \DateTime($timeLimit->time()))->getTimestamp() + $abuseTime; + $time = $timeLimit->time() + $abuseTime; $response ->addHeader('X-RateLimit-Limit', $limit) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 388c9e22ea..cbda6ad544 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -16,7 +16,6 @@ use Appwrite\Utopia\Database\Validator\Queries\Projects; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use PHPMailer\PHPMailer\PHPMailer; -use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\App; use Utopia\Audit\Audit; use Utopia\Cache\Cache; @@ -229,9 +228,6 @@ App::post('/v1/projects') if ($create || $projectTables) { $audit = new Audit($dbForProject); $audit->setup(); - - $abuse = new TimeLimit('', 0, 1, $dbForProject); - $abuse->setup(); } if (!$create && $sharedTablesV1) { @@ -245,17 +241,6 @@ App::post('/v1/projects') 'indexes' => $indexes, 'documentSecurity' => true ])); - - $attributes = \array_map(fn ($attribute) => new Document($attribute), TimeLimit::ATTRIBUTES); - $indexes = \array_map(fn (array $index) => new Document($index), TimeLimit::INDEXES); - $dbForProject->createDocument(Database::METADATA, new Document([ - '$id' => ID::custom('abuse'), - '$permissions' => [Permission::create(Role::any())], - 'name' => 'abuse', - 'attributes' => $attributes, - 'indexes' => $indexes, - 'documentSecurity' => true - ])); } if ($create || $sharedTablesV1) { diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index a0f65eb484..012dd13c73 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -19,7 +19,6 @@ use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\App; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; @@ -420,8 +419,9 @@ App::init() ->inject('queueForBuilds') ->inject('queueForUsage') ->inject('dbForProject') + ->inject('timelimit') ->inject('mode') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Connection $queue, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Usage $queueForUsage, Database $dbForProject, callable $timelimit, string $mode) use ($usageDatabaseListener, $eventDatabaseListener) { $route = $utopia->getRoute(); @@ -444,7 +444,7 @@ App::init() foreach ($abuseKeyLabel as $abuseKey) { $start = $request->getContentRangeStart(); $end = $request->getContentRangeEnd(); - $timeLimit = new TimeLimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600), $dbForProject); + $timeLimit = $timelimit($abuseKey, $route->getLabel('abuse-limit', 0), $route->getLabel('abuse-time', 3600)); $timeLimit ->setParam('{projectId}', $project->getId()) ->setParam('{userId}', $user->getId()) @@ -472,7 +472,7 @@ App::init() $abuse = new Abuse($timeLimit); $remaining = $timeLimit->remaining(); $limit = $timeLimit->limit(); - $time = (new \DateTime($timeLimit->time()))->getTimestamp() + $route->getLabel('abuse-time', 3600); + $time = $timeLimit->time() + $route->getLabel('abuse-time', 3600); if ($limit && ($remaining < $closestLimit || is_null($closestLimit))) { $closestLimit = $remaining; diff --git a/app/http.php b/app/http.php index 74b829c384..61afce3eae 100644 --- a/app/http.php +++ b/app/http.php @@ -10,10 +10,8 @@ use Swoole\Http\Response as SwooleResponse; use Swoole\Http\Server; use Swoole\Process; use Swoole\Table; -use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\App; use Utopia\Audit\Audit; -use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Config\Config; use Utopia\Database\Database; @@ -200,11 +198,6 @@ $http->on(Constant::EVENT_START, function (Server $http) use ($payloadSize, $reg $audit->setup(); } - if ($dbForPlatform->getCollection(TimeLimit::COLLECTION)->isEmpty()) { - $adapter = new TimeLimit("", 0, 1, $dbForPlatform); - $adapter->setup(); - } - /** @var array $collections */ $collections = Config::getParam('collections', []); $consoleCollections = $collections['console']; diff --git a/app/init.php b/app/init.php index 109849fdce..a738a44577 100644 --- a/app/init.php +++ b/app/init.php @@ -48,6 +48,7 @@ use Appwrite\Utopia\Request; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; +use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Cache\Adapter\Sharding; @@ -853,31 +854,31 @@ $register->set('pools', function () { $connections = [ 'console' => [ 'type' => 'database', - 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_CONSOLE', $fallbackForDB), + 'dsns' => $fallbackForDB, 'multiple' => false, 'schemes' => ['mariadb', 'mysql'], ], 'database' => [ 'type' => 'database', - 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_PROJECT', $fallbackForDB), + 'dsns' => $fallbackForDB, 'multiple' => true, 'schemes' => ['mariadb', 'mysql'], ], 'queue' => [ 'type' => 'queue', - 'dsns' => System::getEnv('_APP_CONNECTIONS_QUEUE', $fallbackForRedis), + 'dsns' => $fallbackForRedis, 'multiple' => false, 'schemes' => ['redis'], ], 'pubsub' => [ 'type' => 'pubsub', - 'dsns' => System::getEnv('_APP_CONNECTIONS_PUBSUB', $fallbackForRedis), + 'dsns' => $fallbackForRedis, 'multiple' => false, 'schemes' => ['redis'], ], 'cache' => [ 'type' => 'cache', - 'dsns' => System::getEnv('_APP_CONNECTIONS_CACHE', $fallbackForRedis), + 'dsns' => $fallbackForRedis, 'multiple' => true, 'schemes' => ['redis'], ], @@ -949,12 +950,12 @@ $register->set('pools', function () { }); }, 'redis' => function () use ($dsnHost, $dsnPort, $dsnPass) { - $redis = new Redis(); + $redis = new \Redis(); @$redis->pconnect($dsnHost, (int)$dsnPort); if ($dsnPass) { $redis->auth($dsnPass); } - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); return $redis; }, @@ -1531,6 +1532,27 @@ App::setResource('cache', function (Group $pools) { return new Cache(new Sharding($adapters)); }, ['pools']); +App::setResource('redis', function () { + $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); + $port = System::getEnv('_APP_REDIS_PORT', 6379); + $pass = System::getEnv('_APP_REDIS_PASS', ''); + + $redis = new \Redis(); + @$redis->pconnect($host, (int)$port); + if ($pass) { + $redis->auth($pass); + } + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); + + return $redis; +}); + +App::setResource('timelimit', function (\Redis $redis) { + return function (string $key, int $limit, int $time) use ($redis) { + return new TimeLimitRedis($key, $limit, $time, $redis); + }; +}, ['redis']); + App::setResource('deviceForLocal', function () { return new Local(); }); diff --git a/app/realtime.php b/app/realtime.php index 54fd1e05f7..4f87e4dea1 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -13,7 +13,7 @@ use Swoole\Runtime; use Swoole\Table; use Swoole\Timer; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\Database\TimeLimit; +use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; @@ -138,6 +138,32 @@ if (!function_exists('getCache')) { } } +// Allows overriding +if (!function_exists('getRedis')) { + function getRedis(): \Redis + { + $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); + $port = System::getEnv('_APP_REDIS_PORT', 6379); + $pass = System::getEnv('_APP_REDIS_PASS', ''); + + $redis = new \Redis(); + @$redis->pconnect($host, (int)$port); + if ($pass) { + $redis->auth($pass); + } + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); + + return $redis; + } +} + +if (!function_exists('getTimelimit')) { + function getTimelimit(): TimeLimitRedis + { + return new TimeLimitRedis("", 0, 1, getRedis()); + } +} + if (!function_exists('getRealtime')) { function getRealtime(): Realtime { @@ -481,7 +507,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new AppwriteException(AppwriteException::GENERAL_API_DISABLED); } - $dbForProject = getProjectDB($project); + $timelimit = $app->getResource('timelimit'); $console = $app->getResource('console'); /** @var Document $console */ $user = $app->getResource('user'); /** @var Document $user */ @@ -490,12 +516,12 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, * * Abuse limits are connecting 128 times per minute and ip address. */ - $timeLimit = new TimeLimit('url:{url},ip:{ip}', 128, 60, $dbForProject); - $timeLimit + $timelimit = $timelimit('url:{url},ip:{ip}', 128, 60); + $timelimit ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getURI()); - $abuse = new Abuse($timeLimit); + $abuse = new Abuse($timelimit); if (System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled' && $abuse->check()) { throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many requests'); @@ -593,7 +619,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re * * Abuse limits are sending 32 times per minute and connection. */ - $timeLimit = new TimeLimit('url:{url},connection:{connection}', 32, 60, $database); + $timeLimit = getTimelimit('url:{url},connection:{connection}', 32, 60); $timeLimit ->setParam('{connection}', $connection) diff --git a/app/worker.php b/app/worker.php index 2c7d8acb22..7539a26879 100644 --- a/app/worker.php +++ b/app/worker.php @@ -17,6 +17,7 @@ use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; +use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -200,6 +201,27 @@ Server::setResource('cache', function (Registry $register) { return new Cache(new Sharding($adapters)); }, ['register']); +Server::setResource('redis', function () { + $host = System::getEnv('_APP_REDIS_HOST', 'localhost'); + $port = System::getEnv('_APP_REDIS_PORT', 6379); + $pass = System::getEnv('_APP_REDIS_PASS', ''); + + $redis = new \Redis(); + @$redis->pconnect($host, (int)$port); + if ($pass) { + $redis->auth($pass); + } + $redis->setOption(\Redis::OPT_READ_TIMEOUT, -1); + + return $redis; +}); + +Server::setResource('timelimit', function (\Redis $redis) { + return function (string $key, int $limit, int $time) use ($redis) { + return new TimeLimitRedis($key, $limit, $time, $redis); + }; +}, ['redis']); + Server::setResource('log', fn () => new Log()); Server::setResource('queueForUsage', function (Connection $queue) { diff --git a/composer.json b/composer.json index 512e203a5e..5f81fd7752 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.16.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "0.43.*", + "utopia-php/abuse": "dev-add-support-for-sharding-adapter", "utopia-php/analytics": "0.10.*", "utopia-php/audit": "0.43.*", "utopia-php/cache": "0.11.*", diff --git a/composer.lock b/composer.lock index 732bafd219..371a206a2c 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fae350df93342992edd8f639948e1570", + "content-hash": "d2345c93c578d7ac06f364fc5a175810", "packages": [ { "name": "adhocore/jwt", @@ -709,16 +709,16 @@ }, { "name": "google/protobuf", - "version": "v4.29.1", + "version": "v4.29.2", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "6042b5483f8029e42473faeb8ef75ba266278381" + "reference": "79aa5014efeeec3d137df5cdb0ae2fc163953945" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/6042b5483f8029e42473faeb8ef75ba266278381", - "reference": "6042b5483f8029e42473faeb8ef75ba266278381", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/79aa5014efeeec3d137df5cdb0ae2fc163953945", + "reference": "79aa5014efeeec3d137df5cdb0ae2fc163953945", "shasum": "" }, "require": { @@ -747,9 +747,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.1" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.2" }, - "time": "2024-12-03T22:07:45+00:00" + "time": "2024-12-18T14:11:12+00:00" }, { "name": "jean85/pretty-package-versions", @@ -1237,16 +1237,16 @@ }, { "name": "open-telemetry/api", - "version": "1.1.1", + "version": "1.1.2", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "542064815d38a6df55af7957cd6f1d7d967c99c6" + "reference": "04c85a1e41a3d59fa9bdc801a5de1df6624b95ed" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/542064815d38a6df55af7957cd6f1d7d967c99c6", - "reference": "542064815d38a6df55af7957cd6f1d7d967c99c6", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/04c85a1e41a3d59fa9bdc801a5de1df6624b95ed", + "reference": "04c85a1e41a3d59fa9bdc801a5de1df6624b95ed", "shasum": "" }, "require": { @@ -1260,13 +1260,13 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "1.1.x-dev" - }, "spi": { "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\HookManagerInterface": [ "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\ExtensionHookManager" ] + }, + "branch-alias": { + "dev-main": "1.1.x-dev" } }, "autoload": { @@ -1303,7 +1303,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2024-10-15T22:42:37+00:00" + "time": "2024-11-16T04:32:30+00:00" }, { "name": "open-telemetry/context", @@ -1530,13 +1530,13 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "1.0.x-dev" - }, "spi": { "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\HookManagerInterface": [ "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\ExtensionHookManager" ] + }, + "branch-alias": { + "dev-main": "1.0.x-dev" } }, "autoload": { @@ -2453,23 +2453,23 @@ }, { "name": "symfony/http-client", - "version": "v7.2.0", + "version": "v7.2.1", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "955e43336aff03df1e8a8e17daefabb0127a313b" + "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/955e43336aff03df1e8a8e17daefabb0127a313b", - "reference": "955e43336aff03df1e8a8e17daefabb0127a313b", + "url": "https://api.github.com/repos/symfony/http-client/zipball/ff4df2b68d1c67abb9fef146e6540ea16b58d99e", + "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e", "shasum": "" }, "require": { "php": ">=8.2", "psr/log": "^1|^2|^3", "symfony/deprecation-contracts": "^2.5|^3", - "symfony/http-client-contracts": "~3.4.3|^3.5.1", + "symfony/http-client-contracts": "~3.4.4|^3.5.2", "symfony/service-contracts": "^2.5|^3" }, "conflict": { @@ -2528,7 +2528,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.2.0" + "source": "https://github.com/symfony/http-client/tree/v7.2.1" }, "funding": [ { @@ -2544,20 +2544,20 @@ "type": "tidelift" } ], - "time": "2024-11-29T08:22:02+00:00" + "time": "2024-12-07T08:50:44+00:00" }, { "name": "symfony/http-client-contracts", - "version": "v3.5.1", + "version": "v3.5.2", "source": { "type": "git", "url": "https://github.com/symfony/http-client-contracts.git", - "reference": "c2f3ad828596624ca39ea40f83617ef51ca8bbf9" + "reference": "ee8d807ab20fcb51267fdace50fbe3494c31e645" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/c2f3ad828596624ca39ea40f83617ef51ca8bbf9", - "reference": "c2f3ad828596624ca39ea40f83617ef51ca8bbf9", + "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/ee8d807ab20fcb51267fdace50fbe3494c31e645", + "reference": "ee8d807ab20fcb51267fdace50fbe3494c31e645", "shasum": "" }, "require": { @@ -2606,7 +2606,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/http-client-contracts/tree/v3.5.1" + "source": "https://github.com/symfony/http-client-contracts/tree/v3.5.2" }, "funding": [ { @@ -2622,7 +2622,7 @@ "type": "tidelift" } ], - "time": "2024-11-25T12:02:18+00:00" + "time": "2024-12-07T08:49:48+00:00" }, { "name": "symfony/polyfill-mbstring", @@ -2650,8 +2650,8 @@ "type": "library", "extra": { "thanks": { - "name": "symfony/polyfill", - "url": "https://github.com/symfony/polyfill" + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" } }, "autoload": { @@ -3136,16 +3136,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.43.2", + "version": "dev-add-support-for-sharding-adapter", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "374536b86d8d39066960a7da161d444a099bbc56" + "reference": "d3119238ea05edd59f93ec99fb11545070e86eff" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/374536b86d8d39066960a7da161d444a099bbc56", - "reference": "374536b86d8d39066960a7da161d444a099bbc56", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/d3119238ea05edd59f93ec99fb11545070e86eff", + "reference": "d3119238ea05edd59f93ec99fb11545070e86eff", "shasum": "" }, "require": { @@ -3181,9 +3181,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.43.2" + "source": "https://github.com/utopia-php/abuse/tree/add-support-for-sharding-adapter" }, - "time": "2024-12-12T19:43:24+00:00" + "time": "2024-12-20T14:11:20+00:00" }, { "name": "utopia-php/analytics", @@ -4807,16 +4807,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.39.25", + "version": "0.39.27", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "5b5323636a8d75a1c4faaae9728098dd6a6a47d1" + "reference": "27d8ecde30e40cbfe1124cc0430c406d3e144849" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/5b5323636a8d75a1c4faaae9728098dd6a6a47d1", - "reference": "5b5323636a8d75a1c4faaae9728098dd6a6a47d1", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/27d8ecde30e40cbfe1124cc0430c406d3e144849", + "reference": "27d8ecde30e40cbfe1124cc0430c406d3e144849", "shasum": "" }, "require": { @@ -4852,9 +4852,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.39.25" + "source": "https://github.com/appwrite/sdk-generator/tree/0.39.27" }, - "time": "2024-11-08T10:16:34+00:00" + "time": "2024-12-16T11:32:02+00:00" }, { "name": "doctrine/annotations", @@ -7576,16 +7576,16 @@ }, { "name": "symfony/console", - "version": "v7.2.0", + "version": "v7.2.1", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "23c8aae6d764e2bae02d2a99f7532a7f6ed619cf" + "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/23c8aae6d764e2bae02d2a99f7532a7f6ed619cf", - "reference": "23c8aae6d764e2bae02d2a99f7532a7f6ed619cf", + "url": "https://api.github.com/repos/symfony/console/zipball/fefcc18c0f5d0efe3ab3152f15857298868dc2c3", + "reference": "fefcc18c0f5d0efe3ab3152f15857298868dc2c3", "shasum": "" }, "require": { @@ -7649,7 +7649,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.2.0" + "source": "https://github.com/symfony/console/tree/v7.2.1" }, "funding": [ { @@ -7665,7 +7665,7 @@ "type": "tidelift" } ], - "time": "2024-11-06T14:24:19+00:00" + "time": "2024-12-11T03:49:26+00:00" }, { "name": "symfony/filesystem", @@ -8556,7 +8556,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/abuse": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index f286ed4c4c..bbc808e248 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -8,7 +8,6 @@ use Appwrite\Extend\Exception; use Executor\Executor; use Throwable; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\Database\TimeLimit; use Utopia\Audit\Audit; use Utopia\Cache\Adapter\Filesystem; use Utopia\Cache\Cache; @@ -47,6 +46,7 @@ class Deletes extends Action ->inject('message') ->inject('dbForPlatform') ->inject('getProjectDB') + ->inject('timelimit') ->inject('deviceForFiles') ->inject('deviceForFunctions') ->inject('deviceForBuilds') @@ -57,8 +57,8 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, $dbForPlatform, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => - $this->action($message, $dbForPlatform, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) + fn ($message, $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + $this->action($message, $dbForPlatform, $getProjectDB, $timelimit, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -126,7 +126,7 @@ class Deletes extends Action } break; case DELETE_TYPE_ABUSE: - $this->deleteAbuseLogs($project, $getProjectDB, $abuseRetention); + $this->deleteAbuseLogs($project, $timelimit, $abuseRetention); break; case DELETE_TYPE_REALTIME: $this->deleteRealtimeUsage($dbForPlatform, $datetime); @@ -494,8 +494,7 @@ class Deletes extends Action $projectCollectionIds = [ ...\array_keys(Config::getParam('collections', [])['projects']), - Audit::COLLECTION, - TimeLimit::COLLECTION, + Audit::COLLECTION ]; $limit = \count($projectCollectionIds) + 25; @@ -708,11 +707,10 @@ class Deletes extends Action * @return void * @throws Exception */ - private function deleteAbuseLogs(Document $project, callable $getProjectDB, string $abuseRetention): void + private function deleteAbuseLogs(Document $project, callable $timelimit, string $abuseRetention): void { $projectId = $project->getId(); - $dbForProject = $getProjectDB($project); - $timeLimit = new TimeLimit("", 0, 1, $dbForProject); + $timeLimit = $timelimit("", 0, 1); $abuse = new Abuse($timeLimit); try { From e9c8766a233d51ccdc6ba77751c9069deee65267 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Dec 2024 21:53:12 +0530 Subject: [PATCH 35/41] chore: update abuse library --- composer.json | 2 +- composer.lock | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/composer.json b/composer.json index 5f81fd7752..2879993d3f 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.16.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "dev-add-support-for-sharding-adapter", + "utopia-php/abuse": "0.45.*", "utopia-php/analytics": "0.10.*", "utopia-php/audit": "0.43.*", "utopia-php/cache": "0.11.*", diff --git a/composer.lock b/composer.lock index 371a206a2c..5bce56a022 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d2345c93c578d7ac06f364fc5a175810", + "content-hash": "db3b63ba1be2561ee4a112e5b9c6bd53", "packages": [ { "name": "adhocore/jwt", @@ -3136,16 +3136,16 @@ }, { "name": "utopia-php/abuse", - "version": "dev-add-support-for-sharding-adapter", + "version": "0.45.0", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "d3119238ea05edd59f93ec99fb11545070e86eff" + "reference": "54a58355df1eb26c9be7834738debe6754e5581d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/d3119238ea05edd59f93ec99fb11545070e86eff", - "reference": "d3119238ea05edd59f93ec99fb11545070e86eff", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/54a58355df1eb26c9be7834738debe6754e5581d", + "reference": "54a58355df1eb26c9be7834738debe6754e5581d", "shasum": "" }, "require": { @@ -3181,9 +3181,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/add-support-for-sharding-adapter" + "source": "https://github.com/utopia-php/abuse/tree/0.45.0" }, - "time": "2024-12-20T14:11:20+00:00" + "time": "2024-12-20T16:18:52+00:00" }, { "name": "utopia-php/analytics", @@ -8556,9 +8556,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/abuse": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From 25953732aba87a838f601ea62bcf44531928f710 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 23 Dec 2024 01:30:20 +0530 Subject: [PATCH 36/41] fix: deletes worker abuse retention type --- app/worker.php | 2 +- src/Appwrite/Platform/Workers/Deletes.php | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/worker.php b/app/worker.php index 7539a26879..6eb1363e9b 100644 --- a/app/worker.php +++ b/app/worker.php @@ -174,7 +174,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf }, ['pools', 'dbForPlatform', 'cache']); Server::setResource('abuseRetention', function () { - return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400)); + return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); }); Server::setResource('auditRetention', function () { diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index bbc808e248..5b6cfc1380 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -57,7 +57,7 @@ class Deletes extends Action ->inject('auditRetention') ->inject('log') ->callback( - fn ($message, $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => + fn ($message, $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, int $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForPlatform, $getProjectDB, $timelimit, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $abuseRetention, $executionRetention, $auditRetention, $log) ); } @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, int $abuseRetention, int $executionRetention, int $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -703,11 +703,11 @@ class Deletes extends Action /** * @param Database $dbForPlatform * @param callable $getProjectDB - * @param string $datetime + * @param int $abuseRetention * @return void * @throws Exception */ - private function deleteAbuseLogs(Document $project, callable $timelimit, string $abuseRetention): void + private function deleteAbuseLogs(Document $project, callable $timelimit, int $abuseRetention): void { $projectId = $project->getId(); $timeLimit = $timelimit("", 0, 1); From cddace70a444fdb2396722960f5b678003c87244 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 23 Dec 2024 01:45:57 +0530 Subject: [PATCH 37/41] fix: deletes worker abuse retention type --- src/Appwrite/Platform/Workers/Deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 5b6cfc1380..f29b79e5db 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -66,7 +66,7 @@ class Deletes extends Action * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, int $abuseRetention, int $executionRetention, int $auditRetention, Log $log): void + public function action(Message $message, Database $dbForPlatform, callable $getProjectDB, callable $timelimit, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, CertificatesAdapter $certificates, int $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; From fe19b0205ab94ef83d97fa1feb3ab49735910d12 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 23 Dec 2024 19:44:10 +0200 Subject: [PATCH 38/41] add webhooks usage stats --- app/init.php | 3 ++- src/Appwrite/Platform/Workers/Webhooks.php | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/app/init.php b/app/init.php index a738a44577..ff797e07c9 100644 --- a/app/init.php +++ b/app/init.php @@ -232,7 +232,8 @@ const API_KEY_DYNAMIC = 'dynamic'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; - +const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; +const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; const METRIC_AUTH_METHOD_PHONE = 'auth.method.phone'; const METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE = METRIC_AUTH_METHOD_PHONE . '.{countryCode}'; const METRIC_MESSAGES = 'messages'; diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php index 271c4c00f0..0ccfa036d8 100644 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Workers; use Appwrite\Event\Mail; +use Appwrite\Event\Usage; use Appwrite\Template\Template; use Exception; use Utopia\Database\Database; @@ -33,8 +34,9 @@ class Webhooks extends Action ->inject('message') ->inject('dbForPlatform') ->inject('queueForMails') + ->inject('queueForUsage') ->inject('log') - ->callback(fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Log $log) => $this->action($message, $dbForPlatform, $queueForMails, $log)); + ->callback(fn (Message $message, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log) => $this->action($message, $dbForPlatform, $queueForMails, $queueForUsage, $log)); } /** @@ -45,7 +47,7 @@ class Webhooks extends Action * @return void * @throws Exception */ - public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Log $log): void + public function action(Message $message, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage, Log $log): void { $this->errors = []; $payload = $message->getPayload() ?? []; @@ -64,7 +66,7 @@ class Webhooks extends Action foreach ($project->getAttribute('webhooks', []) as $webhook) { if (array_intersect($webhook->getAttribute('events', []), $events)) { - $this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails); + $this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $queueForUsage); } } @@ -83,7 +85,7 @@ class Webhooks extends Action * @param Mail $queueForMails * @return void */ - private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails): void + private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, Usage $queueForUsage): void { if ($webhook->getAttribute('enabled') !== true) { return; @@ -166,11 +168,18 @@ class Webhooks extends Action $dbForPlatform->purgeCachedDocument('projects', $project->getId()); $this->errors[] = $logs; + $queueForUsage->addMetric(METRIC_WEBHOOKS_FAILED, 1); + } else { $webhook->setAttribute('attempts', 0); // Reset attempts on success $dbForPlatform->updateDocument('webhooks', $webhook->getId(), $webhook); $dbForPlatform->purgeCachedDocument('projects', $project->getId()); + $queueForUsage->addMetric(METRIC_WEBHOOKS_SENT, 1); } + + $queueForUsage + ->setProject($project) + ->trigger(); } /** From 4b033d8f00946e22d81862a7a05d1f4848060105 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 24 Dec 2024 10:50:09 +0200 Subject: [PATCH 39/41] add webhooks usage stats --- app/init.php | 4 ++++ src/Appwrite/Platform/Workers/Webhooks.php | 11 +++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app/init.php b/app/init.php index ff797e07c9..c6847610c2 100644 --- a/app/init.php +++ b/app/init.php @@ -234,6 +234,10 @@ const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; +const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent'; +const METRIC_WEBHOOK_ID_FAILED = '{webhookInternalId}.webhooks.events.failed'; + + const METRIC_AUTH_METHOD_PHONE = 'auth.method.phone'; const METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE = METRIC_AUTH_METHOD_PHONE . '.{countryCode}'; const METRIC_MESSAGES = 'messages'; diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php index 0ccfa036d8..aa217dd6cf 100644 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -168,13 +168,20 @@ class Webhooks extends Action $dbForPlatform->purgeCachedDocument('projects', $project->getId()); $this->errors[] = $logs; - $queueForUsage->addMetric(METRIC_WEBHOOKS_FAILED, 1); + $queueForUsage + ->addMetric(METRIC_WEBHOOKS_FAILED, 1) + ->addMetric(str_replace('{webhookInternalId}', $webhook->getInternalId(), METRIC_WEBHOOK_ID_FAILED), 1) + ; + } else { $webhook->setAttribute('attempts', 0); // Reset attempts on success $dbForPlatform->updateDocument('webhooks', $webhook->getId(), $webhook); $dbForPlatform->purgeCachedDocument('projects', $project->getId()); - $queueForUsage->addMetric(METRIC_WEBHOOKS_SENT, 1); + $queueForUsage + ->addMetric(METRIC_WEBHOOKS_SENT, 1) + ->addMetric(str_replace('{webhookInternalId}', $webhook->getInternalId(), METRIC_WEBHOOK_ID_SENT), 1) + ; } $queueForUsage From dae5bf593ca9015ed555b11f7387392aca5692ce Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 24 Dec 2024 11:09:41 +0200 Subject: [PATCH 40/41] add webhooks usage stats --- src/Appwrite/Platform/Workers/Webhooks.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php index aa217dd6cf..50f64ea02d 100644 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -171,7 +171,7 @@ class Webhooks extends Action $queueForUsage ->addMetric(METRIC_WEBHOOKS_FAILED, 1) ->addMetric(str_replace('{webhookInternalId}', $webhook->getInternalId(), METRIC_WEBHOOK_ID_FAILED), 1) - ; + ; } else { From 452a58de94d1e08e1ab72c63ec35a47720688b29 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 24 Dec 2024 15:38:01 +0200 Subject: [PATCH 41/41] Reset matrics after enqueueing job via usage event --- src/Appwrite/Event/Usage.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index 161c251c8e..89e900d2ab 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -42,6 +42,7 @@ class Usage extends Event */ public function addMetric(string $key, int $value): self { + $this->metrics[] = [ 'key' => $key, 'value' => $value, @@ -62,10 +63,15 @@ class Usage extends Event } $client = new Client($this->queue, $this->connection); - return $client->enqueue([ + + $result = $client->enqueue([ 'project' => $this->getProject(), 'reduce' => $this->reduce, 'metrics' => $this->metrics, ]); + + $this->metrics = []; + + return $result; } }