From 572c41680a6c2ef9530e468d9a9fffe66a3dceb2 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 17 Dec 2024 20:34:51 +0530 Subject: [PATCH 1/8] feat: add support for redis cluster --- composer.lock | 6 +- docker-compose.yml | 48 ++++++ phpunit.xml | 2 +- src/Abuse/Adapter.php | 4 +- src/Abuse/Adapters/TimeLimit.php | 2 - .../TimeLimit.php => TimeLimit/Database.php} | 28 ++-- .../TimeLimit.php => TimeLimit/Redis.php} | 15 +- src/Abuse/Adapters/TimeLimit/RedisCluster.php | 153 ++++++++++++++++++ tests/Abuse/Base.php | 2 +- tests/Abuse/DatabaseTest.php | 7 +- tests/Abuse/RedisClusterTest.php | 39 +++++ tests/Abuse/RedisTest.php | 7 +- 12 files changed, 276 insertions(+), 37 deletions(-) rename src/Abuse/Adapters/{Database/TimeLimit.php => TimeLimit/Database.php} (87%) rename src/Abuse/Adapters/{Redis/TimeLimit.php => TimeLimit/Redis.php} (90%) create mode 100644 src/Abuse/Adapters/TimeLimit/RedisCluster.php create mode 100644 tests/Abuse/RedisClusterTest.php diff --git a/composer.lock b/composer.lock index f8481d3db8..1769d6c2fb 100644 --- a/composer.lock +++ b/composer.lock @@ -758,13 +758,13 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "1.0.x-dev" - }, "spi": { "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\HookManagerInterface": [ "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\ExtensionHookManager" ] + }, + "branch-alias": { + "dev-main": "1.0.x-dev" } }, "autoload": { diff --git a/docker-compose.yml b/docker-compose.yml index 3e2b3faa35..0599ca8545 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,5 +30,53 @@ services: - ./src:/code/src - ./tests:/code/tests + redis-insight: + image: redis/redisinsight:latest + restart: unless-stopped + networks: + - abuse + environment: + - REDIS_HOSTS=redis-cluster-0:6379 + ports: + - "8081:5540" + + redis-cluster-0: + image: docker.io/bitnami/redis-cluster:7.4 + environment: + - ALLOW_EMPTY_PASSWORD=yes + - REDIS_NODES=redis-cluster-0 redis-cluster-1 redis-cluster-2 redis-cluster-3 + - REDIS_CLUSTER_CREATOR=yes + - REDIS_CLUSTER_REPLICAS=0 + networks: + - abuse + depends_on: + - redis-cluster-1 + - redis-cluster-2 + - redis-cluster-3 + + redis-cluster-1: + image: docker.io/bitnami/redis-cluster:7.4 + environment: + - ALLOW_EMPTY_PASSWORD=yes + - REDIS_NODES=redis-cluster-0 redis-cluster-1 redis-cluster-2 redis-cluster-3 + networks: + - abuse + + redis-cluster-2: + image: docker.io/bitnami/redis-cluster:7.4 + environment: + - ALLOW_EMPTY_PASSWORD=yes + - REDIS_NODES=redis-cluster-0 redis-cluster-1 redis-cluster-2 redis-cluster-3 + networks: + - abuse + + redis-cluster-3: + image: docker.io/bitnami/redis-cluster:7.4 + environment: + - ALLOW_EMPTY_PASSWORD=yes + - REDIS_NODES=redis-cluster-0 redis-cluster-1 redis-cluster-2 redis-cluster-3 + networks: + - abuse + networks: abuse: \ No newline at end of file diff --git a/phpunit.xml b/phpunit.xml index ec39a7edea..10d408caa0 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -6,7 +6,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false" + stopOnFailure="true" > diff --git a/src/Abuse/Adapter.php b/src/Abuse/Adapter.php index 4af27dc40a..d78a06f9e2 100644 --- a/src/Abuse/Adapter.php +++ b/src/Abuse/Adapter.php @@ -70,11 +70,11 @@ abstract class Adapter * * Return logs with an offset and limit * - * @param int|null $offset + * @param int $offset * @param int|null $limit * @return array */ - abstract public function getLogs(?int $offset = null, ?int $limit = 25): array; + abstract public function getLogs(int $offset = 0, ?int $limit = 25): array; /** * Delete all logs older than $datetime diff --git a/src/Abuse/Adapters/TimeLimit.php b/src/Abuse/Adapters/TimeLimit.php index 8040dc3b9a..dcafee0835 100644 --- a/src/Abuse/Adapters/TimeLimit.php +++ b/src/Abuse/Adapters/TimeLimit.php @@ -37,8 +37,6 @@ abstract class TimeLimit extends Adapter abstract protected function hit(string $key, string $datetime): void; - - /** * Check * diff --git a/src/Abuse/Adapters/Database/TimeLimit.php b/src/Abuse/Adapters/TimeLimit/Database.php similarity index 87% rename from src/Abuse/Adapters/Database/TimeLimit.php rename to src/Abuse/Adapters/TimeLimit/Database.php index 47f7e899b7..a7ba5662c2 100644 --- a/src/Abuse/Adapters/Database/TimeLimit.php +++ b/src/Abuse/Adapters/TimeLimit/Database.php @@ -1,8 +1,8 @@ db->createCollection( - TimeLimit::COLLECTION, + self::COLLECTION, $attributes, $indexes ); @@ -138,7 +138,7 @@ class TimeLimit extends TimeLimitAdapter /** @var array $result */ $result = Authorization::skip(function () use ($key, $datetime) { - return $this->db->find(TimeLimit::COLLECTION, [ + return $this->db->find(self::COLLECTION, [ Query::equal('key', [$key]), Query::equal('time', [$datetime]), ]); @@ -170,7 +170,7 @@ class TimeLimit extends TimeLimitAdapter } Authorization::skip(function () use ($datetime, $key) { - $data = $this->db->findOne(TimeLimit::COLLECTION, [ + $data = $this->db->findOne(self::COLLECTION, [ Query::equal('key', [$key]), Query::equal('time', [$datetime]), ]); @@ -181,14 +181,14 @@ class TimeLimit extends TimeLimitAdapter 'key' => $key, 'time' => $datetime, 'count' => 1, - '$collection' => TimeLimit::COLLECTION, + '$collection' => self::COLLECTION, ]; try { - $this->db->createDocument(TimeLimit::COLLECTION, new Document($data)); + $this->db->createDocument(self::COLLECTION, new Document($data)); } catch (Duplicate $e) { // Duplicate in case of race condition - $data = $this->db->findOne(TimeLimit::COLLECTION, [ + $data = $this->db->findOne(self::COLLECTION, [ Query::equal('key', [$key]), Query::equal('time', [$datetime]), ]); @@ -198,14 +198,14 @@ class TimeLimit extends TimeLimitAdapter if (\is_numeric($count)) { $this->count = intval($count); } - $this->db->increaseDocumentAttribute(TimeLimit::COLLECTION, $data->getId(), 'count'); + $this->db->increaseDocumentAttribute(self::COLLECTION, $data->getId(), 'count'); } else { throw new \Exception('Document Not Found'); } } } else { /** @var Document $data */ - $this->db->increaseDocumentAttribute(TimeLimit::COLLECTION, $data->getId(), 'count'); + $this->db->increaseDocumentAttribute(self::COLLECTION, $data->getId(), 'count'); } }); @@ -237,7 +237,7 @@ class TimeLimit extends TimeLimitAdapter $queries[] = Query::limit($limit); } - return $this->db->find(TimeLimit::COLLECTION, $queries); + return $this->db->find(self::COLLECTION, $queries); }); return $results; @@ -255,12 +255,12 @@ class TimeLimit extends TimeLimitAdapter { Authorization::skip(function () use ($datetime) { do { - $documents = $this->db->find(TimeLimit::COLLECTION, [ + $documents = $this->db->find(self::COLLECTION, [ Query::lessThan('time', $datetime), ]); foreach ($documents as $document) { - $this->db->deleteDocument(TimeLimit::COLLECTION, $document->getId()); + $this->db->deleteDocument(self::COLLECTION, $document->getId()); } } while (! empty($documents)); }); diff --git a/src/Abuse/Adapters/Redis/TimeLimit.php b/src/Abuse/Adapters/TimeLimit/Redis.php similarity index 90% rename from src/Abuse/Adapters/Redis/TimeLimit.php rename to src/Abuse/Adapters/TimeLimit/Redis.php index f2349212b3..1fbef43597 100644 --- a/src/Abuse/Adapters/Redis/TimeLimit.php +++ b/src/Abuse/Adapters/TimeLimit/Redis.php @@ -1,11 +1,11 @@ */ - public function getLogs(?int $offset = null, ?int $limit = 25): array + public function getLogs(int $offset = 0, ?int $limit = 25): array { // TODO limit potential is SCAN but needs cursor no offset - $cursor = null; - $keys = $this->redis->scan($cursor, self::NAMESPACE . '__*', $limit); + $keys = $this->redis->scan($offset, self::NAMESPACE . '__*', $limit); if (!$keys) { return []; } @@ -110,7 +109,7 @@ class TimeLimit extends TimeLimitAdapter { $iterator = null; while ($iterator !== 0) { - $keys = $this->redis->scan($iterator, self::NAMESPACE . '__*__*', 1000); + $keys = $this->redis->scan($iterator, self::NAMESPACE . '__*__*', 10); $keys = $this->filterKeys($keys ? $keys : [], (int) $datetime); $this->redis->del($keys); } diff --git a/src/Abuse/Adapters/TimeLimit/RedisCluster.php b/src/Abuse/Adapters/TimeLimit/RedisCluster.php new file mode 100644 index 0000000000..426dc22fd5 --- /dev/null +++ b/src/Abuse/Adapters/TimeLimit/RedisCluster.php @@ -0,0 +1,153 @@ +redis = $redis; + $this->key = $key; + $time = (int) \date('U', (int) (\floor(\time() / $seconds)) * $seconds); + $this->time = strval($time); + $this->limit = $limit; + } + + /** + * Undocumented function + * + * @param string $key + * @param string $datetime + * @return integer + */ + protected function count(string $key, string $datetime): int + { + if (0 == $this->limit) { // No limit no point for counting + return 0; + } + + if (! \is_null($this->count)) { // Get fetched result + return $this->count; + } + + /** @var string $count */ + $count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $datetime); + if (!$count) { + $this->count = 0; + } else { + $this->count = intval($count); + } + + return $this->count; + } + + /** + * @param string $key + * @param string $datetime + * @return void + * + */ + protected function hit(string $key, string $datetime): void + { + if (0 == $this->limit) { // No limit no point for counting + return; + } + + /** @var string $count */ + $count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $datetime); + if (!$count) { + $this->count = 0; + } else { + $this->count = intval($count); + } + + $this->redis->incr(self::NAMESPACE . '__'. $key .'__'. $datetime); + $this->count++; + } + + /** + * Get abuse logs + * + * Return logs with an offset and limit + * + * @param int $offset + * @param int|null $limit + * @return array + */ + public function getLogs(int $offset = 0, ?int $limit = 25): array + { + // TODO limit potential is SCAN but needs cursor no offset + $cursor = null; + $keys = $this->scan($cursor, self::NAMESPACE . '__*', $limit); + if (!$keys) { + return []; + } + + $logs = []; + foreach ($keys as $key) { + $logs[$key] = $this->redis->get($key); + } + return $logs; + } + + /** + * Delete all logs older than $datetime + * + * @param string $datetime + * @return bool + */ + public function cleanup(string $datetime): bool + { + $iterator = null; + $keys = $this->scan($iterator, self::NAMESPACE . '__*__*', 1000); + $keys = $this->filterKeys($keys ? $keys : [], (int) $datetime); + $this->redis->del($keys); + return true; + } + + /** + * Filter keys + * + * @param array $keys + * @param integer $timestamp + * @return array + */ + protected function filterKeys(array $keys, int $timestamp): array + { + $filteredKeys = []; + foreach ($keys as $key) { + $parts = explode('__', $key); + $keyTimestamp = (int)end($parts); // Assuming the last part is always the timestamp + if ($keyTimestamp < $timestamp) { + $filteredKeys[] = $key; + } + } + return $filteredKeys; + } + + protected function scan(&$testing, string $pattern, int $count): array|bool + { + $matches = []; + foreach ($this->redis->_masters() as $master) { + $cursor = null; + do { + $keys = $this->redis->scan($cursor, $master, $pattern, $count); + if ($keys !== false) { + $matches = array_merge($matches, $keys); + } + } while ($cursor > 0); + } + + return empty($matches) ? false : $matches; + } +} diff --git a/tests/Abuse/Base.php b/tests/Abuse/Base.php index 1b88f411f6..1eb9f0b7d3 100644 --- a/tests/Abuse/Base.php +++ b/tests/Abuse/Base.php @@ -63,7 +63,7 @@ abstract class Base extends TestCase $status = $this->abuse->cleanup($this->getCleanupDateTime()); $this->assertEquals($status, true); - // Check that there are no logs in the DB + // // Check that there are no logs in the DB $logs = $this->abuse->getLogs(0, 10); $this->assertEquals(0, \count($logs)); } diff --git a/tests/Abuse/DatabaseTest.php b/tests/Abuse/DatabaseTest.php index b3820e6aa4..3890da5fa4 100755 --- a/tests/Abuse/DatabaseTest.php +++ b/tests/Abuse/DatabaseTest.php @@ -5,7 +5,7 @@ namespace Utopia\Tests; use PDO; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapter; -use Utopia\Abuse\Adapters\Database\TimeLimit as AdaptersDatabase; +use Utopia\Abuse\Adapters\TimeLimit\Database as TimeLimitDatabase; use Utopia\Cache\Adapter\None as NoCache; use Utopia\Cache\Cache; use Utopia\Database\Adapter\MariaDB; @@ -37,7 +37,7 @@ class DatabaseTest extends Base $db->setNamespace('namespace'); $this->db = $db; - $adapter = new AdaptersDatabase('login-attempt-from-{{ip}}', 3, 60 * 5, $db); + $adapter = new TimeLimitDatabase('login-attempt-from-{{ip}}', 3, 60 * 5, $db); if (! $db->exists('utopiaTests')) { $db->create(); $adapter->setup(); @@ -45,11 +45,12 @@ class DatabaseTest extends Base $adapter->setParam('{{ip}}', '127.0.0.1'); $this->abuse = new Abuse($adapter); + $this->abuse->cleanup($this->getCleanupDateTime()); } public function getAdapter(string $key, int $limit, int $seconds): Adapter { - return new AdaptersDatabase($key, $limit, $seconds, $this->db); + return new TimeLimitDatabase($key, $limit, $seconds, $this->db); } public function getCleanupDateTime(): string diff --git a/tests/Abuse/RedisClusterTest.php b/tests/Abuse/RedisClusterTest.php new file mode 100644 index 0000000000..9287670479 --- /dev/null +++ b/tests/Abuse/RedisClusterTest.php @@ -0,0 +1,39 @@ +redis = new RedisCluster(null, ['redis-cluster-0:6379', 'redis-cluster-1:6379', 'redis-cluster-2:6379', 'redis-cluster-3:6379']); + $adapter = new TimeLimitRedisCluster('login-attempt-from-{{ip}}', 3, 1, $this->redis); + $adapter->setParam('{{ip}}', '127.0.0.1'); + $this->abuse = new Abuse($adapter); + $this->abuse->cleanup($this->getCleanupDateTime()); + } + + public function getAdapter(string $key, int $limit, int $seconds): Adapter + { + return new TimeLimitRedisCluster($key, $limit, $seconds, $this->redis); + } + + public function getCleanupDateTime(): string + { + $interval = DateInterval::createFromDateString(1 . ' seconds'); + return strval((new \DateTime())->sub($interval)->getTimestamp()); + } +} diff --git a/tests/Abuse/RedisTest.php b/tests/Abuse/RedisTest.php index 517ee94777..b39bc953c8 100644 --- a/tests/Abuse/RedisTest.php +++ b/tests/Abuse/RedisTest.php @@ -6,7 +6,7 @@ use DateInterval; use Redis as Client; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapter; -use Utopia\Abuse\Adapters\Redis\TimeLimit; +use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\Exception; class RedisTest extends Base @@ -21,14 +21,15 @@ class RedisTest extends Base { $this->redis = new Client(); $this->redis->connect('redis', 6379); - $adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 60 * 5, $this->redis); + $adapter = new TimeLimitRedis('login-attempt-from-{{ip}}', 3, 1, $this->redis); $adapter->setParam('{{ip}}', '127.0.0.1'); $this->abuse = new Abuse($adapter); + $this->abuse->cleanup($this->getCleanupDateTime()); } public function getAdapter(string $key, int $limit, int $seconds): Adapter { - return new TimeLimit($key, $limit, $seconds, $this->redis); + return new TimeLimitRedis($key, $limit, $seconds, $this->redis); } public function getCleanupDateTime(): string From 3b78c863cc8590f24b064eb56e8aa519f31a65e0 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 17 Dec 2024 20:38:02 +0530 Subject: [PATCH 2/8] feat: cleanup --- src/Abuse/Adapters/TimeLimit/Redis.php | 7 ++++--- tests/Abuse/Base.php | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Abuse/Adapters/TimeLimit/Redis.php b/src/Abuse/Adapters/TimeLimit/Redis.php index 1fbef43597..11cbd4747c 100644 --- a/src/Abuse/Adapters/TimeLimit/Redis.php +++ b/src/Abuse/Adapters/TimeLimit/Redis.php @@ -80,13 +80,14 @@ class Redis extends TimeLimit * * Return logs with an offset and limit * - * @param int $offset + * @param int|null $offset * @param int|null $limit * @return array */ - public function getLogs(int $offset = 0, ?int $limit = 25): array + public function getLogs(?int $offset = null, ?int $limit = 25): array { // TODO limit potential is SCAN but needs cursor no offset + $cursor = null; $keys = $this->redis->scan($offset, self::NAMESPACE . '__*', $limit); if (!$keys) { return []; @@ -109,7 +110,7 @@ class Redis extends TimeLimit { $iterator = null; while ($iterator !== 0) { - $keys = $this->redis->scan($iterator, self::NAMESPACE . '__*__*', 10); + $keys = $this->redis->scan($iterator, self::NAMESPACE . '__*__*', 1000); $keys = $this->filterKeys($keys ? $keys : [], (int) $datetime); $this->redis->del($keys); } diff --git a/tests/Abuse/Base.php b/tests/Abuse/Base.php index 1eb9f0b7d3..1b88f411f6 100644 --- a/tests/Abuse/Base.php +++ b/tests/Abuse/Base.php @@ -63,7 +63,7 @@ abstract class Base extends TestCase $status = $this->abuse->cleanup($this->getCleanupDateTime()); $this->assertEquals($status, true); - // // Check that there are no logs in the DB + // Check that there are no logs in the DB $logs = $this->abuse->getLogs(0, 10); $this->assertEquals(0, \count($logs)); } From 9b338b6d06a5bb8ea0c824e36abc51128e7ea292 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 17 Dec 2024 20:39:50 +0530 Subject: [PATCH 3/8] feat: cleanup --- src/Abuse/Adapter.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Abuse/Adapter.php b/src/Abuse/Adapter.php index d78a06f9e2..4af27dc40a 100644 --- a/src/Abuse/Adapter.php +++ b/src/Abuse/Adapter.php @@ -70,11 +70,11 @@ abstract class Adapter * * Return logs with an offset and limit * - * @param int $offset + * @param int|null $offset * @param int|null $limit * @return array */ - abstract public function getLogs(int $offset = 0, ?int $limit = 25): array; + abstract public function getLogs(?int $offset = null, ?int $limit = 25): array; /** * Delete all logs older than $datetime From 1dfcdaca0e7cb86e4482bda77f43bf40bc8be4d0 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 17 Dec 2024 20:41:16 +0530 Subject: [PATCH 4/8] feat: cleanup --- src/Abuse/Adapters/TimeLimit/Redis.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Abuse/Adapters/TimeLimit/Redis.php b/src/Abuse/Adapters/TimeLimit/Redis.php index 11cbd4747c..783f5884ed 100644 --- a/src/Abuse/Adapters/TimeLimit/Redis.php +++ b/src/Abuse/Adapters/TimeLimit/Redis.php @@ -88,7 +88,7 @@ class Redis extends TimeLimit { // TODO limit potential is SCAN but needs cursor no offset $cursor = null; - $keys = $this->redis->scan($offset, self::NAMESPACE . '__*', $limit); + $keys = $this->redis->scan($cursor, self::NAMESPACE . '__*', $limit); if (!$keys) { return []; } From aa15eda877a17364e3184b0b7bb4ffbb3f081736 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Tue, 17 Dec 2024 20:41:56 +0530 Subject: [PATCH 5/8] feat: cleanup --- tests/Abuse/DatabaseTest.php | 6 +++--- tests/Abuse/RedisClusterTest.php | 6 +++--- tests/Abuse/RedisTest.php | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/Abuse/DatabaseTest.php b/tests/Abuse/DatabaseTest.php index 3890da5fa4..f4a9252caa 100755 --- a/tests/Abuse/DatabaseTest.php +++ b/tests/Abuse/DatabaseTest.php @@ -5,7 +5,7 @@ namespace Utopia\Tests; use PDO; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapter; -use Utopia\Abuse\Adapters\TimeLimit\Database as TimeLimitDatabase; +use Utopia\Abuse\Adapters\TimeLimit\Database as TimeLimit; use Utopia\Cache\Adapter\None as NoCache; use Utopia\Cache\Cache; use Utopia\Database\Adapter\MariaDB; @@ -37,7 +37,7 @@ class DatabaseTest extends Base $db->setNamespace('namespace'); $this->db = $db; - $adapter = new TimeLimitDatabase('login-attempt-from-{{ip}}', 3, 60 * 5, $db); + $adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 60 * 5, $db); if (! $db->exists('utopiaTests')) { $db->create(); $adapter->setup(); @@ -50,7 +50,7 @@ class DatabaseTest extends Base public function getAdapter(string $key, int $limit, int $seconds): Adapter { - return new TimeLimitDatabase($key, $limit, $seconds, $this->db); + return new TimeLimit($key, $limit, $seconds, $this->db); } public function getCleanupDateTime(): string diff --git a/tests/Abuse/RedisClusterTest.php b/tests/Abuse/RedisClusterTest.php index 9287670479..3d39c56667 100644 --- a/tests/Abuse/RedisClusterTest.php +++ b/tests/Abuse/RedisClusterTest.php @@ -6,7 +6,7 @@ use DateInterval; use RedisCluster; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapter; -use Utopia\Abuse\Adapters\TimeLimit\RedisCluster as TimeLimitRedisCluster; +use Utopia\Abuse\Adapters\TimeLimit\RedisCluster as TimeLimit; use Utopia\Exception; class RedisClusterTest extends Base @@ -20,7 +20,7 @@ class RedisClusterTest extends Base public function setUp(): void { $this->redis = new RedisCluster(null, ['redis-cluster-0:6379', 'redis-cluster-1:6379', 'redis-cluster-2:6379', 'redis-cluster-3:6379']); - $adapter = new TimeLimitRedisCluster('login-attempt-from-{{ip}}', 3, 1, $this->redis); + $adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 1, $this->redis); $adapter->setParam('{{ip}}', '127.0.0.1'); $this->abuse = new Abuse($adapter); $this->abuse->cleanup($this->getCleanupDateTime()); @@ -28,7 +28,7 @@ class RedisClusterTest extends Base public function getAdapter(string $key, int $limit, int $seconds): Adapter { - return new TimeLimitRedisCluster($key, $limit, $seconds, $this->redis); + return new TimeLimit($key, $limit, $seconds, $this->redis); } public function getCleanupDateTime(): string diff --git a/tests/Abuse/RedisTest.php b/tests/Abuse/RedisTest.php index b39bc953c8..8423db8608 100644 --- a/tests/Abuse/RedisTest.php +++ b/tests/Abuse/RedisTest.php @@ -6,7 +6,7 @@ use DateInterval; use Redis as Client; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapter; -use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; +use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimit; use Utopia\Exception; class RedisTest extends Base @@ -21,7 +21,7 @@ class RedisTest extends Base { $this->redis = new Client(); $this->redis->connect('redis', 6379); - $adapter = new TimeLimitRedis('login-attempt-from-{{ip}}', 3, 1, $this->redis); + $adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 1, $this->redis); $adapter->setParam('{{ip}}', '127.0.0.1'); $this->abuse = new Abuse($adapter); $this->abuse->cleanup($this->getCleanupDateTime()); @@ -29,7 +29,7 @@ class RedisTest extends Base public function getAdapter(string $key, int $limit, int $seconds): Adapter { - return new TimeLimitRedis($key, $limit, $seconds, $this->redis); + return new TimeLimit($key, $limit, $seconds, $this->redis); } public function getCleanupDateTime(): string From 76fc3da33bea752165a34911ba6d16a04fca2dd7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 18 Dec 2024 10:46:03 +0530 Subject: [PATCH 6/8] fix: codeql --- composer.lock | 10 ++-- src/Abuse/Adapters/TimeLimit/Redis.php | 7 ++- src/Abuse/Adapters/TimeLimit/RedisCluster.php | 54 +++++++++++-------- tests/Abuse/Bench/Database/TimeLimitBench.php | 2 +- tests/Abuse/Bench/Redis/TimeLimitBench.php | 2 +- 5 files changed, 41 insertions(+), 34 deletions(-) diff --git a/composer.lock b/composer.lock index 1769d6c2fb..0ec24c8c21 100644 --- a/composer.lock +++ b/composer.lock @@ -3040,16 +3040,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.12.12", + "version": "1.12.13", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "b5ae1b88f471d3fd4ba1aa0046234b5ca3776dd0" + "reference": "9b469068840cfa031e1deaf2fa1886d00e20680f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/b5ae1b88f471d3fd4ba1aa0046234b5ca3776dd0", - "reference": "b5ae1b88f471d3fd4ba1aa0046234b5ca3776dd0", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/9b469068840cfa031e1deaf2fa1886d00e20680f", + "reference": "9b469068840cfa031e1deaf2fa1886d00e20680f", "shasum": "" }, "require": { @@ -3094,7 +3094,7 @@ "type": "github" } ], - "time": "2024-11-28T22:13:23+00:00" + "time": "2024-12-17T17:00:20+00:00" }, { "name": "phpunit/php-code-coverage", diff --git a/src/Abuse/Adapters/TimeLimit/Redis.php b/src/Abuse/Adapters/TimeLimit/Redis.php index 783f5884ed..3657a570d5 100644 --- a/src/Abuse/Adapters/TimeLimit/Redis.php +++ b/src/Abuse/Adapters/TimeLimit/Redis.php @@ -2,7 +2,6 @@ namespace Utopia\Abuse\Adapters\TimeLimit; -use Redis as Client; use Utopia\Abuse\Adapters\TimeLimit; class Redis extends TimeLimit @@ -10,11 +9,11 @@ class Redis extends TimeLimit public const NAMESPACE = 'abuse'; /** - * @var Client + * @var \Redis */ - protected Client $redis; + protected \Redis $redis; - public function __construct(string $key, int $limit, int $seconds, Client $redis) + public function __construct(string $key, int $limit, int $seconds, \Redis $redis) { $this->redis = $redis; $this->key = $key; diff --git a/src/Abuse/Adapters/TimeLimit/RedisCluster.php b/src/Abuse/Adapters/TimeLimit/RedisCluster.php index 426dc22fd5..24145d2b4b 100644 --- a/src/Abuse/Adapters/TimeLimit/RedisCluster.php +++ b/src/Abuse/Adapters/TimeLimit/RedisCluster.php @@ -2,7 +2,6 @@ namespace Utopia\Abuse\Adapters\TimeLimit; -use RedisCluster as Client; use Utopia\Abuse\Adapters\TimeLimit; class RedisCluster extends TimeLimit @@ -10,11 +9,11 @@ class RedisCluster extends TimeLimit public const NAMESPACE = 'abuse'; /** - * @var Client + * @var \RedisCluster */ - protected Client $redis; + protected \RedisCluster $redis; - public function __construct(string $key, int $limit, int $seconds, Client $redis) + public function __construct(string $key, int $limit, int $seconds, \RedisCluster $redis) { $this->redis = $redis; $this->key = $key; @@ -23,8 +22,8 @@ class RedisCluster extends TimeLimit $this->limit = $limit; } - /** - * Undocumented function + /** + * Get count for a key at specific datetime * * @param string $key * @param string $datetime @@ -40,9 +39,9 @@ class RedisCluster extends TimeLimit return $this->count; } - /** @var string $count */ + /** @var string|false $count */ $count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $datetime); - if (!$count) { + if ($count === false) { $this->count = 0; } else { $this->count = intval($count); @@ -52,10 +51,11 @@ class RedisCluster extends TimeLimit } /** - * @param string $key - * @param string $datetime - * @return void + * Record a hit for a key at specific datetime * + * @param string $key + * @param string $datetime + * @return void */ protected function hit(string $key, string $datetime): void { @@ -63,9 +63,9 @@ class RedisCluster extends TimeLimit return; } - /** @var string $count */ + /** @var string|false $count */ $count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $datetime); - if (!$count) { + if ($count === false) { $this->count = 0; } else { $this->count = intval($count); @@ -80,16 +80,15 @@ class RedisCluster extends TimeLimit * * Return logs with an offset and limit * - * @param int $offset - * @param int|null $limit + * @param int|null $offset + * @param int|null $limit * @return array */ - public function getLogs(int $offset = 0, ?int $limit = 25): array + public function getLogs(?int $offset = 0, ?int $limit = 25): array { // TODO limit potential is SCAN but needs cursor no offset - $cursor = null; - $keys = $this->scan($cursor, self::NAMESPACE . '__*', $limit); - if (!$keys) { + $keys = $this->scan(self::NAMESPACE . '__*', $offset, $limit); + if ($keys === false) { return []; } @@ -103,14 +102,14 @@ class RedisCluster extends TimeLimit /** * Delete all logs older than $datetime * - * @param string $datetime + * @param string $datetime * @return bool */ public function cleanup(string $datetime): bool { - $iterator = null; - $keys = $this->scan($iterator, self::NAMESPACE . '__*__*', 1000); + $keys = $this->scan(self::NAMESPACE . '__*__*'); $keys = $this->filterKeys($keys ? $keys : [], (int) $datetime); + /** @phpstan-ignore-next-line */ $this->redis->del($keys); return true; } @@ -135,12 +134,21 @@ class RedisCluster extends TimeLimit return $filteredKeys; } - protected function scan(&$testing, string $pattern, int $count): array|bool + /** + * Scan keys across all masters in the Redis cluster + * + * @param string $pattern Pattern to match keys + * @param int|null $cursor Reference to the cursor for scanning + * @param int|null $count Number of keys to return per iteration + * @return array|false + */ + protected function scan(string $pattern, ?int &$cursor = null, ?int $count = 1000): array|false { $matches = []; foreach ($this->redis->_masters() as $master) { $cursor = null; do { + /** @phpstan-ignore-next-line */ $keys = $this->redis->scan($cursor, $master, $pattern, $count); if ($keys !== false) { $matches = array_merge($matches, $keys); diff --git a/tests/Abuse/Bench/Database/TimeLimitBench.php b/tests/Abuse/Bench/Database/TimeLimitBench.php index e2a110c6b7..0525c44d81 100644 --- a/tests/Abuse/Bench/Database/TimeLimitBench.php +++ b/tests/Abuse/Bench/Database/TimeLimitBench.php @@ -4,7 +4,7 @@ namespace Abuse\Bench\Database; use PDO; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\Database\TimeLimit; +use Utopia\Abuse\Adapters\TimeLimit\Database as TimeLimit; use Utopia\Cache\Adapter\None as NoCache; use Utopia\Cache\Cache; use Utopia\Database\Adapter\MariaDB; diff --git a/tests/Abuse/Bench/Redis/TimeLimitBench.php b/tests/Abuse/Bench/Redis/TimeLimitBench.php index 0d8cdb771d..3fdccfb811 100644 --- a/tests/Abuse/Bench/Redis/TimeLimitBench.php +++ b/tests/Abuse/Bench/Redis/TimeLimitBench.php @@ -4,7 +4,7 @@ namespace Abuse\Bench\Redis; use Redis as Client; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\Redis\TimeLimit; +use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimit; use Utopia\Tests\Bench\Base; final class TimeLimitBench extends Base From 2663547e4216d423ad226c1b2f6680d02e936d9b Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 18 Dec 2024 11:36:01 +0530 Subject: [PATCH 7/8] fix: tests --- docker-compose.yml | 9 +++++++ tests/Abuse/Base.php | 4 ++-- .../TimeLimitBench.php => Database.php} | 4 ++-- .../{Redis/TimeLimitBench.php => Redis.php} | 4 ++-- tests/Abuse/Bench/RedisCluster.php | 24 +++++++++++++++++++ tests/Abuse/DatabaseTest.php | 3 +-- tests/Abuse/RedisClusterTest.php | 3 +-- tests/Abuse/RedisTest.php | 3 +-- 8 files changed, 42 insertions(+), 12 deletions(-) rename tests/Abuse/Bench/{Database/TimeLimitBench.php => Database.php} (94%) rename tests/Abuse/Bench/{Redis/TimeLimitBench.php => Redis.php} (87%) create mode 100644 tests/Abuse/Bench/RedisCluster.php diff --git a/docker-compose.yml b/docker-compose.yml index 0599ca8545..a139da027b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,15 @@ services: ports: - "9307:3306" + adminer: + image: adminer + container_name: abuse-adminer + restart: always + ports: + - 9506:8080 + networks: + - abuse + redis: image: redis:6.0-alpine container_name: redis diff --git a/tests/Abuse/Base.php b/tests/Abuse/Base.php index 1b88f411f6..2b7e291bed 100644 --- a/tests/Abuse/Base.php +++ b/tests/Abuse/Base.php @@ -53,11 +53,11 @@ abstract class Base extends TestCase public function testCleanup(): void { - // Check that there is only one log + // Check that there are 3 logs $logs = $this->abuse->getLogs(0, 10); $this->assertEquals(3, \count($logs)); - sleep(5); + sleep(2); // Delete the log $status = $this->abuse->cleanup($this->getCleanupDateTime()); diff --git a/tests/Abuse/Bench/Database/TimeLimitBench.php b/tests/Abuse/Bench/Database.php similarity index 94% rename from tests/Abuse/Bench/Database/TimeLimitBench.php rename to tests/Abuse/Bench/Database.php index 0525c44d81..a24c9ed4a7 100644 --- a/tests/Abuse/Bench/Database/TimeLimitBench.php +++ b/tests/Abuse/Bench/Database.php @@ -1,6 +1,6 @@ redis = new Client(null, ['redis-cluster-0:6379', 'redis-cluster-1:6379', 'redis-cluster-2:6379', 'redis-cluster-3:6379']); + $this->adapter = new RedisCluster('login-attempt-from-{{ip}}', 3, 60 * 5, $this->redis); + $this->abuse = new Abuse($this->adapter); + } +} diff --git a/tests/Abuse/DatabaseTest.php b/tests/Abuse/DatabaseTest.php index f4a9252caa..8ca37426bf 100755 --- a/tests/Abuse/DatabaseTest.php +++ b/tests/Abuse/DatabaseTest.php @@ -45,7 +45,6 @@ class DatabaseTest extends Base $adapter->setParam('{{ip}}', '127.0.0.1'); $this->abuse = new Abuse($adapter); - $this->abuse->cleanup($this->getCleanupDateTime()); } public function getAdapter(string $key, int $limit, int $seconds): Adapter @@ -55,6 +54,6 @@ class DatabaseTest extends Base public function getCleanupDateTime(): string { - return DateTime::addSeconds(new \DateTime(), -1); + return DateTime::format(new \DateTime()); } } diff --git a/tests/Abuse/RedisClusterTest.php b/tests/Abuse/RedisClusterTest.php index 3d39c56667..ba1a60da96 100644 --- a/tests/Abuse/RedisClusterTest.php +++ b/tests/Abuse/RedisClusterTest.php @@ -23,7 +23,6 @@ class RedisClusterTest extends Base $adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 1, $this->redis); $adapter->setParam('{{ip}}', '127.0.0.1'); $this->abuse = new Abuse($adapter); - $this->abuse->cleanup($this->getCleanupDateTime()); } public function getAdapter(string $key, int $limit, int $seconds): Adapter @@ -33,7 +32,7 @@ class RedisClusterTest extends Base public function getCleanupDateTime(): string { - $interval = DateInterval::createFromDateString(1 . ' seconds'); + $interval = DateInterval::createFromDateString('now'); return strval((new \DateTime())->sub($interval)->getTimestamp()); } } diff --git a/tests/Abuse/RedisTest.php b/tests/Abuse/RedisTest.php index 8423db8608..e0bf69858a 100644 --- a/tests/Abuse/RedisTest.php +++ b/tests/Abuse/RedisTest.php @@ -24,7 +24,6 @@ class RedisTest extends Base $adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 1, $this->redis); $adapter->setParam('{{ip}}', '127.0.0.1'); $this->abuse = new Abuse($adapter); - $this->abuse->cleanup($this->getCleanupDateTime()); } public function getAdapter(string $key, int $limit, int $seconds): Adapter @@ -34,7 +33,7 @@ class RedisTest extends Base public function getCleanupDateTime(): string { - $interval = DateInterval::createFromDateString(1 . ' seconds'); + $interval = DateInterval::createFromDateString('now'); return strval((new \DateTime())->sub($interval)->getTimestamp()); } } From 627d1ba56451dd70ca26a1253c42e039b4bfc1a7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 18 Dec 2024 11:38:34 +0530 Subject: [PATCH 8/8] fix: tests --- tests/Abuse/Bench/RedisCluster.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Abuse/Bench/RedisCluster.php b/tests/Abuse/Bench/RedisCluster.php index 5367557ae4..74b730e70f 100644 --- a/tests/Abuse/Bench/RedisCluster.php +++ b/tests/Abuse/Bench/RedisCluster.php @@ -4,7 +4,6 @@ namespace Abuse\Bench; use RedisCluster as Client; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\TimeLimit\RedisCluster as TimeLimit; use Utopia\Abuse\Adapters\TimeLimit\RedisCluster; use Utopia\Tests\Bench\Base;