diff --git a/composer.lock b/composer.lock index f8481d3db8..0ec24c8c21 100644 --- a/composer.lock +++ b/composer.lock @@ -758,13 +758,13 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "1.0.x-dev" - }, "spi": { "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\HookManagerInterface": [ "OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\ExtensionHookManager" ] + }, + "branch-alias": { + "dev-main": "1.0.x-dev" } }, "autoload": { @@ -3040,16 +3040,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.12.12", + "version": "1.12.13", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "b5ae1b88f471d3fd4ba1aa0046234b5ca3776dd0" + "reference": "9b469068840cfa031e1deaf2fa1886d00e20680f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/b5ae1b88f471d3fd4ba1aa0046234b5ca3776dd0", - "reference": "b5ae1b88f471d3fd4ba1aa0046234b5ca3776dd0", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/9b469068840cfa031e1deaf2fa1886d00e20680f", + "reference": "9b469068840cfa031e1deaf2fa1886d00e20680f", "shasum": "" }, "require": { @@ -3094,7 +3094,7 @@ "type": "github" } ], - "time": "2024-11-28T22:13:23+00:00" + "time": "2024-12-17T17:00:20+00:00" }, { "name": "phpunit/php-code-coverage", diff --git a/docker-compose.yml b/docker-compose.yml index 3e2b3faa35..a139da027b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,15 @@ services: ports: - "9307:3306" + adminer: + image: adminer + container_name: abuse-adminer + restart: always + ports: + - 9506:8080 + networks: + - abuse + redis: image: redis:6.0-alpine container_name: redis @@ -30,5 +39,53 @@ services: - ./src:/code/src - ./tests:/code/tests + redis-insight: + image: redis/redisinsight:latest + restart: unless-stopped + networks: + - abuse + environment: + - REDIS_HOSTS=redis-cluster-0:6379 + ports: + - "8081:5540" + + redis-cluster-0: + image: docker.io/bitnami/redis-cluster:7.4 + environment: + - ALLOW_EMPTY_PASSWORD=yes + - REDIS_NODES=redis-cluster-0 redis-cluster-1 redis-cluster-2 redis-cluster-3 + - REDIS_CLUSTER_CREATOR=yes + - REDIS_CLUSTER_REPLICAS=0 + networks: + - abuse + depends_on: + - redis-cluster-1 + - redis-cluster-2 + - redis-cluster-3 + + redis-cluster-1: + image: docker.io/bitnami/redis-cluster:7.4 + environment: + - ALLOW_EMPTY_PASSWORD=yes + - REDIS_NODES=redis-cluster-0 redis-cluster-1 redis-cluster-2 redis-cluster-3 + networks: + - abuse + + redis-cluster-2: + image: docker.io/bitnami/redis-cluster:7.4 + environment: + - ALLOW_EMPTY_PASSWORD=yes + - REDIS_NODES=redis-cluster-0 redis-cluster-1 redis-cluster-2 redis-cluster-3 + networks: + - abuse + + redis-cluster-3: + image: docker.io/bitnami/redis-cluster:7.4 + environment: + - ALLOW_EMPTY_PASSWORD=yes + - REDIS_NODES=redis-cluster-0 redis-cluster-1 redis-cluster-2 redis-cluster-3 + networks: + - abuse + networks: abuse: \ No newline at end of file diff --git a/phpunit.xml b/phpunit.xml index ec39a7edea..10d408caa0 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -6,7 +6,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false" + stopOnFailure="true" > diff --git a/src/Abuse/Adapters/TimeLimit.php b/src/Abuse/Adapters/TimeLimit.php index 8040dc3b9a..dcafee0835 100644 --- a/src/Abuse/Adapters/TimeLimit.php +++ b/src/Abuse/Adapters/TimeLimit.php @@ -37,8 +37,6 @@ abstract class TimeLimit extends Adapter abstract protected function hit(string $key, string $datetime): void; - - /** * Check * diff --git a/src/Abuse/Adapters/Database/TimeLimit.php b/src/Abuse/Adapters/TimeLimit/Database.php similarity index 87% rename from src/Abuse/Adapters/Database/TimeLimit.php rename to src/Abuse/Adapters/TimeLimit/Database.php index 47f7e899b7..a7ba5662c2 100644 --- a/src/Abuse/Adapters/Database/TimeLimit.php +++ b/src/Abuse/Adapters/TimeLimit/Database.php @@ -1,8 +1,8 @@ db->createCollection( - TimeLimit::COLLECTION, + self::COLLECTION, $attributes, $indexes ); @@ -138,7 +138,7 @@ class TimeLimit extends TimeLimitAdapter /** @var array $result */ $result = Authorization::skip(function () use ($key, $datetime) { - return $this->db->find(TimeLimit::COLLECTION, [ + return $this->db->find(self::COLLECTION, [ Query::equal('key', [$key]), Query::equal('time', [$datetime]), ]); @@ -170,7 +170,7 @@ class TimeLimit extends TimeLimitAdapter } Authorization::skip(function () use ($datetime, $key) { - $data = $this->db->findOne(TimeLimit::COLLECTION, [ + $data = $this->db->findOne(self::COLLECTION, [ Query::equal('key', [$key]), Query::equal('time', [$datetime]), ]); @@ -181,14 +181,14 @@ class TimeLimit extends TimeLimitAdapter 'key' => $key, 'time' => $datetime, 'count' => 1, - '$collection' => TimeLimit::COLLECTION, + '$collection' => self::COLLECTION, ]; try { - $this->db->createDocument(TimeLimit::COLLECTION, new Document($data)); + $this->db->createDocument(self::COLLECTION, new Document($data)); } catch (Duplicate $e) { // Duplicate in case of race condition - $data = $this->db->findOne(TimeLimit::COLLECTION, [ + $data = $this->db->findOne(self::COLLECTION, [ Query::equal('key', [$key]), Query::equal('time', [$datetime]), ]); @@ -198,14 +198,14 @@ class TimeLimit extends TimeLimitAdapter if (\is_numeric($count)) { $this->count = intval($count); } - $this->db->increaseDocumentAttribute(TimeLimit::COLLECTION, $data->getId(), 'count'); + $this->db->increaseDocumentAttribute(self::COLLECTION, $data->getId(), 'count'); } else { throw new \Exception('Document Not Found'); } } } else { /** @var Document $data */ - $this->db->increaseDocumentAttribute(TimeLimit::COLLECTION, $data->getId(), 'count'); + $this->db->increaseDocumentAttribute(self::COLLECTION, $data->getId(), 'count'); } }); @@ -237,7 +237,7 @@ class TimeLimit extends TimeLimitAdapter $queries[] = Query::limit($limit); } - return $this->db->find(TimeLimit::COLLECTION, $queries); + return $this->db->find(self::COLLECTION, $queries); }); return $results; @@ -255,12 +255,12 @@ class TimeLimit extends TimeLimitAdapter { Authorization::skip(function () use ($datetime) { do { - $documents = $this->db->find(TimeLimit::COLLECTION, [ + $documents = $this->db->find(self::COLLECTION, [ Query::lessThan('time', $datetime), ]); foreach ($documents as $document) { - $this->db->deleteDocument(TimeLimit::COLLECTION, $document->getId()); + $this->db->deleteDocument(self::COLLECTION, $document->getId()); } } while (! empty($documents)); }); diff --git a/src/Abuse/Adapters/Redis/TimeLimit.php b/src/Abuse/Adapters/TimeLimit/Redis.php similarity index 93% rename from src/Abuse/Adapters/Redis/TimeLimit.php rename to src/Abuse/Adapters/TimeLimit/Redis.php index f2349212b3..3657a570d5 100644 --- a/src/Abuse/Adapters/Redis/TimeLimit.php +++ b/src/Abuse/Adapters/TimeLimit/Redis.php @@ -1,20 +1,19 @@ redis = $redis; $this->key = $key; diff --git a/src/Abuse/Adapters/TimeLimit/RedisCluster.php b/src/Abuse/Adapters/TimeLimit/RedisCluster.php new file mode 100644 index 0000000000..24145d2b4b --- /dev/null +++ b/src/Abuse/Adapters/TimeLimit/RedisCluster.php @@ -0,0 +1,161 @@ +redis = $redis; + $this->key = $key; + $time = (int) \date('U', (int) (\floor(\time() / $seconds)) * $seconds); + $this->time = strval($time); + $this->limit = $limit; + } + + /** + * Get count for a key at specific datetime + * + * @param string $key + * @param string $datetime + * @return integer + */ + protected function count(string $key, string $datetime): int + { + if (0 == $this->limit) { // No limit no point for counting + return 0; + } + + if (! \is_null($this->count)) { // Get fetched result + return $this->count; + } + + /** @var string|false $count */ + $count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $datetime); + if ($count === false) { + $this->count = 0; + } else { + $this->count = intval($count); + } + + return $this->count; + } + + /** + * Record a hit for a key at specific datetime + * + * @param string $key + * @param string $datetime + * @return void + */ + protected function hit(string $key, string $datetime): void + { + if (0 == $this->limit) { // No limit no point for counting + return; + } + + /** @var string|false $count */ + $count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $datetime); + if ($count === false) { + $this->count = 0; + } else { + $this->count = intval($count); + } + + $this->redis->incr(self::NAMESPACE . '__'. $key .'__'. $datetime); + $this->count++; + } + + /** + * Get abuse logs + * + * Return logs with an offset and limit + * + * @param int|null $offset + * @param int|null $limit + * @return array + */ + public function getLogs(?int $offset = 0, ?int $limit = 25): array + { + // TODO limit potential is SCAN but needs cursor no offset + $keys = $this->scan(self::NAMESPACE . '__*', $offset, $limit); + if ($keys === false) { + return []; + } + + $logs = []; + foreach ($keys as $key) { + $logs[$key] = $this->redis->get($key); + } + return $logs; + } + + /** + * Delete all logs older than $datetime + * + * @param string $datetime + * @return bool + */ + public function cleanup(string $datetime): bool + { + $keys = $this->scan(self::NAMESPACE . '__*__*'); + $keys = $this->filterKeys($keys ? $keys : [], (int) $datetime); + /** @phpstan-ignore-next-line */ + $this->redis->del($keys); + return true; + } + + /** + * Filter keys + * + * @param array $keys + * @param integer $timestamp + * @return array + */ + protected function filterKeys(array $keys, int $timestamp): array + { + $filteredKeys = []; + foreach ($keys as $key) { + $parts = explode('__', $key); + $keyTimestamp = (int)end($parts); // Assuming the last part is always the timestamp + if ($keyTimestamp < $timestamp) { + $filteredKeys[] = $key; + } + } + return $filteredKeys; + } + + /** + * Scan keys across all masters in the Redis cluster + * + * @param string $pattern Pattern to match keys + * @param int|null $cursor Reference to the cursor for scanning + * @param int|null $count Number of keys to return per iteration + * @return array|false + */ + protected function scan(string $pattern, ?int &$cursor = null, ?int $count = 1000): array|false + { + $matches = []; + foreach ($this->redis->_masters() as $master) { + $cursor = null; + do { + /** @phpstan-ignore-next-line */ + $keys = $this->redis->scan($cursor, $master, $pattern, $count); + if ($keys !== false) { + $matches = array_merge($matches, $keys); + } + } while ($cursor > 0); + } + + return empty($matches) ? false : $matches; + } +} diff --git a/tests/Abuse/Base.php b/tests/Abuse/Base.php index 1b88f411f6..2b7e291bed 100644 --- a/tests/Abuse/Base.php +++ b/tests/Abuse/Base.php @@ -53,11 +53,11 @@ abstract class Base extends TestCase public function testCleanup(): void { - // Check that there is only one log + // Check that there are 3 logs $logs = $this->abuse->getLogs(0, 10); $this->assertEquals(3, \count($logs)); - sleep(5); + sleep(2); // Delete the log $status = $this->abuse->cleanup($this->getCleanupDateTime()); diff --git a/tests/Abuse/Bench/Database/TimeLimitBench.php b/tests/Abuse/Bench/Database.php similarity index 91% rename from tests/Abuse/Bench/Database/TimeLimitBench.php rename to tests/Abuse/Bench/Database.php index e2a110c6b7..a24c9ed4a7 100644 --- a/tests/Abuse/Bench/Database/TimeLimitBench.php +++ b/tests/Abuse/Bench/Database.php @@ -1,10 +1,10 @@ redis = new Client(null, ['redis-cluster-0:6379', 'redis-cluster-1:6379', 'redis-cluster-2:6379', 'redis-cluster-3:6379']); + $this->adapter = new RedisCluster('login-attempt-from-{{ip}}', 3, 60 * 5, $this->redis); + $this->abuse = new Abuse($this->adapter); + } +} diff --git a/tests/Abuse/DatabaseTest.php b/tests/Abuse/DatabaseTest.php index b3820e6aa4..8ca37426bf 100755 --- a/tests/Abuse/DatabaseTest.php +++ b/tests/Abuse/DatabaseTest.php @@ -5,7 +5,7 @@ namespace Utopia\Tests; use PDO; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapter; -use Utopia\Abuse\Adapters\Database\TimeLimit as AdaptersDatabase; +use Utopia\Abuse\Adapters\TimeLimit\Database as TimeLimit; use Utopia\Cache\Adapter\None as NoCache; use Utopia\Cache\Cache; use Utopia\Database\Adapter\MariaDB; @@ -37,7 +37,7 @@ class DatabaseTest extends Base $db->setNamespace('namespace'); $this->db = $db; - $adapter = new AdaptersDatabase('login-attempt-from-{{ip}}', 3, 60 * 5, $db); + $adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 60 * 5, $db); if (! $db->exists('utopiaTests')) { $db->create(); $adapter->setup(); @@ -49,11 +49,11 @@ class DatabaseTest extends Base public function getAdapter(string $key, int $limit, int $seconds): Adapter { - return new AdaptersDatabase($key, $limit, $seconds, $this->db); + return new TimeLimit($key, $limit, $seconds, $this->db); } public function getCleanupDateTime(): string { - return DateTime::addSeconds(new \DateTime(), -1); + return DateTime::format(new \DateTime()); } } diff --git a/tests/Abuse/RedisClusterTest.php b/tests/Abuse/RedisClusterTest.php new file mode 100644 index 0000000000..ba1a60da96 --- /dev/null +++ b/tests/Abuse/RedisClusterTest.php @@ -0,0 +1,38 @@ +redis = new RedisCluster(null, ['redis-cluster-0:6379', 'redis-cluster-1:6379', 'redis-cluster-2:6379', 'redis-cluster-3:6379']); + $adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 1, $this->redis); + $adapter->setParam('{{ip}}', '127.0.0.1'); + $this->abuse = new Abuse($adapter); + } + + public function getAdapter(string $key, int $limit, int $seconds): Adapter + { + return new TimeLimit($key, $limit, $seconds, $this->redis); + } + + public function getCleanupDateTime(): string + { + $interval = DateInterval::createFromDateString('now'); + return strval((new \DateTime())->sub($interval)->getTimestamp()); + } +} diff --git a/tests/Abuse/RedisTest.php b/tests/Abuse/RedisTest.php index 517ee94777..e0bf69858a 100644 --- a/tests/Abuse/RedisTest.php +++ b/tests/Abuse/RedisTest.php @@ -6,7 +6,7 @@ use DateInterval; use Redis as Client; use Utopia\Abuse\Abuse; use Utopia\Abuse\Adapter; -use Utopia\Abuse\Adapters\Redis\TimeLimit; +use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimit; use Utopia\Exception; class RedisTest extends Base @@ -21,7 +21,7 @@ class RedisTest extends Base { $this->redis = new Client(); $this->redis->connect('redis', 6379); - $adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 60 * 5, $this->redis); + $adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 1, $this->redis); $adapter->setParam('{{ip}}', '127.0.0.1'); $this->abuse = new Abuse($adapter); } @@ -33,7 +33,7 @@ class RedisTest extends Base public function getCleanupDateTime(): string { - $interval = DateInterval::createFromDateString(1 . ' seconds'); + $interval = DateInterval::createFromDateString('now'); return strval((new \DateTime())->sub($interval)->getTimestamp()); } }