From d985da2a80b129caf901ecfd597b0db3827439b1 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:24:45 +0000 Subject: [PATCH 01/13] chore: optimise abuse --- src/Abuse/Adapters/TimeLimit/Redis.php | 73 +++++++++----------------- 1 file changed, 26 insertions(+), 47 deletions(-) diff --git a/src/Abuse/Adapters/TimeLimit/Redis.php b/src/Abuse/Adapters/TimeLimit/Redis.php index 80a55726f2..d4b27e4d42 100644 --- a/src/Abuse/Adapters/TimeLimit/Redis.php +++ b/src/Abuse/Adapters/TimeLimit/Redis.php @@ -12,11 +12,17 @@ class Redis extends TimeLimit * @var \Redis */ protected \Redis $redis; + + /** + * @var int + */ + protected int $ttl; public function __construct(string $key, int $limit, int $seconds, \Redis $redis) { $this->redis = $redis; $this->key = $key; + $this->ttl = $seconds; $now = \time(); $this->timestamp = (int)($now - ($now % $seconds)); $this->limit = $limit; @@ -62,16 +68,13 @@ class Redis extends TimeLimit return; } - /** @var string $count */ - $count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $timestamp); - if (!$count) { - $this->count = 0; - } else { - $this->count = intval($count); - } - - $this->redis->incr(self::NAMESPACE . '__'. $key .'__'. $timestamp); - $this->count++; + $key = self::NAMESPACE . '__' . $key . '__' . $timestamp; + $this->redis->multi() + ->incr($key) + ->expire($key, $this->ttl) + ->exec(); + + $this->count = ($this->count ?? 0) + 1; } /** @@ -85,18 +88,19 @@ class Redis extends TimeLimit */ public function getLogs(?int $offset = null, ?int $limit = 25): array { - // TODO limit potential is SCAN but needs cursor no offset - $cursor = null; - $keys = $this->redis->scan($cursor, self::NAMESPACE . '__*', $limit); - if (!$keys) { - return []; - } + $cursor = 0; + $matches = []; + $pattern = self::NAMESPACE . '__*'; + + do { + [$cursor, $keys] = $this->redis->scan($cursor, $pattern, $limit); + if (!empty($keys)) { + $values = $this->redis->mget($keys); + $matches = array_merge($matches, array_combine($keys, $values)); + } + } while ($cursor > 0 && (!$limit || count($matches) < $limit)); - $logs = []; - foreach ($keys as $key) { - $logs[$key] = $this->redis->get($key); - } - return $logs; + return $matches; } /** @@ -107,32 +111,7 @@ class Redis extends TimeLimit */ public function cleanup(int $timestamp): bool { - $iterator = null; - while ($iterator !== 0) { - $keys = $this->redis->scan($iterator, self::NAMESPACE . '__*__*', 1000); - $keys = $this->filterKeys($keys ? $keys : [], $timestamp); - $this->redis->del($keys); - } + // No need for manual cleanup - Redis TTL handles this automatically return true; } - - /** - * Filter keys - * - * @param array $keys - * @param integer $timestamp - * @return array - */ - protected function filterKeys(array $keys, int $timestamp): array - { - $filteredKeys = []; - foreach ($keys as $key) { - $parts = explode('__', $key); - $keyTimestamp = (int)end($parts); // Assuming the last part is always the timestamp - if ($keyTimestamp < $timestamp) { - $filteredKeys[] = $key; - } - } - return $filteredKeys; - } } From a9c7993286bf688b91ef84a2a101282da3b41d41 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:35:54 +0000 Subject: [PATCH 02/13] fix: check --- src/Abuse/Adapters/TimeLimit/Redis.php | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/Abuse/Adapters/TimeLimit/Redis.php b/src/Abuse/Adapters/TimeLimit/Redis.php index d4b27e4d42..3b93684f9d 100644 --- a/src/Abuse/Adapters/TimeLimit/Redis.php +++ b/src/Abuse/Adapters/TimeLimit/Redis.php @@ -84,7 +84,7 @@ class Redis extends TimeLimit * * @param int|null $offset * @param int|null $limit - * @return array + * @return array */ public function getLogs(?int $offset = null, ?int $limit = 25): array { @@ -93,10 +93,26 @@ class Redis extends TimeLimit $pattern = self::NAMESPACE . '__*'; do { - [$cursor, $keys] = $this->redis->scan($cursor, $pattern, $limit); + /** @var array{0: string|false, 1: array} */ + $result = $this->redis->scan($cursor, $pattern, $limit); + + if ($result === false) { + break; + } + + [$newCursor, $keys] = $result; + $cursor = (int)$newCursor; + if (!empty($keys)) { + /** @var array */ $values = $this->redis->mget($keys); - $matches = array_merge($matches, array_combine($keys, $values)); + if ($values !== false) { + /** @var array */ + $combinedArray = array_combine($keys, $values); + if ($combinedArray !== false) { + $matches = array_merge($matches, $combinedArray); + } + } } } while ($cursor > 0 && (!$limit || count($matches) < $limit)); From 79c8ce63ec244207762ee1ee9bd6d0ff39ddea52 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:36:05 +0000 Subject: [PATCH 03/13] chore: fmt --- src/Abuse/Adapters/TimeLimit/Redis.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Abuse/Adapters/TimeLimit/Redis.php b/src/Abuse/Adapters/TimeLimit/Redis.php index 3b93684f9d..d0afa8ba70 100644 --- a/src/Abuse/Adapters/TimeLimit/Redis.php +++ b/src/Abuse/Adapters/TimeLimit/Redis.php @@ -12,7 +12,7 @@ class Redis extends TimeLimit * @var \Redis */ protected \Redis $redis; - + /** * @var int */ @@ -73,7 +73,7 @@ class Redis extends TimeLimit ->incr($key) ->expire($key, $this->ttl) ->exec(); - + $this->count = ($this->count ?? 0) + 1; } @@ -91,18 +91,18 @@ class Redis extends TimeLimit $cursor = 0; $matches = []; $pattern = self::NAMESPACE . '__*'; - + do { /** @var array{0: string|false, 1: array} */ $result = $this->redis->scan($cursor, $pattern, $limit); - + if ($result === false) { break; } - + [$newCursor, $keys] = $result; $cursor = (int)$newCursor; - + if (!empty($keys)) { /** @var array */ $values = $this->redis->mget($keys); From cf97de1722ed95cceef426c76000a6204ec5d11e Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:37:05 +0000 Subject: [PATCH 04/13] chore: remove obselete version --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f614d13765..1756697386 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3' - services: mysql: image: mysql:8 From 03cf65cbe360fa8f6c760838af2d4f577294cbea Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:42:43 +0000 Subject: [PATCH 05/13] feat: redis cluster --- src/Abuse/Adapters/TimeLimit/RedisCluster.php | 112 +++++++----------- 1 file changed, 42 insertions(+), 70 deletions(-) diff --git a/src/Abuse/Adapters/TimeLimit/RedisCluster.php b/src/Abuse/Adapters/TimeLimit/RedisCluster.php index 80bd0e519a..47fa140b2e 100644 --- a/src/Abuse/Adapters/TimeLimit/RedisCluster.php +++ b/src/Abuse/Adapters/TimeLimit/RedisCluster.php @@ -13,10 +13,16 @@ class RedisCluster extends TimeLimit */ protected \RedisCluster $redis; + /** + * @var int + */ + protected int $ttl; + public function __construct(string $key, int $limit, int $seconds, \RedisCluster $redis) { $this->redis = $redis; $this->key = $key; + $this->ttl = $seconds; $now = \time(); $this->timestamp = (int)($now - ($now % $seconds)); $this->limit = $limit; @@ -63,22 +69,18 @@ class RedisCluster extends TimeLimit return; } - /** @var string|false $count */ - $count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $timestamp); - if ($count === false) { - $this->count = 0; - } else { - $this->count = intval($count); - } + $key = self::NAMESPACE . '__'. $key .'__'. $timestamp; + + $this->redis->multi(\RedisCluster::MULTI); + $this->redis->incr($key); + $this->redis->expire($key, $this->ttl); + $this->redis->exec(); - $this->redis->incr(self::NAMESPACE . '__'. $key .'__'. $timestamp); - $this->count++; + $this->count = ($this->count ?? 0) + 1; } /** - * Get abuse logs - * - * Return logs with an offset and limit + * Get abuse logs with proper cursor-based pagination * * @param int|null $offset * @param int|null $limit @@ -86,76 +88,46 @@ class RedisCluster extends TimeLimit */ public function getLogs(?int $offset = 0, ?int $limit = 25): array { - // TODO limit potential is SCAN but needs cursor no offset - $keys = $this->scan(self::NAMESPACE . '__*', $offset, $limit); - if ($keys === false) { + $offset = $offset ?? 0; + $limit = $limit ?? 25; + $matches = []; + $pattern = self::NAMESPACE . '__*'; + + // Get all keys from each master + foreach ($this->redis->_masters() as $master) { + $cursor = null; + do { + /** @phpstan-ignore-next-line */ + $keys = $this->redis->scan($cursor, $master, $pattern, 100); + if ($keys !== false) { + $matches = array_merge($matches, $keys); + } + } while ($cursor > 0 && count($matches) < $offset + $limit); + } + + // Sort to ensure consistent ordering + sort($matches); + + // Apply offset and limit + $matches = array_slice($matches, $offset, $limit); + + if (empty($matches)) { return []; } - $logs = []; - foreach ($keys as $key) { - $logs[$key] = $this->redis->get($key); - } - return $logs; + // Batch fetch values using mget + $values = $this->redis->mget($matches); + return array_combine($matches, $values); } /** - * Delete all logs older than $timestamp + * No need for manual cleanup - using Redis TTL * * @param int $timestamp * @return bool */ public function cleanup(int $timestamp): bool { - $keys = $this->scan(self::NAMESPACE . '__*__*'); - $keys = $this->filterKeys($keys ? $keys : [], $timestamp); - /** @phpstan-ignore-next-line */ - $this->redis->del($keys); return true; } - - /** - * Filter keys - * - * @param array $keys - * @param integer $timestamp - * @return array - */ - protected function filterKeys(array $keys, int $timestamp): array - { - $filteredKeys = []; - foreach ($keys as $key) { - $parts = explode('__', $key); - $keyTimestamp = (int)end($parts); // Assuming the last part is always the timestamp - if ($keyTimestamp < $timestamp) { - $filteredKeys[] = $key; - } - } - return $filteredKeys; - } - - /** - * Scan keys across all masters in the Redis cluster - * - * @param string $pattern Pattern to match keys - * @param int|null $cursor Reference to the cursor for scanning - * @param int|null $count Number of keys to return per iteration - * @return array|false - */ - protected function scan(string $pattern, ?int &$cursor = null, ?int $count = 1000): array|false - { - $matches = []; - foreach ($this->redis->_masters() as $master) { - $cursor = null; - do { - /** @phpstan-ignore-next-line */ - $keys = $this->redis->scan($cursor, $master, $pattern, $count); - if ($keys !== false) { - $matches = array_merge($matches, $keys); - } - } while ($cursor > 0); - } - - return empty($matches) ? false : $matches; - } } From 7f2b6b032cf7e83613af29bad60a3ce03e39e1a9 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:54:53 +0000 Subject: [PATCH 06/13] fix: redis scan --- src/Abuse/Adapters/TimeLimit/Redis.php | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Abuse/Adapters/TimeLimit/Redis.php b/src/Abuse/Adapters/TimeLimit/Redis.php index d0afa8ba70..fc4cb8fc64 100644 --- a/src/Abuse/Adapters/TimeLimit/Redis.php +++ b/src/Abuse/Adapters/TimeLimit/Redis.php @@ -93,20 +93,19 @@ class Redis extends TimeLimit $pattern = self::NAMESPACE . '__*'; do { - /** @var array{0: string|false, 1: array} */ $result = $this->redis->scan($cursor, $pattern, $limit); - - if ($result === false) { - break; + + // Early return if scan failed + if (!is_array($result)) { + return []; } [$newCursor, $keys] = $result; $cursor = (int)$newCursor; if (!empty($keys)) { - /** @var array */ $values = $this->redis->mget($keys); - if ($values !== false) { + if (is_array($values)) { /** @var array */ $combinedArray = array_combine($keys, $values); if ($combinedArray !== false) { From e7c208fa993741e7cce90b40ca6b96cc3356237c Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 12:03:46 +0000 Subject: [PATCH 07/13] chore: revert redis --- src/Abuse/Adapters/TimeLimit/Redis.php | 40 ++++++++------------------ 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/src/Abuse/Adapters/TimeLimit/Redis.php b/src/Abuse/Adapters/TimeLimit/Redis.php index fc4cb8fc64..19602613a2 100644 --- a/src/Abuse/Adapters/TimeLimit/Redis.php +++ b/src/Abuse/Adapters/TimeLimit/Redis.php @@ -84,38 +84,22 @@ class Redis extends TimeLimit * * @param int|null $offset * @param int|null $limit - * @return array + * @return array */ public function getLogs(?int $offset = null, ?int $limit = 25): array { - $cursor = 0; - $matches = []; - $pattern = self::NAMESPACE . '__*'; + // TODO limit potential is SCAN but needs cursor no offset + $cursor = null; + $keys = $this->redis->scan($cursor, self::NAMESPACE . '__*', $limit); + if (!$keys) { + return []; + } - do { - $result = $this->redis->scan($cursor, $pattern, $limit); - - // Early return if scan failed - if (!is_array($result)) { - return []; - } - - [$newCursor, $keys] = $result; - $cursor = (int)$newCursor; - - if (!empty($keys)) { - $values = $this->redis->mget($keys); - if (is_array($values)) { - /** @var array */ - $combinedArray = array_combine($keys, $values); - if ($combinedArray !== false) { - $matches = array_merge($matches, $combinedArray); - } - } - } - } while ($cursor > 0 && (!$limit || count($matches) < $limit)); - - return $matches; + $logs = []; + foreach ($keys as $key) { + $logs[$key] = $this->redis->get($key); + } + return $logs; } /** From b63b78274506e617ce6e2f93f2bcddbd970014eb Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 12:15:56 +0000 Subject: [PATCH 08/13] chore: fmt --- src/Abuse/Adapters/TimeLimit/RedisCluster.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Abuse/Adapters/TimeLimit/RedisCluster.php b/src/Abuse/Adapters/TimeLimit/RedisCluster.php index 47fa140b2e..72c297edbf 100644 --- a/src/Abuse/Adapters/TimeLimit/RedisCluster.php +++ b/src/Abuse/Adapters/TimeLimit/RedisCluster.php @@ -70,7 +70,7 @@ class RedisCluster extends TimeLimit } $key = self::NAMESPACE . '__'. $key .'__'. $timestamp; - + $this->redis->multi(\RedisCluster::MULTI); $this->redis->incr($key); $this->redis->expire($key, $this->ttl); From d7b8cf59966511fc7181f5ed26e50b780e93430b Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 12:17:16 +0000 Subject: [PATCH 09/13] fix: tests --- src/Abuse/Adapters/TimeLimit/RedisCluster.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Abuse/Adapters/TimeLimit/RedisCluster.php b/src/Abuse/Adapters/TimeLimit/RedisCluster.php index 72c297edbf..6acc99a8f4 100644 --- a/src/Abuse/Adapters/TimeLimit/RedisCluster.php +++ b/src/Abuse/Adapters/TimeLimit/RedisCluster.php @@ -71,7 +71,7 @@ class RedisCluster extends TimeLimit $key = self::NAMESPACE . '__'. $key .'__'. $timestamp; - $this->redis->multi(\RedisCluster::MULTI); + $this->redis->multi(); $this->redis->incr($key); $this->redis->expire($key, $this->ttl); $this->redis->exec(); From 91091d66e97fa02c25f02ae35f286937b87ed8cf Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 12:27:21 +0000 Subject: [PATCH 10/13] chore: remove test cleanup --- tests/Abuse/Base.php | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tests/Abuse/Base.php b/tests/Abuse/Base.php index a1d008fffe..5cb59c7da7 100644 --- a/tests/Abuse/Base.php +++ b/tests/Abuse/Base.php @@ -86,26 +86,6 @@ abstract class Base extends TestCase $this->assertEquals($abuse->check(), false); } - /** - * Test logs are deleted after cleanup - */ - public function testCleanup(): void - { - $adapter = $this->getAdapter('', 1, 1); - $abuse = new Abuse($adapter); - - $logs = $abuse->getLogs(0, 100); - $this->assertEquals(6, \count($logs)); // 6 keys are created in the test - - sleep(1); - - $status = $abuse->cleanup(time()); - $this->assertEquals($status, true); - - $logs = $abuse->getLogs(0, 100); - $this->assertEquals(0, \count($logs)); - } - /** * Verify that the time format is correct */ From a59fe2a949e499db56647e28e01511ab0ef3e9bd Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 12:37:59 +0000 Subject: [PATCH 11/13] chore: fix benchmarking --- composer.lock | 12 ++++++------ phpbench.json | 10 ++++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/composer.lock b/composer.lock index 0ec24c8c21..e9343924bd 100644 --- a/composer.lock +++ b/composer.lock @@ -149,16 +149,16 @@ }, { "name": "google/protobuf", - "version": "v4.29.1", + "version": "v4.29.2", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "6042b5483f8029e42473faeb8ef75ba266278381" + "reference": "79aa5014efeeec3d137df5cdb0ae2fc163953945" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/6042b5483f8029e42473faeb8ef75ba266278381", - "reference": "6042b5483f8029e42473faeb8ef75ba266278381", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/79aa5014efeeec3d137df5cdb0ae2fc163953945", + "reference": "79aa5014efeeec3d137df5cdb0ae2fc163953945", "shasum": "" }, "require": { @@ -187,9 +187,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.1" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.2" }, - "time": "2024-12-03T22:07:45+00:00" + "time": "2024-12-18T14:11:12+00:00" }, { "name": "jean85/pretty-package-versions", diff --git a/phpbench.json b/phpbench.json index adc40d1294..45395c8d87 100644 --- a/phpbench.json +++ b/phpbench.json @@ -1,6 +1,8 @@ { - "$schema":"vendor/phpbench/phpbench/phpbench.schema.json", - "runner.bootstrap": "vendor/autoload.php", - "runner.path": "tests", - "runner.file_pattern": "*Bench.php" + "$schema": "vendor/phpbench/phpbench/phpbench.schema.json", + "runner": { + "bootstrap": "vendor/autoload.php", + "path": "tests/Abuse/Bench", + "file_pattern": "*.php" + } } \ No newline at end of file From 440ebd62c35b79f176d4ec5674f91a6d5528d6e3 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 12:39:32 +0000 Subject: [PATCH 12/13] fix: bench --- composer.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/composer.json b/composer.json index 5edc270736..cc801b6c31 100755 --- a/composer.json +++ b/composer.json @@ -29,5 +29,11 @@ "phpstan/phpstan": "^1.9", "laravel/pint": "1.5.*", "phpbench/phpbench": "^1.2" + }, + "config": { + "allow-plugins": { + "php-http/discovery": true, + "tbachert/spi": true + } } } From 072725daa471ae53eb7483d06ed9689632525e0c Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 27 Dec 2024 12:44:21 +0000 Subject: [PATCH 13/13] chore: bench --- phpbench.json | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/phpbench.json b/phpbench.json index 45395c8d87..19e8aa0594 100644 --- a/phpbench.json +++ b/phpbench.json @@ -1,8 +1,6 @@ { "$schema": "vendor/phpbench/phpbench/phpbench.schema.json", - "runner": { - "bootstrap": "vendor/autoload.php", - "path": "tests/Abuse/Bench", - "file_pattern": "*.php" - } + "runner.bootstrap": "vendor/autoload.php", + "runner.path": "tests/Abuse/Bench", + "runner.file_pattern": "*.php" } \ No newline at end of file