Merge pull request #92 from utopia-php/feat-improve-redis

feat: improve redis
This commit is contained in:
Eldad A. Fux authored and GitHub committed 2024-12-27 18:46:08 +01:00
commit bef426a9a2
7 files changed
+70 -136

No files matched your search

+6
View File
@@ -29,5 +29,11 @@
"phpstan/phpstan": "^1.9",
"laravel/pint": "1.5.*",
"phpbench/phpbench": "^1.2"
},
"config": {
"allow-plugins": {
"php-http/discovery": true,
"tbachert/spi": true
}
}
}
Generated
+6 -6
View File
@@ -149,16 +149,16 @@
},
{
"name": "google/protobuf",
"version": "v4.29.1",
"version": "v4.29.2",
"source": {
"type": "git",
"url": "https://github.com/protocolbuffers/protobuf-php.git",
"reference": "6042b5483f8029e42473faeb8ef75ba266278381"
"reference": "79aa5014efeeec3d137df5cdb0ae2fc163953945"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/6042b5483f8029e42473faeb8ef75ba266278381",
"reference": "6042b5483f8029e42473faeb8ef75ba266278381",
"url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/79aa5014efeeec3d137df5cdb0ae2fc163953945",
"reference": "79aa5014efeeec3d137df5cdb0ae2fc163953945",
"shasum": ""
},
"require": {
@@ -187,9 +187,9 @@
"proto"
],
"support": {
"source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.1"
"source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.2"
},
"time": "2024-12-03T22:07:45+00:00"
"time": "2024-12-18T14:11:12+00:00"
},
{
"name": "jean85/pretty-package-versions",
-2
View File
@@ -1,5 +1,3 @@
version: '3'
services:
mysql:
image: mysql:8
+3 -3
View File
@@ -1,6 +1,6 @@
{
"$schema":"vendor/phpbench/phpbench/phpbench.schema.json",
"$schema": "vendor/phpbench/phpbench/phpbench.schema.json",
"runner.bootstrap": "vendor/autoload.php",
"runner.path": "tests",
"runner.file_pattern": "*Bench.php"
"runner.path": "tests/Abuse/Bench",
"runner.file_pattern": "*.php"
}
+13 -35
View File
@@ -13,10 +13,16 @@ class Redis extends TimeLimit
*/
protected \Redis $redis;
/**
* @var int
*/
protected int $ttl;
public function __construct(string $key, int $limit, int $seconds, \Redis $redis)
{
$this->redis = $redis;
$this->key = $key;
$this->ttl = $seconds;
$now = \time();
$this->timestamp = (int)($now - ($now % $seconds));
$this->limit = $limit;
@@ -62,16 +68,13 @@ class Redis extends TimeLimit
return;
}
/** @var string $count */
$count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $timestamp);
if (!$count) {
$this->count = 0;
} else {
$this->count = intval($count);
}
$key = self::NAMESPACE . '__' . $key . '__' . $timestamp;
$this->redis->multi()
->incr($key)
->expire($key, $this->ttl)
->exec();
$this->redis->incr(self::NAMESPACE . '__'. $key .'__'. $timestamp);
$this->count++;
$this->count = ($this->count ?? 0) + 1;
}
/**
@@ -107,32 +110,7 @@ class Redis extends TimeLimit
*/
public function cleanup(int $timestamp): bool
{
$iterator = null;
while ($iterator !== 0) {
$keys = $this->redis->scan($iterator, self::NAMESPACE . '__*__*', 1000);
$keys = $this->filterKeys($keys ? $keys : [], $timestamp);
$this->redis->del($keys);
}
// No need for manual cleanup - Redis TTL handles this automatically
return true;
}
/**
* Filter keys
*
* @param array<string> $keys
* @param integer $timestamp
* @return array<string>
*/
protected function filterKeys(array $keys, int $timestamp): array
{
$filteredKeys = [];
foreach ($keys as $key) {
$parts = explode('__', $key);
$keyTimestamp = (int)end($parts); // Assuming the last part is always the timestamp
if ($keyTimestamp < $timestamp) {
$filteredKeys[] = $key;
}
}
return $filteredKeys;
}
}
+42 -70
View File
@@ -13,10 +13,16 @@ class RedisCluster extends TimeLimit
*/
protected \RedisCluster $redis;
/**
* @var int
*/
protected int $ttl;
public function __construct(string $key, int $limit, int $seconds, \RedisCluster $redis)
{
$this->redis = $redis;
$this->key = $key;
$this->ttl = $seconds;
$now = \time();
$this->timestamp = (int)($now - ($now % $seconds));
$this->limit = $limit;
@@ -63,22 +69,18 @@ class RedisCluster extends TimeLimit
return;
}
/** @var string|false $count */
$count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $timestamp);
if ($count === false) {
$this->count = 0;
} else {
$this->count = intval($count);
}
$key = self::NAMESPACE . '__'. $key .'__'. $timestamp;
$this->redis->incr(self::NAMESPACE . '__'. $key .'__'. $timestamp);
$this->count++;
$this->redis->multi();
$this->redis->incr($key);
$this->redis->expire($key, $this->ttl);
$this->redis->exec();
$this->count = ($this->count ?? 0) + 1;
}
/**
* Get abuse logs
*
* Return logs with an offset and limit
* Get abuse logs with proper cursor-based pagination
*
* @param int|null $offset
* @param int|null $limit
@@ -86,76 +88,46 @@ class RedisCluster extends TimeLimit
*/
public function getLogs(?int $offset = 0, ?int $limit = 25): array
{
// TODO limit potential is SCAN but needs cursor no offset
$keys = $this->scan(self::NAMESPACE . '__*', $offset, $limit);
if ($keys === false) {
$offset = $offset ?? 0;
$limit = $limit ?? 25;
$matches = [];
$pattern = self::NAMESPACE . '__*';
// Get all keys from each master
foreach ($this->redis->_masters() as $master) {
$cursor = null;
do {
/** @phpstan-ignore-next-line */
$keys = $this->redis->scan($cursor, $master, $pattern, 100);
if ($keys !== false) {
$matches = array_merge($matches, $keys);
}
} while ($cursor > 0 && count($matches) < $offset + $limit);
}
// Sort to ensure consistent ordering
sort($matches);
// Apply offset and limit
$matches = array_slice($matches, $offset, $limit);
if (empty($matches)) {
return [];
}
$logs = [];
foreach ($keys as $key) {
$logs[$key] = $this->redis->get($key);
}
return $logs;
// Batch fetch values using mget
$values = $this->redis->mget($matches);
return array_combine($matches, $values);
}
/**
* Delete all logs older than $timestamp
* No need for manual cleanup - using Redis TTL
*
* @param int $timestamp
* @return bool
*/
public function cleanup(int $timestamp): bool
{
$keys = $this->scan(self::NAMESPACE . '__*__*');
$keys = $this->filterKeys($keys ? $keys : [], $timestamp);
/** @phpstan-ignore-next-line */
$this->redis->del($keys);
return true;
}
/**
* Filter keys
*
* @param array<string> $keys
* @param integer $timestamp
* @return array<string>
*/
protected function filterKeys(array $keys, int $timestamp): array
{
$filteredKeys = [];
foreach ($keys as $key) {
$parts = explode('__', $key);
$keyTimestamp = (int)end($parts); // Assuming the last part is always the timestamp
if ($keyTimestamp < $timestamp) {
$filteredKeys[] = $key;
}
}
return $filteredKeys;
}
/**
* Scan keys across all masters in the Redis cluster
*
* @param string $pattern Pattern to match keys
* @param int|null $cursor Reference to the cursor for scanning
* @param int|null $count Number of keys to return per iteration
* @return array<string>|false
*/
protected function scan(string $pattern, ?int &$cursor = null, ?int $count = 1000): array|false
{
$matches = [];
foreach ($this->redis->_masters() as $master) {
$cursor = null;
do {
/** @phpstan-ignore-next-line */
$keys = $this->redis->scan($cursor, $master, $pattern, $count);
if ($keys !== false) {
$matches = array_merge($matches, $keys);
}
} while ($cursor > 0);
}
return empty($matches) ? false : $matches;
}
}
-20
View File
@@ -86,26 +86,6 @@ abstract class Base extends TestCase
$this->assertEquals($abuse->check(), false);
}
/**
* Test logs are deleted after cleanup
*/
public function testCleanup(): void
{
$adapter = $this->getAdapter('', 1, 1);
$abuse = new Abuse($adapter);
$logs = $abuse->getLogs(0, 100);
$this->assertEquals(6, \count($logs)); // 6 keys are created in the test
sleep(1);
$status = $abuse->cleanup(time());
$this->assertEquals($status, true);
$logs = $abuse->getLogs(0, 100);
$this->assertEquals(0, \count($logs));
}
/**
* Verify that the time format is correct
*/