mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Merge pull request #92 from utopia-php/feat-improve-redis
feat: improve redis
This commit is contained in:
7 files changed
+70
-136
No files matched your search
@@ -29,5 +29,11 @@
|
||||
"phpstan/phpstan": "^1.9",
|
||||
"laravel/pint": "1.5.*",
|
||||
"phpbench/phpbench": "^1.2"
|
||||
},
|
||||
"config": {
|
||||
"allow-plugins": {
|
||||
"php-http/discovery": true,
|
||||
"tbachert/spi": true
|
||||
}
|
||||
}
|
||||
}
|
||||
Generated
+6
-6
@@ -149,16 +149,16 @@
|
||||
},
|
||||
{
|
||||
"name": "google/protobuf",
|
||||
"version": "v4.29.1",
|
||||
"version": "v4.29.2",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/protocolbuffers/protobuf-php.git",
|
||||
"reference": "6042b5483f8029e42473faeb8ef75ba266278381"
|
||||
"reference": "79aa5014efeeec3d137df5cdb0ae2fc163953945"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/6042b5483f8029e42473faeb8ef75ba266278381",
|
||||
"reference": "6042b5483f8029e42473faeb8ef75ba266278381",
|
||||
"url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/79aa5014efeeec3d137df5cdb0ae2fc163953945",
|
||||
"reference": "79aa5014efeeec3d137df5cdb0ae2fc163953945",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
@@ -187,9 +187,9 @@
|
||||
"proto"
|
||||
],
|
||||
"support": {
|
||||
"source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.1"
|
||||
"source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.2"
|
||||
},
|
||||
"time": "2024-12-03T22:07:45+00:00"
|
||||
"time": "2024-12-18T14:11:12+00:00"
|
||||
},
|
||||
{
|
||||
"name": "jean85/pretty-package-versions",
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
version: '3'
|
||||
|
||||
services:
|
||||
mysql:
|
||||
image: mysql:8
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema":"vendor/phpbench/phpbench/phpbench.schema.json",
|
||||
"$schema": "vendor/phpbench/phpbench/phpbench.schema.json",
|
||||
"runner.bootstrap": "vendor/autoload.php",
|
||||
"runner.path": "tests",
|
||||
"runner.file_pattern": "*Bench.php"
|
||||
"runner.path": "tests/Abuse/Bench",
|
||||
"runner.file_pattern": "*.php"
|
||||
}
|
||||
@@ -13,10 +13,16 @@ class Redis extends TimeLimit
|
||||
*/
|
||||
protected \Redis $redis;
|
||||
|
||||
/**
|
||||
* @var int
|
||||
*/
|
||||
protected int $ttl;
|
||||
|
||||
public function __construct(string $key, int $limit, int $seconds, \Redis $redis)
|
||||
{
|
||||
$this->redis = $redis;
|
||||
$this->key = $key;
|
||||
$this->ttl = $seconds;
|
||||
$now = \time();
|
||||
$this->timestamp = (int)($now - ($now % $seconds));
|
||||
$this->limit = $limit;
|
||||
@@ -62,16 +68,13 @@ class Redis extends TimeLimit
|
||||
return;
|
||||
}
|
||||
|
||||
/** @var string $count */
|
||||
$count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $timestamp);
|
||||
if (!$count) {
|
||||
$this->count = 0;
|
||||
} else {
|
||||
$this->count = intval($count);
|
||||
}
|
||||
$key = self::NAMESPACE . '__' . $key . '__' . $timestamp;
|
||||
$this->redis->multi()
|
||||
->incr($key)
|
||||
->expire($key, $this->ttl)
|
||||
->exec();
|
||||
|
||||
$this->redis->incr(self::NAMESPACE . '__'. $key .'__'. $timestamp);
|
||||
$this->count++;
|
||||
$this->count = ($this->count ?? 0) + 1;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -107,32 +110,7 @@ class Redis extends TimeLimit
|
||||
*/
|
||||
public function cleanup(int $timestamp): bool
|
||||
{
|
||||
$iterator = null;
|
||||
while ($iterator !== 0) {
|
||||
$keys = $this->redis->scan($iterator, self::NAMESPACE . '__*__*', 1000);
|
||||
$keys = $this->filterKeys($keys ? $keys : [], $timestamp);
|
||||
$this->redis->del($keys);
|
||||
}
|
||||
// No need for manual cleanup - Redis TTL handles this automatically
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter keys
|
||||
*
|
||||
* @param array<string> $keys
|
||||
* @param integer $timestamp
|
||||
* @return array<string>
|
||||
*/
|
||||
protected function filterKeys(array $keys, int $timestamp): array
|
||||
{
|
||||
$filteredKeys = [];
|
||||
foreach ($keys as $key) {
|
||||
$parts = explode('__', $key);
|
||||
$keyTimestamp = (int)end($parts); // Assuming the last part is always the timestamp
|
||||
if ($keyTimestamp < $timestamp) {
|
||||
$filteredKeys[] = $key;
|
||||
}
|
||||
}
|
||||
return $filteredKeys;
|
||||
}
|
||||
}
|
||||
@@ -13,10 +13,16 @@ class RedisCluster extends TimeLimit
|
||||
*/
|
||||
protected \RedisCluster $redis;
|
||||
|
||||
/**
|
||||
* @var int
|
||||
*/
|
||||
protected int $ttl;
|
||||
|
||||
public function __construct(string $key, int $limit, int $seconds, \RedisCluster $redis)
|
||||
{
|
||||
$this->redis = $redis;
|
||||
$this->key = $key;
|
||||
$this->ttl = $seconds;
|
||||
$now = \time();
|
||||
$this->timestamp = (int)($now - ($now % $seconds));
|
||||
$this->limit = $limit;
|
||||
@@ -63,22 +69,18 @@ class RedisCluster extends TimeLimit
|
||||
return;
|
||||
}
|
||||
|
||||
/** @var string|false $count */
|
||||
$count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $timestamp);
|
||||
if ($count === false) {
|
||||
$this->count = 0;
|
||||
} else {
|
||||
$this->count = intval($count);
|
||||
}
|
||||
$key = self::NAMESPACE . '__'. $key .'__'. $timestamp;
|
||||
|
||||
$this->redis->incr(self::NAMESPACE . '__'. $key .'__'. $timestamp);
|
||||
$this->count++;
|
||||
$this->redis->multi();
|
||||
$this->redis->incr($key);
|
||||
$this->redis->expire($key, $this->ttl);
|
||||
$this->redis->exec();
|
||||
|
||||
$this->count = ($this->count ?? 0) + 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get abuse logs
|
||||
*
|
||||
* Return logs with an offset and limit
|
||||
* Get abuse logs with proper cursor-based pagination
|
||||
*
|
||||
* @param int|null $offset
|
||||
* @param int|null $limit
|
||||
@@ -86,76 +88,46 @@ class RedisCluster extends TimeLimit
|
||||
*/
|
||||
public function getLogs(?int $offset = 0, ?int $limit = 25): array
|
||||
{
|
||||
// TODO limit potential is SCAN but needs cursor no offset
|
||||
$keys = $this->scan(self::NAMESPACE . '__*', $offset, $limit);
|
||||
if ($keys === false) {
|
||||
$offset = $offset ?? 0;
|
||||
$limit = $limit ?? 25;
|
||||
$matches = [];
|
||||
$pattern = self::NAMESPACE . '__*';
|
||||
|
||||
// Get all keys from each master
|
||||
foreach ($this->redis->_masters() as $master) {
|
||||
$cursor = null;
|
||||
do {
|
||||
/** @phpstan-ignore-next-line */
|
||||
$keys = $this->redis->scan($cursor, $master, $pattern, 100);
|
||||
if ($keys !== false) {
|
||||
$matches = array_merge($matches, $keys);
|
||||
}
|
||||
} while ($cursor > 0 && count($matches) < $offset + $limit);
|
||||
}
|
||||
|
||||
// Sort to ensure consistent ordering
|
||||
sort($matches);
|
||||
|
||||
// Apply offset and limit
|
||||
$matches = array_slice($matches, $offset, $limit);
|
||||
|
||||
if (empty($matches)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$logs = [];
|
||||
foreach ($keys as $key) {
|
||||
$logs[$key] = $this->redis->get($key);
|
||||
}
|
||||
return $logs;
|
||||
// Batch fetch values using mget
|
||||
$values = $this->redis->mget($matches);
|
||||
return array_combine($matches, $values);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete all logs older than $timestamp
|
||||
* No need for manual cleanup - using Redis TTL
|
||||
*
|
||||
* @param int $timestamp
|
||||
* @return bool
|
||||
*/
|
||||
public function cleanup(int $timestamp): bool
|
||||
{
|
||||
$keys = $this->scan(self::NAMESPACE . '__*__*');
|
||||
$keys = $this->filterKeys($keys ? $keys : [], $timestamp);
|
||||
/** @phpstan-ignore-next-line */
|
||||
$this->redis->del($keys);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter keys
|
||||
*
|
||||
* @param array<string> $keys
|
||||
* @param integer $timestamp
|
||||
* @return array<string>
|
||||
*/
|
||||
protected function filterKeys(array $keys, int $timestamp): array
|
||||
{
|
||||
$filteredKeys = [];
|
||||
foreach ($keys as $key) {
|
||||
$parts = explode('__', $key);
|
||||
$keyTimestamp = (int)end($parts); // Assuming the last part is always the timestamp
|
||||
if ($keyTimestamp < $timestamp) {
|
||||
$filteredKeys[] = $key;
|
||||
}
|
||||
}
|
||||
return $filteredKeys;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan keys across all masters in the Redis cluster
|
||||
*
|
||||
* @param string $pattern Pattern to match keys
|
||||
* @param int|null $cursor Reference to the cursor for scanning
|
||||
* @param int|null $count Number of keys to return per iteration
|
||||
* @return array<string>|false
|
||||
*/
|
||||
protected function scan(string $pattern, ?int &$cursor = null, ?int $count = 1000): array|false
|
||||
{
|
||||
$matches = [];
|
||||
foreach ($this->redis->_masters() as $master) {
|
||||
$cursor = null;
|
||||
do {
|
||||
/** @phpstan-ignore-next-line */
|
||||
$keys = $this->redis->scan($cursor, $master, $pattern, $count);
|
||||
if ($keys !== false) {
|
||||
$matches = array_merge($matches, $keys);
|
||||
}
|
||||
} while ($cursor > 0);
|
||||
}
|
||||
|
||||
return empty($matches) ? false : $matches;
|
||||
}
|
||||
}
|
||||
@@ -86,26 +86,6 @@ abstract class Base extends TestCase
|
||||
$this->assertEquals($abuse->check(), false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test logs are deleted after cleanup
|
||||
*/
|
||||
public function testCleanup(): void
|
||||
{
|
||||
$adapter = $this->getAdapter('', 1, 1);
|
||||
$abuse = new Abuse($adapter);
|
||||
|
||||
$logs = $abuse->getLogs(0, 100);
|
||||
$this->assertEquals(6, \count($logs)); // 6 keys are created in the test
|
||||
|
||||
sleep(1);
|
||||
|
||||
$status = $abuse->cleanup(time());
|
||||
$this->assertEquals($status, true);
|
||||
|
||||
$logs = $abuse->getLogs(0, 100);
|
||||
$this->assertEquals(0, \count($logs));
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify that the time format is correct
|
||||
*/
|
||||
|
||||
Reference in new issue
Block a user