Merge pull request #86 from utopia-php/add-support-for-sharding-adapter

feat: add support for redis cluster
This commit is contained in:
Christy Jacob authored and GitHub committed 2024-12-18 20:39:47 +05:30
commit 25b9654ddf
14 files changed
+323 -47

No files matched your search

Generated
+8 -8
View File
@@ -758,13 +758,13 @@
},
"type": "library",
"extra": {
"branch-alias": {
"dev-main": "1.0.x-dev"
},
"spi": {
"OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\HookManagerInterface": [
"OpenTelemetry\\API\\Instrumentation\\AutoInstrumentation\\ExtensionHookManager"
]
},
"branch-alias": {
"dev-main": "1.0.x-dev"
}
},
"autoload": {
@@ -3040,16 +3040,16 @@
},
{
"name": "phpstan/phpstan",
"version": "1.12.12",
"version": "1.12.13",
"source": {
"type": "git",
"url": "https://github.com/phpstan/phpstan.git",
"reference": "b5ae1b88f471d3fd4ba1aa0046234b5ca3776dd0"
"reference": "9b469068840cfa031e1deaf2fa1886d00e20680f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/phpstan/phpstan/zipball/b5ae1b88f471d3fd4ba1aa0046234b5ca3776dd0",
"reference": "b5ae1b88f471d3fd4ba1aa0046234b5ca3776dd0",
"url": "https://api.github.com/repos/phpstan/phpstan/zipball/9b469068840cfa031e1deaf2fa1886d00e20680f",
"reference": "9b469068840cfa031e1deaf2fa1886d00e20680f",
"shasum": ""
},
"require": {
@@ -3094,7 +3094,7 @@
"type": "github"
}
],
"time": "2024-11-28T22:13:23+00:00"
"time": "2024-12-17T17:00:20+00:00"
},
{
"name": "phpunit/php-code-coverage",
+57
View File
@@ -10,6 +10,15 @@ services:
ports:
- "9307:3306"
adminer:
image: adminer
container_name: abuse-adminer
restart: always
ports:
- 9506:8080
networks:
- abuse
redis:
image: redis:6.0-alpine
container_name: redis
@@ -30,5 +39,53 @@ services:
- ./src:/code/src
- ./tests:/code/tests
redis-insight:
image: redis/redisinsight:latest
restart: unless-stopped
networks:
- abuse
environment:
- REDIS_HOSTS=redis-cluster-0:6379
ports:
- "8081:5540"
redis-cluster-0:
image: docker.io/bitnami/redis-cluster:7.4
environment:
- ALLOW_EMPTY_PASSWORD=yes
- REDIS_NODES=redis-cluster-0 redis-cluster-1 redis-cluster-2 redis-cluster-3
- REDIS_CLUSTER_CREATOR=yes
- REDIS_CLUSTER_REPLICAS=0
networks:
- abuse
depends_on:
- redis-cluster-1
- redis-cluster-2
- redis-cluster-3
redis-cluster-1:
image: docker.io/bitnami/redis-cluster:7.4
environment:
- ALLOW_EMPTY_PASSWORD=yes
- REDIS_NODES=redis-cluster-0 redis-cluster-1 redis-cluster-2 redis-cluster-3
networks:
- abuse
redis-cluster-2:
image: docker.io/bitnami/redis-cluster:7.4
environment:
- ALLOW_EMPTY_PASSWORD=yes
- REDIS_NODES=redis-cluster-0 redis-cluster-1 redis-cluster-2 redis-cluster-3
networks:
- abuse
redis-cluster-3:
image: docker.io/bitnami/redis-cluster:7.4
environment:
- ALLOW_EMPTY_PASSWORD=yes
- REDIS_NODES=redis-cluster-0 redis-cluster-1 redis-cluster-2 redis-cluster-3
networks:
- abuse
networks:
abuse:
+1 -1
View File
@@ -6,7 +6,7 @@
convertNoticesToExceptions="true"
convertWarningsToExceptions="true"
processIsolation="false"
stopOnFailure="false"
stopOnFailure="true"
>
<testsuites>
<testsuite name="Application Test Suite">
-2
View File
@@ -37,8 +37,6 @@ abstract class TimeLimit extends Adapter
abstract protected function hit(string $key, string $datetime): void;
/**
* Check
*
@@ -1,8 +1,8 @@
<?php
namespace Utopia\Abuse\Adapters\Database;
namespace Utopia\Abuse\Adapters\TimeLimit;
use Utopia\Abuse\Adapters\TimeLimit as TimeLimitAdapter;
use Utopia\Abuse\Adapters\TimeLimit;
use Utopia\Database\Database as UtopiaDB;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
@@ -13,7 +13,7 @@ use Utopia\Database\Query;
use Utopia\Database\Validator\Authorization;
use Utopia\Exception;
class TimeLimit extends TimeLimitAdapter
class Database extends TimeLimit
{
public const COLLECTION = 'abuse';
@@ -106,7 +106,7 @@ class TimeLimit extends TimeLimitAdapter
try {
$this->db->createCollection(
TimeLimit::COLLECTION,
self::COLLECTION,
$attributes,
$indexes
);
@@ -138,7 +138,7 @@ class TimeLimit extends TimeLimitAdapter
/** @var array<Document> $result */
$result = Authorization::skip(function () use ($key, $datetime) {
return $this->db->find(TimeLimit::COLLECTION, [
return $this->db->find(self::COLLECTION, [
Query::equal('key', [$key]),
Query::equal('time', [$datetime]),
]);
@@ -170,7 +170,7 @@ class TimeLimit extends TimeLimitAdapter
}
Authorization::skip(function () use ($datetime, $key) {
$data = $this->db->findOne(TimeLimit::COLLECTION, [
$data = $this->db->findOne(self::COLLECTION, [
Query::equal('key', [$key]),
Query::equal('time', [$datetime]),
]);
@@ -181,14 +181,14 @@ class TimeLimit extends TimeLimitAdapter
'key' => $key,
'time' => $datetime,
'count' => 1,
'$collection' => TimeLimit::COLLECTION,
'$collection' => self::COLLECTION,
];
try {
$this->db->createDocument(TimeLimit::COLLECTION, new Document($data));
$this->db->createDocument(self::COLLECTION, new Document($data));
} catch (Duplicate $e) {
// Duplicate in case of race condition
$data = $this->db->findOne(TimeLimit::COLLECTION, [
$data = $this->db->findOne(self::COLLECTION, [
Query::equal('key', [$key]),
Query::equal('time', [$datetime]),
]);
@@ -198,14 +198,14 @@ class TimeLimit extends TimeLimitAdapter
if (\is_numeric($count)) {
$this->count = intval($count);
}
$this->db->increaseDocumentAttribute(TimeLimit::COLLECTION, $data->getId(), 'count');
$this->db->increaseDocumentAttribute(self::COLLECTION, $data->getId(), 'count');
} else {
throw new \Exception('Document Not Found');
}
}
} else {
/** @var Document $data */
$this->db->increaseDocumentAttribute(TimeLimit::COLLECTION, $data->getId(), 'count');
$this->db->increaseDocumentAttribute(self::COLLECTION, $data->getId(), 'count');
}
});
@@ -237,7 +237,7 @@ class TimeLimit extends TimeLimitAdapter
$queries[] = Query::limit($limit);
}
return $this->db->find(TimeLimit::COLLECTION, $queries);
return $this->db->find(self::COLLECTION, $queries);
});
return $results;
@@ -255,12 +255,12 @@ class TimeLimit extends TimeLimitAdapter
{
Authorization::skip(function () use ($datetime) {
do {
$documents = $this->db->find(TimeLimit::COLLECTION, [
$documents = $this->db->find(self::COLLECTION, [
Query::lessThan('time', $datetime),
]);
foreach ($documents as $document) {
$this->db->deleteDocument(TimeLimit::COLLECTION, $document->getId());
$this->db->deleteDocument(self::COLLECTION, $document->getId());
}
} while (! empty($documents));
});
@@ -1,20 +1,19 @@
<?php
namespace Utopia\Abuse\Adapters\Redis;
namespace Utopia\Abuse\Adapters\TimeLimit;
use Redis as Client;
use Utopia\Abuse\Adapters\TimeLimit as TimeLimitAdapter;
use Utopia\Abuse\Adapters\TimeLimit;
class TimeLimit extends TimeLimitAdapter
class Redis extends TimeLimit
{
public const NAMESPACE = 'abuse';
/**
* @var Client
* @var \Redis
*/
protected Client $redis;
protected \Redis $redis;
public function __construct(string $key, int $limit, int $seconds, Client $redis)
public function __construct(string $key, int $limit, int $seconds, \Redis $redis)
{
$this->redis = $redis;
$this->key = $key;
@@ -0,0 +1,161 @@
<?php
namespace Utopia\Abuse\Adapters\TimeLimit;
use Utopia\Abuse\Adapters\TimeLimit;
class RedisCluster extends TimeLimit
{
public const NAMESPACE = 'abuse';
/**
* @var \RedisCluster
*/
protected \RedisCluster $redis;
public function __construct(string $key, int $limit, int $seconds, \RedisCluster $redis)
{
$this->redis = $redis;
$this->key = $key;
$time = (int) \date('U', (int) (\floor(\time() / $seconds)) * $seconds);
$this->time = strval($time);
$this->limit = $limit;
}
/**
* Get count for a key at specific datetime
*
* @param string $key
* @param string $datetime
* @return integer
*/
protected function count(string $key, string $datetime): int
{
if (0 == $this->limit) { // No limit no point for counting
return 0;
}
if (! \is_null($this->count)) { // Get fetched result
return $this->count;
}
/** @var string|false $count */
$count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $datetime);
if ($count === false) {
$this->count = 0;
} else {
$this->count = intval($count);
}
return $this->count;
}
/**
* Record a hit for a key at specific datetime
*
* @param string $key
* @param string $datetime
* @return void
*/
protected function hit(string $key, string $datetime): void
{
if (0 == $this->limit) { // No limit no point for counting
return;
}
/** @var string|false $count */
$count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $datetime);
if ($count === false) {
$this->count = 0;
} else {
$this->count = intval($count);
}
$this->redis->incr(self::NAMESPACE . '__'. $key .'__'. $datetime);
$this->count++;
}
/**
* Get abuse logs
*
* Return logs with an offset and limit
*
* @param int|null $offset
* @param int|null $limit
* @return array<string, mixed>
*/
public function getLogs(?int $offset = 0, ?int $limit = 25): array
{
// TODO limit potential is SCAN but needs cursor no offset
$keys = $this->scan(self::NAMESPACE . '__*', $offset, $limit);
if ($keys === false) {
return [];
}
$logs = [];
foreach ($keys as $key) {
$logs[$key] = $this->redis->get($key);
}
return $logs;
}
/**
* Delete all logs older than $datetime
*
* @param string $datetime
* @return bool
*/
public function cleanup(string $datetime): bool
{
$keys = $this->scan(self::NAMESPACE . '__*__*');
$keys = $this->filterKeys($keys ? $keys : [], (int) $datetime);
/** @phpstan-ignore-next-line */
$this->redis->del($keys);
return true;
}
/**
* Filter keys
*
* @param array<string> $keys
* @param integer $timestamp
* @return array<string>
*/
protected function filterKeys(array $keys, int $timestamp): array
{
$filteredKeys = [];
foreach ($keys as $key) {
$parts = explode('__', $key);
$keyTimestamp = (int)end($parts); // Assuming the last part is always the timestamp
if ($keyTimestamp < $timestamp) {
$filteredKeys[] = $key;
}
}
return $filteredKeys;
}
/**
* Scan keys across all masters in the Redis cluster
*
* @param string $pattern Pattern to match keys
* @param int|null $cursor Reference to the cursor for scanning
* @param int|null $count Number of keys to return per iteration
* @return array<string>|false
*/
protected function scan(string $pattern, ?int &$cursor = null, ?int $count = 1000): array|false
{
$matches = [];
foreach ($this->redis->_masters() as $master) {
$cursor = null;
do {
/** @phpstan-ignore-next-line */
$keys = $this->redis->scan($cursor, $master, $pattern, $count);
if ($keys !== false) {
$matches = array_merge($matches, $keys);
}
} while ($cursor > 0);
}
return empty($matches) ? false : $matches;
}
}
+2 -2
View File
@@ -53,11 +53,11 @@ abstract class Base extends TestCase
public function testCleanup(): void
{
// Check that there is only one log
// Check that there are 3 logs
$logs = $this->abuse->getLogs(0, 10);
$this->assertEquals(3, \count($logs));
sleep(5);
sleep(2);
// Delete the log
$status = $this->abuse->cleanup($this->getCleanupDateTime());
@@ -1,10 +1,10 @@
<?php
namespace Abuse\Bench\Database;
namespace Abuse\Bench;
use PDO;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\Database\TimeLimit;
use Utopia\Abuse\Adapters\TimeLimit\Database as TimeLimit;
use Utopia\Cache\Adapter\None as NoCache;
use Utopia\Cache\Cache;
use Utopia\Database\Adapter\MariaDB;
@@ -13,7 +13,7 @@ use Utopia\Database\Database;
use Utopia\Exception;
use Utopia\Tests\Bench\Base;
class TimeLimitBench extends Base
class DatabaseBench extends Base
{
protected Database $db;
@@ -1,13 +1,13 @@
<?php
namespace Abuse\Bench\Redis;
namespace Abuse\Bench;
use Redis as Client;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\Redis\TimeLimit;
use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimit;
use Utopia\Tests\Bench\Base;
final class TimeLimitBench extends Base
final class RedisBench extends Base
{
protected Client $redis;
+23
View File
@@ -0,0 +1,23 @@
<?php
namespace Abuse\Bench;
use RedisCluster as Client;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\TimeLimit\RedisCluster;
use Utopia\Tests\Bench\Base;
final class RedisClusterBench extends Base
{
protected Client $redis;
/**
* @throws \Exception
*/
public function setUp(): void
{
$this->redis = new Client(null, ['redis-cluster-0:6379', 'redis-cluster-1:6379', 'redis-cluster-2:6379', 'redis-cluster-3:6379']);
$this->adapter = new RedisCluster('login-attempt-from-{{ip}}', 3, 60 * 5, $this->redis);
$this->abuse = new Abuse($this->adapter);
}
}
+4 -4
View File
@@ -5,7 +5,7 @@ namespace Utopia\Tests;
use PDO;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapter;
use Utopia\Abuse\Adapters\Database\TimeLimit as AdaptersDatabase;
use Utopia\Abuse\Adapters\TimeLimit\Database as TimeLimit;
use Utopia\Cache\Adapter\None as NoCache;
use Utopia\Cache\Cache;
use Utopia\Database\Adapter\MariaDB;
@@ -37,7 +37,7 @@ class DatabaseTest extends Base
$db->setNamespace('namespace');
$this->db = $db;
$adapter = new AdaptersDatabase('login-attempt-from-{{ip}}', 3, 60 * 5, $db);
$adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 60 * 5, $db);
if (! $db->exists('utopiaTests')) {
$db->create();
$adapter->setup();
@@ -49,11 +49,11 @@ class DatabaseTest extends Base
public function getAdapter(string $key, int $limit, int $seconds): Adapter
{
return new AdaptersDatabase($key, $limit, $seconds, $this->db);
return new TimeLimit($key, $limit, $seconds, $this->db);
}
public function getCleanupDateTime(): string
{
return DateTime::addSeconds(new \DateTime(), -1);
return DateTime::format(new \DateTime());
}
}
+38
View File
@@ -0,0 +1,38 @@
<?php
namespace Utopia\Tests;
use DateInterval;
use RedisCluster;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapter;
use Utopia\Abuse\Adapters\TimeLimit\RedisCluster as TimeLimit;
use Utopia\Exception;
class RedisClusterTest extends Base
{
protected RedisCluster $redis;
/**
* @throws Exception
* @throws \Exception
*/
public function setUp(): void
{
$this->redis = new RedisCluster(null, ['redis-cluster-0:6379', 'redis-cluster-1:6379', 'redis-cluster-2:6379', 'redis-cluster-3:6379']);
$adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 1, $this->redis);
$adapter->setParam('{{ip}}', '127.0.0.1');
$this->abuse = new Abuse($adapter);
}
public function getAdapter(string $key, int $limit, int $seconds): Adapter
{
return new TimeLimit($key, $limit, $seconds, $this->redis);
}
public function getCleanupDateTime(): string
{
$interval = DateInterval::createFromDateString('now');
return strval((new \DateTime())->sub($interval)->getTimestamp());
}
}
+3 -3
View File
@@ -6,7 +6,7 @@ use DateInterval;
use Redis as Client;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapter;
use Utopia\Abuse\Adapters\Redis\TimeLimit;
use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimit;
use Utopia\Exception;
class RedisTest extends Base
@@ -21,7 +21,7 @@ class RedisTest extends Base
{
$this->redis = new Client();
$this->redis->connect('redis', 6379);
$adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 60 * 5, $this->redis);
$adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 1, $this->redis);
$adapter->setParam('{{ip}}', '127.0.0.1');
$this->abuse = new Abuse($adapter);
}
@@ -33,7 +33,7 @@ class RedisTest extends Base
public function getCleanupDateTime(): string
{
$interval = DateInterval::createFromDateString(1 . ' seconds');
$interval = DateInterval::createFromDateString('now');
return strval((new \DateTime())->sub($interval)->getTimestamp());
}
}