diff --git a/.gitignore b/.gitignore index 62899f334c..56506c35eb 100755 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /vendor/ /.idea/ .phpunit.result.cache -tests/data \ No newline at end of file +tests/data +tests/Cache/E2E/tests/pool/ diff --git a/src/Cache/Adapter/CircuitBreaker.php b/src/Cache/Adapter/CircuitBreaker.php index 4e79c91059..fa9275936e 100644 --- a/src/Cache/Adapter/CircuitBreaker.php +++ b/src/Cache/Adapter/CircuitBreaker.php @@ -7,7 +7,7 @@ use Utopia\Cache\Feature; use Utopia\CircuitBreaker\CircuitBreaker as UtopiaCircuitBreaker; use Utopia\Telemetry\Adapter as Telemetry; -class CircuitBreaker implements Adapter, Feature\Telemetry +class CircuitBreaker implements Adapter, Feature\Leasable, Feature\Telemetry { public function __construct( private readonly Adapter $adapter, @@ -45,6 +45,30 @@ class CircuitBreaker implements Adapter, Feature\Telemetry return $result; } + public function getGeneration(string $key): string + { + if (! $this->adapter instanceof Feature\Leasable) { + return '0'; + } + + /** @var string $result */ + $result = $this->delegate(__FUNCTION__, \func_get_args(), '0'); + + return $result; + } + + public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array + { + if (! $this->adapter instanceof Feature\Leasable) { + return $this->save($key, $data, $hash); + } + + /** @var bool|string|array $result */ + $result = $this->delegate(__FUNCTION__, \func_get_args(), false); + + return $result; + } + public function touch(string $key, string $hash = ''): bool { /** @var bool $result */ diff --git a/src/Cache/Adapter/Pool.php b/src/Cache/Adapter/Pool.php index 5dc4714129..a2f8b3c367 100644 --- a/src/Cache/Adapter/Pool.php +++ b/src/Cache/Adapter/Pool.php @@ -3,9 +3,10 @@ namespace Utopia\Cache\Adapter; use Utopia\Cache\Adapter; +use Utopia\Cache\Feature\Leasable; use Utopia\Pools\Pool as UtopiaPool; -class Pool implements Adapter +class Pool implements Adapter, Leasable { /** * @var UtopiaPool @@ -59,6 +60,27 @@ class Pool implements Adapter return $result; } + public function getGeneration(string $key): string + { + return $this->pool->use(function (Adapter $adapter) use ($key): string { + return $adapter instanceof Leasable ? $adapter->getGeneration($key) : '0'; + }); + } + + public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array + { + /** + * @var bool|string|array $result + */ + $result = $this->pool->use(function (Adapter $adapter) use ($key, $data, $hash, $generation) { + return $adapter instanceof Leasable + ? $adapter->saveWithLease($key, $data, $hash, $generation) + : $adapter->save($key, $data, $hash); + }); + + return $result; + } + public function touch(string $key, string $hash = ''): bool { /** diff --git a/src/Cache/Adapter/Redis.php b/src/Cache/Adapter/Redis.php index 4427e99639..ae14401f98 100644 --- a/src/Cache/Adapter/Redis.php +++ b/src/Cache/Adapter/Redis.php @@ -7,10 +7,68 @@ use Redis as Client; use Throwable; use Utopia\Cache\Adapter; use Utopia\Cache\Adapter\Redis\Envelope; +use Utopia\Cache\Feature\Leasable; use Utopia\Cache\Feature\Retryable; -class Redis implements Adapter, Retryable +class Redis implements Adapter, Leasable, Retryable { + /** + * Reserved hash field that holds a key's generation alongside its value + * fields. Keeping the generation inside the value's own hash makes every + * lease operation single-key (one shard): no sidecar key to collide with or + * scan, and the scripts stay correct under Redis Cluster / multi-threaded + * backends (e.g. Dragonfly), where a multi-key script would span shards. + * Callers must not use this as a cache hash/field. + */ + private const GENERATION_FIELD = '__utopia_gen__'; + + /** + * Save $hash field into hash $key only when the key's generation field still + * equals the caller's token. Single-key. KEYS[1]=key; ARGV[1]=hash, + * ARGV[2]=value, ARGV[3]=expected generation. + */ + private const LUA_SAVE_WITH_LEASE = <<<'LUA' + local current = redis.call('HGET', KEYS[1], '__utopia_gen__') + if current == false then current = '0' end + if current ~= ARGV[3] then return 0 end + redis.call('HSET', KEYS[1], ARGV[1], ARGV[2]) + return 1 + LUA; + + /** + * Drop $key's value fields and advance its generation in one step, so an + * in-flight reader holding the previous generation cannot re-cache stale + * data. The bump is unconditional: a reader may have read a row the writer is + * now deleting before anything was cached, so the generation must move even + * when nothing was cached. Returns the number of value fields removed (the + * generation field is excluded), preserving purge() semantics. The + * generation field is the only thing that survives a purge, so it outlives + * any reader holding an older generation. Single-key. KEYS[1]=key. + */ + private const LUA_PURGE_BUMP = <<<'LUA' + local field = '__utopia_gen__' + local removed = redis.call('HLEN', KEYS[1]) - redis.call('HEXISTS', KEYS[1], field) + local current = redis.call('HGET', KEYS[1], field) + local next = (tonumber(current) or 0) + 1 + redis.call('DEL', KEYS[1]) + redis.call('HSET', KEYS[1], field, next) + return removed + LUA; + + /** + * Delete a single $hash field and advance the key's generation in one step, + * so a field-level purge invalidates in-flight leases just like a full purge. + * Returns the number of fields removed, preserving purge()'s deletion-result + * semantics. Single-key. KEYS[1]=key; ARGV[1]=field. + */ + private const LUA_PURGE_FIELD = <<<'LUA' + local removed = redis.call('HDEL', KEYS[1], ARGV[1]) + local current = redis.call('HGET', KEYS[1], '__utopia_gen__') + local next = (tonumber(current) or 0) + 1 + redis.call('HSET', KEYS[1], '__utopia_gen__', next) + return removed + LUA; + /** * @var Client */ @@ -101,6 +159,13 @@ class Redis implements Adapter, Retryable $hash = $key; } + // The generation lives in this same hash; never let a caller read, write + // or delete it through the public field API, or they could reset the + // generation and revive stale lease tokens. + if ($hash === self::GENERATION_FIELD) { + return false; + } + $redis_string = $this->execute(fn () => $this->redis->hGet($key, $hash)); if (! is_string($redis_string)) { @@ -126,6 +191,13 @@ class Redis implements Adapter, Retryable $hash = $key; } + // The generation lives in this same hash; never let a caller read, write + // or delete it through the public field API, or they could reset the + // generation and revive stale lease tokens. + if ($hash === self::GENERATION_FIELD) { + return false; + } + try { $value = Envelope::encode($data, time()); $this->execute(fn () => $this->redis->hSet($key, $hash, $value)); @@ -136,6 +208,44 @@ class Redis implements Adapter, Retryable } } + public function getGeneration(string $key): string + { + $gen = $this->execute(fn () => $this->redis->hGet($key, self::GENERATION_FIELD)); + + return \is_string($gen) ? $gen : '0'; + } + + public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array + { + if (empty($key) || empty($data)) { + return false; + } + + if (empty($hash)) { + $hash = $key; + } + + // The generation lives in this same hash; never let a caller read, write + // or delete it through the public field API, or they could reset the + // generation and revive stale lease tokens. + if ($hash === self::GENERATION_FIELD) { + return false; + } + + try { + $value = Envelope::encode($data, time()); + $stored = $this->execute(fn () => $this->redis->eval( + self::LUA_SAVE_WITH_LEASE, + [$key, $hash, $value, $generation], + 1 + )); + + return $stored ? $data : false; + } catch (Throwable $th) { + return false; + } + } + /** * @param string $key * @param string $hash optional @@ -147,6 +257,13 @@ class Redis implements Adapter, Retryable $hash = $key; } + // The generation lives in this same hash; never let a caller read, write + // or delete it through the public field API, or they could reset the + // generation and revive stale lease tokens. + if ($hash === self::GENERATION_FIELD) { + return false; + } + $redis_string = $this->execute(fn () => $this->redis->hGet($key, $hash)); if (! is_string($redis_string)) { @@ -174,7 +291,8 @@ class Redis implements Adapter, Retryable return []; } - return $keys; + // Don't expose the internal generation field as a listable cache field. + return \array_values(\array_filter($keys, fn (string $field): bool => $field !== self::GENERATION_FIELD)); } /** @@ -185,10 +303,25 @@ class Redis implements Adapter, Retryable public function purge(string $key, string $hash = ''): bool { if (! empty($hash)) { - return (bool) $this->execute(fn () => $this->redis->hdel($key, $hash)); + if ($hash === self::GENERATION_FIELD) { + return false; + } + + return (bool) $this->execute(fn () => $this->redis->eval( + self::LUA_PURGE_FIELD, + [$key, $hash], + 1 + )); } - return (bool) $this->execute(fn () => $this->redis->del($key)); + // Drop the value fields and advance the in-hash generation in one atomic + // step (single-key Redis Lua) so an in-flight reader holding the previous + // generation cannot re-cache stale data after this purge. + return (bool) $this->execute(fn () => $this->redis->eval( + self::LUA_PURGE_BUMP, + [$key], + 1 + )); } /** @@ -220,7 +353,12 @@ class Redis implements Adapter, Retryable */ public function getSize(): int { - /** @var int */ + // Trade-off of the in-hash generation: a purged key keeps a tiny hash + // holding only its generation field until it is re-cached, so it still + // counts here. Filtering those out would need an O(N) scan with a per-key + // HLEN check; for a diagnostic counter that isn't worth it, so DBSIZE may + // slightly over-count purged-but-not-yet-recached keys. + /** @var int $size */ $size = $this->execute(fn () => $this->redis->dbSize()); return $size; diff --git a/src/Cache/Adapter/Sharding.php b/src/Cache/Adapter/Sharding.php index 53f9c8b3da..e267362d2a 100644 --- a/src/Cache/Adapter/Sharding.php +++ b/src/Cache/Adapter/Sharding.php @@ -3,8 +3,9 @@ namespace Utopia\Cache\Adapter; use Utopia\Cache\Adapter; +use Utopia\Cache\Feature\Leasable; -class Sharding implements Adapter +class Sharding implements Adapter, Leasable { /** * @var Adapter[] @@ -66,6 +67,22 @@ class Sharding implements Adapter return $this->getAdapter($key)->save($key, $data, $hash); } + public function getGeneration(string $key): string + { + $adapter = $this->getAdapter($key); + + return $adapter instanceof Leasable ? $adapter->getGeneration($key) : '0'; + } + + public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array + { + $adapter = $this->getAdapter($key); + + return $adapter instanceof Leasable + ? $adapter->saveWithLease($key, $data, $hash, $generation) + : $adapter->save($key, $data, $hash); + } + /** * @param string $key * @param string $hash optional diff --git a/src/Cache/Cache.php b/src/Cache/Cache.php index 177e53fd90..4da205deac 100644 --- a/src/Cache/Cache.php +++ b/src/Cache/Cache.php @@ -2,6 +2,7 @@ namespace Utopia\Cache; +use Utopia\Cache\Feature\Leasable; use Utopia\Telemetry\Adapter as Telemetry; use Utopia\Telemetry\Adapter\None as NoTelemetry; use Utopia\Telemetry\Counter; @@ -140,6 +141,57 @@ class Cache } } + /** + * Current generation token for $key (advances on each purge). Returns '0' + * when the underlying adapter does not support leasing. + * + * @param string $key + * @return string + */ + public function getGeneration(string $key): string + { + if (! $this->adapter instanceof Leasable) { + return '0'; + } + + $key = $this->caseSensitive ? $key : \strtolower($key); + + return $this->adapter->getGeneration($key); + } + + /** + * Save $data only if the key's generation still equals $generation (i.e. no + * purge happened since the caller captured it via getGeneration). Closes the + * cache-aside read-after-write race. Falls back to an unconditional save when + * the adapter does not support leasing. + * + * @param string $key + * @param string|array $data + * @param string $hash + * @param string $generation + * @return bool|string|array + */ + public function saveWithLease(string $key, mixed $data, string $hash, string $generation): bool|string|array + { + $key = $this->caseSensitive ? $key : \strtolower($key); + $hash = $this->caseSensitive ? $hash : \strtolower($hash); + $start = microtime(true); + + try { + if (! $this->adapter instanceof Leasable) { + return $this->adapter->save($key, $data, $hash); + } + + return $this->adapter->saveWithLease($key, $data, $hash, $generation); + } finally { + $duration = microtime(true) - $start; + $this->getOperationDuration()->record($duration, [ + 'operation' => 'saveWithLease', + 'adapter' => $this->adapter->getName($key), + ]); + } + } + /** * Refresh a cache entry timestamp without replacing its data. * diff --git a/src/Cache/Feature/Leasable.php b/src/Cache/Feature/Leasable.php new file mode 100644 index 0000000000..c838e8f6a7 --- /dev/null +++ b/src/Cache/Feature/Leasable.php @@ -0,0 +1,33 @@ +|string $data + * @return bool|string|array + */ + public function saveWithLease(string $key, array|string $data, string $hash, string $generation): bool|string|array; +} diff --git a/tests/Cache/E2E/PoolTest.php b/tests/Cache/E2E/PoolTest.php index b52c7790ee..90663533cd 100644 --- a/tests/Cache/E2E/PoolTest.php +++ b/tests/Cache/E2E/PoolTest.php @@ -29,4 +29,14 @@ class PoolTest extends Base self::$cache->save('test', 'test'); $this->assertEquals(4, self::$cache->getSize()); } + + public function testLeaseFallbackForNonLeasableAdapter(): void + { + // The pool holds Filesystem adapters, which are not Leasable. The Pool + // adapter must degrade gracefully (no "Call to undefined method" fatal): + // getGeneration() returns '0' and saveWithLease() falls back to save(). + $this->assertSame('0', self::$cache->getGeneration('lease:fallback')); + $this->assertNotFalse(self::$cache->saveWithLease('lease:fallback', 'value', 'lease:fallback', '0')); + $this->assertSame('value', self::$cache->load('lease:fallback', 60, 'lease:fallback')); + } } diff --git a/tests/Cache/E2E/Redis/LeasableTest.php b/tests/Cache/E2E/Redis/LeasableTest.php new file mode 100644 index 0000000000..e33ac11aae --- /dev/null +++ b/tests/Cache/E2E/Redis/LeasableTest.php @@ -0,0 +1,144 @@ +connect('redis', 6379); + $this->cache = new Cache(new RedisAdapter($redis)); + $this->cache->flush(); + } + + public function testFreshKeyHasZeroGeneration(): void + { + $this->assertSame('0', $this->cache->getGeneration('doc:1')); + } + + public function testSaveWithLeaseStoresWhenGenerationUnchanged(): void + { + $generation = $this->cache->getGeneration('doc:1'); + + $this->assertNotFalse($this->cache->saveWithLease('doc:1', ['v' => 1], 'doc:1', $generation)); + $this->assertSame(['v' => 1], $this->cache->load('doc:1', 60, 'doc:1')); + } + + public function testPurgeAdvancesGeneration(): void + { + $before = $this->cache->getGeneration('doc:1'); + $this->cache->saveWithLease('doc:1', ['v' => 1], 'doc:1', $before); + $this->cache->purge('doc:1'); + + $this->assertNotSame($before, $this->cache->getGeneration('doc:1')); + } + + public function testHashPurgeAdvancesGeneration(): void + { + $captured = $this->cache->getGeneration('doc:1'); + $this->assertNotFalse($this->cache->saveWithLease('doc:1', ['v' => 1], 'field-a', $captured)); + + // A field-level purge must advance the per-key generation, just like a + // full purge, or the lease is bypassed for hashed entries. + $this->assertTrue($this->cache->purge('doc:1', 'field-a')); + $this->assertNotSame($captured, $this->cache->getGeneration('doc:1')); + + // A reader holding the pre-purge generation can no longer re-cache a + // stale field via saveWithLease. + $this->assertFalse($this->cache->saveWithLease('doc:1', ['stale' => true], 'field-a', $captured)); + $this->assertFalse($this->cache->load('doc:1', 60, 'field-a')); + } + + /** + * The core read-after-write guarantee: a reader that captured the generation + * BEFORE a concurrent purge must NOT be able to re-cache its now-stale value + * AFTER the purge. Ordering here simulates the race deterministically. + */ + public function testStaleSaveAfterPurgeIsRejected(): void + { + $captured = $this->cache->getGeneration('doc:1'); + + // A writer commits and purges; the generation advances. + $this->cache->purge('doc:1'); + + // The reader tries to re-cache the value it read before the purge. + $result = $this->cache->saveWithLease('doc:1', ['stale' => true], 'doc:1', $captured); + + $this->assertFalse($result, 'Stale re-cache must be rejected after a purge advanced the generation'); + $this->assertFalse($this->cache->load('doc:1', 60, 'doc:1'), 'Cache must not hold the stale value'); + } + + public function testFreshSaveAfterPurgeSucceeds(): void + { + $this->cache->purge('doc:1'); + + $generation = $this->cache->getGeneration('doc:1'); + $this->assertNotFalse($this->cache->saveWithLease('doc:1', ['v' => 2], 'doc:1', $generation)); + $this->assertSame(['v' => 2], $this->cache->load('doc:1', 60, 'doc:1')); + } + + public function testPurgePreservesDeletionResult(): void + { + $this->assertFalse($this->cache->purge('doc:absent'), 'Purging a missing key returns false'); + + $this->cache->saveWithLease('doc:1', ['v' => 1], 'doc:1', $this->cache->getGeneration('doc:1')); + $this->assertTrue($this->cache->purge('doc:1'), 'Purging an existing key returns true'); + } + + public function testPurgeDropsValueButKeepsGenerationField(): void + { + $this->cache->flush(); + $this->cache->saveWithLease('doc:1', ['v' => 1], 'doc:1', $this->cache->getGeneration('doc:1')); + $this->assertSame(['v' => 1], $this->cache->load('doc:1', 60, 'doc:1')); + + $this->cache->purge('doc:1'); + + // The value is gone, but the key keeps its in-hash generation so a stale + // reader is still rejected. The residual hash therefore persists and is + // counted by getSize() — the documented trade-off of storing the + // generation in-hash rather than in a separately-countable sidecar key. + $this->assertFalse($this->cache->load('doc:1', 60, 'doc:1')); + $this->assertSame('1', $this->cache->getGeneration('doc:1')); + $this->assertSame(1, $this->cache->getSize()); + } + + public function testReservedGenerationFieldIsProtected(): void + { + $this->cache->flush(); + $this->cache->saveWithLease('doc:1', ['v' => 1], 'doc:1', $this->cache->getGeneration('doc:1')); + $this->cache->purge('doc:1'); + $this->assertSame('1', $this->cache->getGeneration('doc:1')); + + // Reading, writing, or deleting the reserved generation field through the + // public API must be rejected, so a caller can't clobber it and reset the + // generation sequence (which would revive stale lease tokens). + $this->assertFalse($this->cache->save('doc:1', 'attacker', '__utopia_gen__')); + $this->assertFalse($this->cache->saveWithLease('doc:1', 'attacker', '__utopia_gen__', '1')); + $this->assertFalse($this->cache->purge('doc:1', '__utopia_gen__')); + $this->assertFalse($this->cache->load('doc:1', 60, '__utopia_gen__')); + + // Generation is untouched, so an old token is still rejected. + $this->assertSame('1', $this->cache->getGeneration('doc:1')); + $this->assertFalse($this->cache->saveWithLease('doc:1', 'stale', 'doc:1', '0')); + } + + public function testListHidesGenerationField(): void + { + $this->cache->flush(); + $this->cache->saveWithLease('doc:1', ['v' => 1], 'field-a', $this->cache->getGeneration('doc:1')); + $this->assertSame(['field-a'], $this->cache->list('doc:1')); + + // After a purge the key holds only its internal generation field, which + // must not surface as a listable cache field. + $this->cache->purge('doc:1'); + $this->assertSame([], $this->cache->list('doc:1')); + } +}