feat: refactor EventTailRegistry for improved subscription management and isolation

This commit is contained in:
ArnabChatterjee20k committed 2026-06-06 18:38:50 +05:30
1 parent 61227fddc4
commit 6b94c966b1
4 files changed
+272 -128

No files matched your search

+134 -103
View File
@@ -5,29 +5,32 @@ namespace Appwrite\Realtime;
use Appwrite\Utopia\Database\RuntimeQuery;
use Utopia\WebSocket\Server;
// token bucket + buffers per subId of a project per worker
/**
* Per-worker token-bucket + buffers for console event-tail subscriptions.
*
* Each tail of a single project is one entry, keyed by (connId, subId, projectId).
* Subscription IDs are only unique WITHIN a connection, so the connection id is part
* of the key — subscriptions from different connections (or different projects under
* the same subscription id) never collide. A subscription that names several tail
* channels gets one entry per project under the same (connId, subId).
*
* Source of truth: $entries[connId][subId][projectId] => state.
* Hot-path index: $projectIndex[projectId][connId][subId] => true.
*/
class EventTailRegistry
{
/** @var array<string, array{
* connId:int,
* projectId:string,
/** @var array<int, array<string, array<string, array{
* filters:array<string,mixed>,
* tokens:float,
* lastRefill:float,
* dropped:int,
* delivered:int,
* buffer:array<int,array<string,mixed>>
* }> keyed by subscriptionId */
private array $subs = [];
* }>>> connId => subId => projectId => state */
private array $entries = [];
/** @var array<string, array<string,true>> projectId => set of subIds (hot-path lookup) */
private array $projectSubscriptionsMap = [];
/** @var array<int, array<string,true>> connId => set of subIds (cleanup on close) */
private array $byConnection = [];
/** @var array<string,string> projectId => teamId (resolved once per worker at subscribe) */
private array $teamCache = [];
/** @var array<string, array<int, array<string,true>>> projectId => connId => set of subIds */
private array $projectIndex = [];
/**
* @param int $rate tokens per second per subscription
@@ -45,6 +48,10 @@ class EventTailRegistry
return CONSOLE_TAIL_CHANNEL_PREFIX . '.' . $projectId;
}
/**
* Extract the projectId from a `console.tail.<projectId>` channel, or null if the
* channel isn't a tail channel.
*/
public static function projectFromChannel(string $channel): ?string
{
$prefix = CONSOLE_TAIL_CHANNEL_PREFIX . '.';
@@ -55,26 +62,19 @@ class EventTailRegistry
return $projectId === '' ? null : $projectId;
}
public function cachedTeam(string $projectId): ?string
{
return $this->teamCache[$projectId] ?? null;
}
public function cacheTeam(string $projectId, string $teamId): void
{
$this->teamCache[$projectId] = $teamId;
}
/**
* Register a tail subscription owned by this worker.
* Register a tail of one project under (connId, subId). Additive, mirroring the
* realtime adapter's subscribe(): adding a new channel never drops channels already
* associated with the subscription id. Re-adding the SAME (connId, subId, projectId)
* overwrites in place with fresh state — that's how a reused subscription id updates
* its filter without leaving a stale entry behind. Entries are dropped only by
* remove()/removeConnection() (i.e. unsubscribe or disconnect), matching the tree.
*
* @param array<string,mixed> $filters result of RuntimeQuery::compile()
*/
public function add(int $connId, string $subId, string $projectId, array $filters, float $now): void
{
$this->subs[$subId] = [
'connId' => $connId,
'projectId' => $projectId,
$this->entries[$connId][$subId][$projectId] = [
'filters' => $filters,
'tokens' => (float) $this->rate,
'lastRefill' => $now,
@@ -82,110 +82,141 @@ class EventTailRegistry
'delivered' => 0,
'buffer' => [],
];
$this->projectSubscriptionsMap[$projectId][$subId] = true;
$this->byConnection[$connId][$subId] = true;
$this->projectIndex[$projectId][$connId][$subId] = true;
}
public function remove(string $subId): void
/**
* Remove every entry for a (connId, subId) — covers a subscription that tailed
* multiple projects. No-op for unknown subscriptions.
*/
public function remove(int $connId, string $subId): void
{
$sub = $this->subs[$subId] ?? null;
if ($sub === null) {
return;
foreach (\array_keys($this->entries[$connId][$subId] ?? []) as $projectId) {
$this->detachFromProjectIndex((string) $projectId, $connId, $subId);
}
unset($this->projectSubscriptionsMap[$sub['projectId']][$subId]);
if (empty($this->projectSubscriptionsMap[$sub['projectId']])) {
unset($this->projectSubscriptionsMap[$sub['projectId']]);
unset($this->entries[$connId][$subId]);
if (empty($this->entries[$connId])) {
unset($this->entries[$connId]);
}
unset($this->byConnection[$sub['connId']][$subId]);
if (empty($this->byConnection[$sub['connId']])) {
unset($this->byConnection[$sub['connId']]);
}
unset($this->subs[$subId]);
}
public function removeConnection(int $connId): void
{
foreach (\array_keys($this->byConnection[$connId] ?? []) as $subId) {
$this->remove($subId);
foreach (\array_keys($this->entries[$connId] ?? []) as $subId) {
$this->remove($connId, (string) $subId);
}
}
private function detachFromProjectIndex(string $projectId, int $connId, string $subId): void
{
unset($this->projectIndex[$projectId][$connId][$subId]);
if (empty($this->projectIndex[$projectId][$connId])) {
unset($this->projectIndex[$projectId][$connId]);
}
if (empty($this->projectIndex[$projectId])) {
unset($this->projectIndex[$projectId]);
}
}
/**
* O(1) gate for the hot path: does THIS worker hold any tail for the project?
*/
public function isTailed(string $projectId): bool
{
return isset($this->projectSubscriptionsMap[$projectId]);
return !empty($this->projectIndex[$projectId]);
}
/**
* For one decoded firehose event: filter FIRST, then sample into each matching
* entry's buffer. The sampling budget is therefore spent only on events that pass
* the filter.
*
* @param array<string,mixed> $payload compact metadata from Realtime::toTailMetadata()
*/
public function ingest(string $projectId, array $payload, float $now): void
{
foreach (\array_keys($this->projectSubscriptionsMap[$projectId] ?? []) as $subId) {
$sub = &$this->subs[$subId];
foreach (($this->projectIndex[$projectId] ?? []) as $connId => $subs) {
foreach (\array_keys($subs) as $subId) {
$entry = &$this->entries[$connId][$subId][$projectId];
if (RuntimeQuery::filter($sub['filters'], $payload) === null) {
continue;
// 1) FILTER FIRST — reuse RuntimeQuery against the compact frame
if (RuntimeQuery::filter($entry['filters'], $payload) === null) {
unset($entry);
continue;
}
// 2) THEN SAMPLE — lazy token-bucket refill + consume.
// tokens is a fractional value that refills with elapsed time, capped at rate.
$elapsed = \max(0.0, $now - $entry['lastRefill']);
$entry['tokens'] = \min((float) $this->rate, $entry['tokens'] + $elapsed * $this->rate);
$entry['lastRefill'] = $now;
if ($entry['tokens'] >= 1.0) {
$entry['tokens'] -= 1.0;
$entry['buffer'][] = $payload;
$entry['delivered']++;
} else {
$entry['dropped']++;
}
unset($entry);
}
// tokens will be always a fractional value lazily increasing with the rate
$elapsed = \max(0.0, $now - $sub['lastRefill']);
$sub['tokens'] = \min((float) $this->rate, $sub['tokens'] + $elapsed * $this->rate);
$sub['lastRefill'] = $now;
if ($sub['tokens'] >= 1.0) {
$sub['tokens'] -= 1.0;
$sub['buffer'][] = $payload;
$sub['delivered']++;
} else {
$sub['dropped']++;
}
unset($sub);
}
}
/**
* Flush all non-empty buffers as one (or more) batched websocket frame(s) per
* entry, plus a counter frame whenever events were dropped. Call from a Timer::tick
* inside onWorkerStart.
*/
public function flush(Server $server): void
{
foreach ($this->subs as $subId => &$sub) {
$channel = self::channel($sub['projectId']);
foreach ($this->entries as $connId => $bySub) {
foreach ($bySub as $subId => $byProject) {
foreach (\array_keys($byProject) as $projectId) {
$entry = &$this->entries[$connId][$subId][$projectId];
$channel = self::channel((string) $projectId);
if (!empty($sub['buffer'])) {
// Chunk so a single frame stays under the websocket package max length.
foreach (\array_chunk($sub['buffer'], $this->batchMax) as $chunk) {
$server->send([$sub['connId']], (string) \json_encode([
'type' => 'event',
'data' => [
'channels' => [$channel],
'events' => ['console.tail'],
'subscriptions' => [$subId],
'payload' => $chunk,
],
]));
if (!empty($entry['buffer'])) {
// Chunk so a single frame stays under the websocket package max length.
foreach (\array_chunk($entry['buffer'], $this->batchMax) as $chunk) {
$server->send([$connId], (string) \json_encode([
'type' => 'event',
'data' => [
'channels' => [$channel],
'events' => ['console.tail'],
'subscriptions' => [(string) $subId],
'payload' => $chunk,
],
]));
}
$entry['buffer'] = [];
}
// Counter frame — emit the true rate vs dropped count when sampling kicked in.
if ($entry['dropped'] > 0) {
$server->send([$connId], (string) \json_encode([
'type' => 'event',
'data' => [
'channels' => [$channel],
'events' => ['console.tail.stats'],
'subscriptions' => [(string) $subId],
'payload' => [
'$type' => 'tail.stats',
'delivered' => $entry['delivered'],
'dropped' => $entry['dropped'],
],
],
]));
}
// Reset counters every flush window (~150ms).
$entry['delivered'] = 0;
$entry['dropped'] = 0;
unset($entry);
}
$sub['buffer'] = [];
}
// Counter frame — emit the true rate vs dropped count when sampling kicked in.
if ($sub['dropped'] > 0) {
$server->send([$sub['connId']], (string) \json_encode([
'type' => 'event',
'data' => [
'channels' => [$channel],
'events' => ['console.tail.stats'],
'subscriptions' => [$subId],
'payload' => [
'$type' => 'tail.stats',
'delivered' => $sub['delivered'],
'dropped' => $sub['dropped'],
],
],
]));
}
// Reset counters every flush window (~150ms).
$sub['delivered'] = 0;
$sub['dropped'] = 0;
}
unset($sub);
}
}
@@ -90,22 +90,26 @@ class Subscribe extends Action
// Console live event tail: any `console.tail.<projectId>` channel is authorized by
// team membership of the target project's owning team, then registered with its
// compiled filter so the worker's pub/sub loop can filter+sample before delivery.
//
// Registration is additive, mirroring Realtime::subscribe(): a reused subscription
// id accumulates channels (it never drops the ones already subscribed), and the
// registry is keyed by (connId, subId, projectId) so re-subscribing the same
// channel overwrites its filter in place rather than leaving a stale entry.
$now = \microtime(true);
foreach ($parsedPayloads as $parsedPayload) {
$compiled = null;
foreach ($parsedPayload['convertedChannels'] as $channel) {
$targetProjectId = EventTailRegistry::projectFromChannel($channel);
if ($targetProjectId === null) {
continue;
}
$teamId = $eventTailRegistry->cachedTeam($targetProjectId);
if ($teamId === null) {
$project = $dbForConsole->getAuthorization()->skip(
fn () => $dbForConsole->getDocument('projects', $targetProjectId)
);
$teamId = (string) $project->getAttribute('teamId', '');
$eventTailRegistry->cacheTeam($targetProjectId, $teamId);
}
// Resolve ownership fresh on every subscribe — projects can be transferred
// between teams, so a cached teamId could authorize the wrong team.
$project = $dbForConsole->getAuthorization()->skip(
fn () => $dbForConsole->getDocument('projects', $targetProjectId)
);
$teamId = (string) $project->getAttribute('teamId', '');
if ($teamId === '' || !\in_array(Role::team($teamId)->toString(), $roles, true)) {
throw new Exception(
@@ -114,13 +118,8 @@ class Subscribe extends Action
);
}
$eventTailRegistry->add(
$connectionId,
$parsedPayload['subscriptionId'],
$targetProjectId,
RuntimeQuery::compile($parsedPayload['queries']),
$now,
);
$compiled ??= RuntimeQuery::compile($parsedPayload['queries']);
$eventTailRegistry->add($connectionId, $parsedPayload['subscriptionId'], $targetProjectId, $compiled, $now);
}
}
@@ -46,8 +46,9 @@ class Unsubscribe extends Action
'subscriptionId' => $subscriptionId,
'removed' => $realtime->unsubscribeSubscription($connectionId, $subscriptionId),
];
// No-op if it wasn't a tail subscription.
$eventTailRegistry->remove($subscriptionId);
// No-op if it wasn't a tail subscription. Connection-scoped: subscription
// IDs are only unique within a connection.
$eventTailRegistry->remove($connectionId, $subscriptionId);
}
$subscriptionsAfter = \count($realtime->getSubscriptionMetadata($connectionId));
+121 -8
View File
@@ -36,6 +36,23 @@ final class EventTailRegistryTest extends TestCase
return $server;
}
/**
* Like server(), but records the target connection id alongside each frame so tests
* can assert per-connection isolation.
*
* @param array<int,array{connId:mixed, frame:array<string,mixed>}> $sent
*/
private function serverWithConn(array &$sent): Server
{
$server = $this->createStub(Server::class);
$server->method('send')->willReturnCallback(
function (array $connections, string $message) use (&$sent): void {
$sent[] = ['connId' => $connections[0] ?? null, 'frame' => json_decode($message, true)];
}
);
return $server;
}
/** @param array<string,mixed> $payload */
private function compile(array $queries): array
{
@@ -345,22 +362,118 @@ final class EventTailRegistryTest extends TestCase
$registry->add(7, 'sub-a', 'projX', $this->compile([]), 0.0);
$registry->add(7, 'sub-b', 'projX', $this->compile([]), 0.0);
$registry->remove('sub-a');
$registry->remove(7, 'sub-a');
$this->assertTrue($registry->isTailed('projX'), 'sub-b still tails projX');
$registry->remove('sub-b');
$registry->remove(7, 'sub-b');
$this->assertFalse($registry->isTailed('projX'));
// Removing an unknown subscription is a no-op.
$registry->remove('does-not-exist');
$registry->remove(7, 'does-not-exist');
$this->assertFalse($registry->isTailed('projX'));
}
public function testTeamCache(): void
public function testSameSubscriptionIdOnDifferentConnectionsIsolated(): void
{
$registry = new EventTailRegistry();
$this->assertNull($registry->cachedTeam('projX'));
$registry->cacheTeam('projX', 'team-1');
$this->assertSame('team-1', $registry->cachedTeam('projX'));
// Subscription IDs are only unique within a connection. Two connections both
// using 'dup' must stay fully isolated — no cross-project/connection leakage.
$registry = new EventTailRegistry(rate: 100);
$registry->add(1, 'dup', 'projA', $this->compile([]), 0.0);
$registry->add(2, 'dup', 'projB', $this->compile([]), 0.0);
$registry->ingest('projA', ['type' => 'databases', 'resourceId' => 'a1'], 0.0);
$registry->ingest('projB', ['type' => 'databases', 'resourceId' => 'b1'], 0.0);
$sent = [];
$registry->flush($this->serverWithConn($sent));
$byConn = [];
foreach ($sent as $s) {
$byConn[$s['connId']][] = $s['frame']['data']['channels'][0];
}
// Connection 1 only ever receives projA's channel; connection 2 only projB's.
$this->assertSame(['console.tail.projA'], $byConn[1] ?? []);
$this->assertSame(['console.tail.projB'], $byConn[2] ?? []);
// Removing one connection's 'dup' must not touch the other's.
$registry->remove(1, 'dup');
$this->assertFalse($registry->isTailed('projA'));
$this->assertTrue($registry->isTailed('projB'));
}
public function testMultipleProjectsUnderOneSubscription(): void
{
// A subscription naming several tail channels tails each project independently;
// none overwrites another, and removing the subscription clears them all.
$registry = new EventTailRegistry(rate: 100);
$registry->add(1, 'sub-multi', 'projA', $this->compile([]), 0.0);
$registry->add(1, 'sub-multi', 'projB', $this->compile([]), 0.0);
$this->assertTrue($registry->isTailed('projA'));
$this->assertTrue($registry->isTailed('projB'));
$registry->ingest('projA', ['type' => 'databases', 'resourceId' => 'a1'], 0.0);
$registry->ingest('projB', ['type' => 'databases', 'resourceId' => 'b1'], 0.0);
$sent = [];
$registry->flush($this->serverWithConn($sent));
$channels = array_map(fn ($s) => $s['frame']['data']['channels'][0], $sent);
$this->assertContains('console.tail.projA', $channels);
$this->assertContains('console.tail.projB', $channels);
// Both frames carry the same (real) subscription id.
foreach ($sent as $s) {
$this->assertSame(['sub-multi'], $s['frame']['data']['subscriptions']);
}
$registry->remove(1, 'sub-multi');
$this->assertFalse($registry->isTailed('projA'));
$this->assertFalse($registry->isTailed('projB'));
}
public function testReAddingSameChannelOverwritesFilterWithoutStaleEntry(): void
{
// Reusing a subscription id on the SAME tail channel updates its filter in place
// (keyed by connId/subId/projectId) — no stale entry, no duplicate delivery.
$registry = new EventTailRegistry(rate: 100);
$registry->add(1, 'x', 'projA', $this->compile([Query::equal('action', ['create'])]), 0.0);
$registry->add(1, 'x', 'projA', $this->compile([Query::equal('action', ['delete'])]), 0.0);
$registry->ingest('projA', ['action' => 'create', 'resourceId' => 'c1'], 0.0);
$registry->ingest('projA', ['action' => 'delete', 'resourceId' => 'd1'], 0.0);
$sent = [];
$registry->flush($this->server($sent));
$eventFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail']));
// A lingering create-filter entry would surface the create and/or add a 2nd frame.
$this->assertCount(1, $eventFrames);
$this->assertCount(1, $eventFrames[0]['data']['payload']);
$this->assertSame('delete', $eventFrames[0]['data']['payload'][0]['action']);
$this->assertSame('d1', $eventFrames[0]['data']['payload'][0]['resourceId']);
}
public function testAddingChannelPreservesExistingTail(): void
{
// Review scenario: subscribe id 'x' to tail A, later add tail B under the same id.
// Registration is additive (mirrors Realtime::subscribe()), so A must keep delivering.
$registry = new EventTailRegistry(rate: 100);
$registry->add(1, 'x', 'projA', $this->compile([]), 0.0);
// ... later, the same subscription id also tails projB.
$registry->add(1, 'x', 'projB', $this->compile([]), 0.0);
$this->assertTrue($registry->isTailed('projA'), 'projA must remain tailed after adding projB');
$this->assertTrue($registry->isTailed('projB'));
$registry->ingest('projA', ['type' => 'databases', 'resourceId' => 'a1'], 0.0);
$registry->ingest('projB', ['type' => 'databases', 'resourceId' => 'b1'], 0.0);
$sent = [];
$registry->flush($this->serverWithConn($sent));
$channels = array_map(fn ($s) => $s['frame']['data']['channels'][0], $sent);
$this->assertContains('console.tail.projA', $channels, 'projA stopped delivering after projB was added');
$this->assertContains('console.tail.projB', $channels);
}
}