From 1dbddc848c0bd27fd7566a2e55bed8b54c652246 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sat, 6 Jun 2026 17:04:07 +0530 Subject: [PATCH 01/15] added EventTailRegistry --- app/init/constants.php | 4 + src/Appwrite/Realtime/EventTailRegistry.php | 139 ++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 src/Appwrite/Realtime/EventTailRegistry.php diff --git a/app/init/constants.php b/app/init/constants.php index 4fe7c418a9..2f89771b70 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -276,6 +276,10 @@ const API_KEY_STANDARD = 'standard'; const API_KEY_EPHEMERAL = 'ephemeral'; const API_KEY_ORGANIZATION = 'organization'; const API_KEY_ACCOUNT = 'account'; + +// Realtime +const CONSOLE_TAIL_CHANNEL_PREFIX = 'console.tail'; + // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; diff --git a/src/Appwrite/Realtime/EventTailRegistry.php b/src/Appwrite/Realtime/EventTailRegistry.php new file mode 100644 index 0000000000..c721326db7 --- /dev/null +++ b/src/Appwrite/Realtime/EventTailRegistry.php @@ -0,0 +1,139 @@ + + * }> + * keyed by subscriptionId */ + private array $subs; + + /** @var array> projectId => set of subIds (hot-path lookup) */ + private array $projectSubscriptionsMap = []; + + /** @var array> connId => set of subIds (cleanup on close) */ + private array $byConnection = []; + + /** @var array projectId => teamId (resolved at subscribe) */ + private array $teamCache = []; + + /** + * @param $rate tokens per second per subscriptions + * @param $batchMax max frame per flush + */ + public function __construct(private int $rate=50, private int $batchMax=200) {} + public function add(int $connId, string $subId, string $projectId, array $filters, float $now): void + { + $this->subs[$subId] = [ + 'connId' => $connId, + 'projectId' => $projectId, + 'filters' => $filters, // RuntimeQuery::compile(...) result + 'tokens' => (float) $this->rate, + 'lastRefill' => $now, + 'dropped' => 0, + 'delivered' => 0, + 'buffer' => [], + ]; + $this->projectSubscriptionsMap[$projectId][$subId] = true; + $this->byConnection[$connId][$subId] = true; + } + + public function remove(string $subId){ + $sub = $this->subs[$subId] ?? null; + if ($sub === null) { + return; + } + unset($this->projectSubscriptionsMap[$sub['projectId']][$subId]); + if (empty($this->projectSubscriptionsMap[$sub['projectId']])) { + unset($this->projectSubscriptionsMap[$sub['projectId']]); + } + unset($this->byConnection[$sub['connId']][$subId]); + if (empty($this->byConnection[$sub['connId']])) { + unset($this->byConnection[$sub['connId']]); + } + unset($this->subs[$subId]); + } + + public function removeConnection(int $connId){ + foreach (\array_keys($this->byConnection[$connId] ?? []) as $subId) { + $this->remove($subId); + } + } + + public function ingest(string $projectId, array $payload, float $now){ + // filter + sampler + foreach(array_keys($this->projectSubscriptionsMap[$projectId] ?? []) as $subId){ + $sub = &$this->subs[$subId]; + + if(RuntimeQuery::filter($sub['filters'], $payload) === null) continue; + + $elapsed = \max(0.0, $now - $sub['lastRefill']); + // tokens will be always a fractional value lazily increasing with the rate + $sub['tokens'] = \min((float) $this->rate, $sub['tokens'] + $elapsed*$this->rate); + + if ($sub['tokens'] >= 1.0) { + $sub['tokens'] -= 1.0; + $sub['buffer'][] = $payload; + $sub['delivered']++; + } else { + $sub['dropped']++; + } + + unset($sub); + } + } + public function flush(Server $server){ + foreach ($this->subs as $subId => &$sub) { + $channel = CONSOLE_TAIL_CHANNEL_PREFIX . '.' . $sub['projectId']; + + if (!empty($sub['buffer'])) { + // TODO: if count($sub['buffer']) or encoded size could exceed batchMax / + // setPackageMaxLength(64000), split into chunks and send multiple frames. + foreach (\array_chunk($sub['buffer'], $this->batchMax) as $chunk) { + $server->send([$sub['connId']], (string) \json_encode([ + 'type' => 'event', + 'data' => [ + 'channels' => [$channel], + 'events' => ['console.tail'], + 'subscriptions' => [$subId], + 'payload' => $chunk, + ], + ])); + } + $sub['buffer'] = []; + } + + // counter frame (rate + dropped) — emit when something was dropped + if ($sub['dropped'] > 0) { + $server->send([$sub['connId']], (string) \json_encode([ + 'type' => 'event', + 'data' => [ + 'channels' => [$channel], + 'events' => ['console.tail.stats'], + 'subscriptions' => [$subId], + 'payload' => [ + '$type' => 'tail.stats', + 'delivered' => $sub['delivered'], + 'dropped' => $sub['dropped'], + ], + ], + ])); + } + // TODO: decide reset cadence. Simplest: reset counters every flush. + $sub['delivered'] = 0; + $sub['dropped'] = 0; + } + unset($sub); + } +} \ No newline at end of file From d207fa74fa39e88d7869237c99a96052cf917759 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sat, 6 Jun 2026 18:19:40 +0530 Subject: [PATCH 02/15] updated env --- .env | 2 ++ docker-compose.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/.env b/.env index 57deb8cc0a..9c7e6923b8 100644 --- a/.env +++ b/.env @@ -153,3 +153,5 @@ _TESTS_OAUTH2_GITHUB_CLIENT_SECRET= _APP_WORKERS_NUM=1 _APP_WORKER_MAX_COROUTINES=1 + +_APP_REALTIME_TAIL_RATE=50 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 2907ab864f..20553a4db9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -310,6 +310,7 @@ services: - _APP_WORKER_PER_CORE - _APP_OPTIONS_ABUSE - _APP_OPTIONS_ROUTER_PROTECTION + - _APP_REALTIME_TAIL_RATE - _APP_OPENSSL_KEY_V1 - _APP_REDIS_HOST - _APP_REDIS_PORT From 7a3580b5496cdd7de8cd1ada33ffe3ec0040f27b Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sat, 6 Jun 2026 18:20:10 +0530 Subject: [PATCH 03/15] refactor: improve EventTailRegistry class structure and documentation --- src/Appwrite/Realtime/EventTailRegistry.php | 120 ++++++++++++++------ 1 file changed, 86 insertions(+), 34 deletions(-) diff --git a/src/Appwrite/Realtime/EventTailRegistry.php b/src/Appwrite/Realtime/EventTailRegistry.php index c721326db7..a108de1d50 100644 --- a/src/Appwrite/Realtime/EventTailRegistry.php +++ b/src/Appwrite/Realtime/EventTailRegistry.php @@ -1,23 +1,24 @@ - * }> - * keyed by subscriptionId */ - private array $subs; +// token bucket + buffers per subId of a project per worker +class EventTailRegistry +{ + /** @var array, + * tokens:float, + * lastRefill:float, + * dropped:int, + * delivered:int, + * buffer:array> + * }> keyed by subscriptionId */ + private array $subs = []; /** @var array> projectId => set of subIds (hot-path lookup) */ private array $projectSubscriptionsMap = []; @@ -25,20 +26,56 @@ class EventTailRegistry{ /** @var array> connId => set of subIds (cleanup on close) */ private array $byConnection = []; - /** @var array projectId => teamId (resolved at subscribe) */ + /** @var array projectId => teamId (resolved once per worker at subscribe) */ private array $teamCache = []; /** - * @param $rate tokens per second per subscriptions - * @param $batchMax max frame per flush + * @param int $rate tokens per second per subscription + * @param int $batchMax max frames per flushed websocket frame (kept under setPackageMaxLength) + */ + public function __construct(private int $rate = 50, private int $batchMax = 200) + { + } + + /** + * Build the tail channel string for a project. + */ + public static function channel(string $projectId): string + { + return CONSOLE_TAIL_CHANNEL_PREFIX . '.' . $projectId; + } + + public static function projectFromChannel(string $channel): ?string + { + $prefix = CONSOLE_TAIL_CHANNEL_PREFIX . '.'; + if (!\str_starts_with($channel, $prefix)) { + return null; + } + $projectId = \substr($channel, \strlen($prefix)); + return $projectId === '' ? null : $projectId; + } + + public function cachedTeam(string $projectId): ?string + { + return $this->teamCache[$projectId] ?? null; + } + + public function cacheTeam(string $projectId, string $teamId): void + { + $this->teamCache[$projectId] = $teamId; + } + + /** + * Register a tail subscription owned by this worker. + * + * @param array $filters result of RuntimeQuery::compile() */ - public function __construct(private int $rate=50, private int $batchMax=200) {} public function add(int $connId, string $subId, string $projectId, array $filters, float $now): void { $this->subs[$subId] = [ 'connId' => $connId, 'projectId' => $projectId, - 'filters' => $filters, // RuntimeQuery::compile(...) result + 'filters' => $filters, 'tokens' => (float) $this->rate, 'lastRefill' => $now, 'dropped' => 0, @@ -49,38 +86,51 @@ class EventTailRegistry{ $this->byConnection[$connId][$subId] = true; } - public function remove(string $subId){ + public function remove(string $subId): void + { $sub = $this->subs[$subId] ?? null; if ($sub === null) { return; } + unset($this->projectSubscriptionsMap[$sub['projectId']][$subId]); if (empty($this->projectSubscriptionsMap[$sub['projectId']])) { unset($this->projectSubscriptionsMap[$sub['projectId']]); } + unset($this->byConnection[$sub['connId']][$subId]); if (empty($this->byConnection[$sub['connId']])) { unset($this->byConnection[$sub['connId']]); } + unset($this->subs[$subId]); } - public function removeConnection(int $connId){ + public function removeConnection(int $connId): void + { foreach (\array_keys($this->byConnection[$connId] ?? []) as $subId) { $this->remove($subId); } } - public function ingest(string $projectId, array $payload, float $now){ - // filter + sampler - foreach(array_keys($this->projectSubscriptionsMap[$projectId] ?? []) as $subId){ + public function isTailed(string $projectId): bool + { + return isset($this->projectSubscriptionsMap[$projectId]); + } + + public function ingest(string $projectId, array $payload, float $now): void + { + foreach (\array_keys($this->projectSubscriptionsMap[$projectId] ?? []) as $subId) { $sub = &$this->subs[$subId]; - if(RuntimeQuery::filter($sub['filters'], $payload) === null) continue; + if (RuntimeQuery::filter($sub['filters'], $payload) === null) { + continue; + } - $elapsed = \max(0.0, $now - $sub['lastRefill']); // tokens will be always a fractional value lazily increasing with the rate - $sub['tokens'] = \min((float) $this->rate, $sub['tokens'] + $elapsed*$this->rate); + $elapsed = \max(0.0, $now - $sub['lastRefill']); + $sub['tokens'] = \min((float) $this->rate, $sub['tokens'] + $elapsed * $this->rate); + $sub['lastRefill'] = $now; if ($sub['tokens'] >= 1.0) { $sub['tokens'] -= 1.0; @@ -93,13 +143,14 @@ class EventTailRegistry{ unset($sub); } } - public function flush(Server $server){ + + public function flush(Server $server): void + { foreach ($this->subs as $subId => &$sub) { - $channel = CONSOLE_TAIL_CHANNEL_PREFIX . '.' . $sub['projectId']; + $channel = self::channel($sub['projectId']); if (!empty($sub['buffer'])) { - // TODO: if count($sub['buffer']) or encoded size could exceed batchMax / - // setPackageMaxLength(64000), split into chunks and send multiple frames. + // Chunk so a single frame stays under the websocket package max length. foreach (\array_chunk($sub['buffer'], $this->batchMax) as $chunk) { $server->send([$sub['connId']], (string) \json_encode([ 'type' => 'event', @@ -114,7 +165,7 @@ class EventTailRegistry{ $sub['buffer'] = []; } - // counter frame (rate + dropped) — emit when something was dropped + // Counter frame — emit the true rate vs dropped count when sampling kicked in. if ($sub['dropped'] > 0) { $server->send([$sub['connId']], (string) \json_encode([ 'type' => 'event', @@ -130,10 +181,11 @@ class EventTailRegistry{ ], ])); } - // TODO: decide reset cadence. Simplest: reset counters every flush. + + // Reset counters every flush window (~150ms). $sub['delivered'] = 0; $sub['dropped'] = 0; } unset($sub); } -} \ No newline at end of file +} From 5999c3971c80dcb09bac8432c47c95a585c5cbd8 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sat, 6 Jun 2026 18:20:27 +0530 Subject: [PATCH 04/15] feat: integrate EventTailRegistry for enhanced realtime event handling --- app/realtime.php | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index c99e4c0857..3203a66430 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -9,6 +9,7 @@ use Appwrite\Messaging\Adapter\Realtime; use Appwrite\Network\Validator\Origin; use Appwrite\Presences\State as PresenceState; use Appwrite\PubSub\Adapter\Pool as PubSubPool; +use Appwrite\Realtime\EventTailRegistry; use Appwrite\Realtime\Message\Dispatcher as MessageDispatcher; use Appwrite\Realtime\Message\Handlers\Authentication as AuthenticationHandler; use Appwrite\Realtime\Message\Handlers\Ping as PingHandler; @@ -336,6 +337,9 @@ if (!function_exists('checkForProjectUsage')) { $realtime = getRealtime(); $presenceState = new PresenceState(); +$eventTailRegistry = new EventTailRegistry( + rate: (int) System::getEnv('_APP_REALTIME_TAIL_RATE', 50) +); $messageDispatcher = (new MessageDispatcher()) ->addHandler(new PingHandler()) @@ -496,7 +500,7 @@ $server->onStart(function () use ($stats, $containerId, &$statsDocument) { } }); -$server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $realtime) { +$server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $realtime, $eventTailRegistry) { Console::success('Worker ' . $workerId . ' started successfully'); $telemetry = getTelemetry($workerId); @@ -525,6 +529,13 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $attempts = 0; $start = time(); + // Flush coalesced console live-tail buffers to their connections (~150ms). + // Must run per-worker: the registry is mutated by this worker's subscribe handler + // and pub/sub ingest, which are not visible to the master process. + Timer::tick(150, function () use ($server, $eventTailRegistry) { + $eventTailRegistry->flush($server); + }); + Timer::tick(5000, function () use ($server, $realtime, $stats) { /** * Sending current connections to project channels on the console project every 5 seconds. @@ -634,7 +645,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, Console::error('Pub/sub failed (worker: ' . $workerId . ')'); } - $pubsub->subscribe(['realtime'], function (mixed $redis, string $channel, string $payload) use ($server, $workerId, $stats, $register, $realtime) { + $pubsub->subscribe(['realtime'], function (mixed $redis, string $channel, string $payload) use ($server, $workerId, $stats, $register, $realtime, $eventTailRegistry) { $event = json_decode($payload, true); $eventTimestamp = $event['data']['timestamp'] ?? null; @@ -790,6 +801,16 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, triggerStats($metrics, $projectId); } + + } + + // Console live event tail: runs for EVERY firehose event, regardless of + // whether it had regular subscribers ($total). The firehose is published + // unconditionally, so a project tailed from the console with no other + // realtime clients must still be observed here. + $src = $event['project'] ?? ''; + if ($src !== '' && $src !== 'console' && $eventTailRegistry->isTailed($src)) { + $eventTailRegistry->ingest($src, Realtime::toTailMetadata($event), \microtime(true)); } }); } catch (Throwable $th) { @@ -1071,7 +1092,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, } }); -$server->onMessage(function (int $connection, string $message) use ($container, $server, $realtime, $containerId, $register, $presenceState, $messageDispatcher) { +$server->onMessage(function (int $connection, string $message) use ($container, $server, $realtime, $containerId, $register, $presenceState, $messageDispatcher, $eventTailRegistry) { $project = null; $authorization = null; $projectId = $realtime->connections[$connection]['projectId'] ?? null; @@ -1191,6 +1212,8 @@ $server->onMessage(function (int $connection, string $message) use ($container, $messageContainer->set('projectId', fn () => $projectId); $messageContainer->set('queueForEvents', fn () => getQueueForEvents()); $messageContainer->set('queueForRealtime', fn () => getQueueForRealtime()); + $messageContainer->set('eventTailRegistry', fn () => $eventTailRegistry); + $messageContainer->set('dbForConsole', fn () => getConsoleDB()); $responsePayload = $messageDispatcher->dispatch($messageContainer, $message); @@ -1263,7 +1286,7 @@ $server->onMessage(function (int $connection, string $message) use ($container, } }); -$server->onClose(function (int $connection) use ($realtime, $stats, $register, $container, $presenceState) { +$server->onClose(function (int $connection) use ($realtime, $stats, $register, $container, $presenceState, $eventTailRegistry) { $projectId = null; $userId = null; $subscriptionsBeforeClose = 0; @@ -1401,6 +1424,13 @@ $server->onClose(function (int $connection) use ($realtime, $stats, $register, $ Span::error($th); } + try { + $eventTailRegistry->removeConnection($connection); + } catch (\Throwable $th) { + Console::error('Realtime onClose tail cleanup error: ' . $th->getMessage()); + Span::error($th); + } + Span::add('realtime.success', $success); if (!empty($projectId)) { Span::add('project.id', $projectId); From 5802643e11d55fd75c25d9a023c57ccc3c0fe4c4 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sat, 6 Jun 2026 18:20:41 +0530 Subject: [PATCH 05/15] feat: add toTailMetadata method for enhanced event metadata extraction --- src/Appwrite/Messaging/Adapter/Realtime.php | 55 +++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/Appwrite/Messaging/Adapter/Realtime.php b/src/Appwrite/Messaging/Adapter/Realtime.php index dbb2c826bd..f7602f8c9d 100644 --- a/src/Appwrite/Messaging/Adapter/Realtime.php +++ b/src/Appwrite/Messaging/Adapter/Realtime.php @@ -500,6 +500,61 @@ class Realtime extends MessagingAdapter return $receivers; } + /** + * + * @param array $event decoded event published to the `realtime` channel + * @return array + */ + public static function toTailMetadata(array $event): array + { + $events = $event['data']['events'] ?? []; + $name = \is_array($events) ? ($events[0] ?? '') : ''; + $parts = $name === '' ? [] : \explode('.', $name); + $count = \count($parts); + $type = $parts[0] ?? null; + + $action = null; + if ($count >= 1 && \in_array($parts[$count - 1], self::SUPPORTED_ACTIONS, true)) { + $action = $parts[$count - 1]; + } elseif ($count >= 2 && \in_array($parts[$count - 2], self::SUPPORTED_ACTIONS, true)) { + $action = $parts[$count - 2]; + } + + $payload = $event['data']['payload'] ?? []; + if (!\is_array($payload)) { + $payload = []; + } + + $metadata = [ + 'event' => $name, + 'type' => $type, + 'action' => $action, + 'userId' => $event['userId'] ?? null, + 'timestamp' => $event['data']['timestamp'] ?? null, + 'resourceId' => $payload['$id'] ?? null, + ]; + + // Attach only the scope ids that belong to this resource type. + $scope = match ($type) { + 'databases' => [ + 'databaseId' => $payload['$databaseId'] ?? null, + 'collectionId' => $payload['$collectionId'] ?? $payload['$tableId'] ?? null, + ], + 'buckets' => ['bucketId' => $payload['bucketId'] ?? null], + 'functions' => ['functionId' => $payload['functionId'] ?? null], + 'teams' => ['teamId' => $payload['teamId'] ?? null], + default => [], + }; + + foreach ($scope as $key => $value) { + if ($value !== null) { + $metadata[$key] = $value; + } + } + + return $metadata; + } + /** * Converts the channels from the Query Params into an array. * Also renames the account channel to account.USER_ID, rewrites action-suffixed From 54f04abd28373d0cd7b841825dd3a4efb0584706 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sat, 6 Jun 2026 18:20:56 +0530 Subject: [PATCH 06/15] feat: enhance Subscribe handler with event tail authorization and caching --- .../Realtime/Message/Handlers/Subscribe.php | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/Appwrite/Realtime/Message/Handlers/Subscribe.php b/src/Appwrite/Realtime/Message/Handlers/Subscribe.php index 140cd800c3..537dba8cdd 100644 --- a/src/Appwrite/Realtime/Message/Handlers/Subscribe.php +++ b/src/Appwrite/Realtime/Message/Handlers/Subscribe.php @@ -4,8 +4,11 @@ namespace Appwrite\Realtime\Message\Handlers; use Appwrite\Extend\Exception; use Appwrite\Messaging\Adapter\Realtime; +use Appwrite\Realtime\EventTailRegistry; use Appwrite\Realtime\Message\Dispatcher; use Appwrite\Realtime\Message\Validators\SubscribePayload as SubscribePayloadValidator; +use Appwrite\Utopia\Database\RuntimeQuery; +use Utopia\Database\Database; use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Role; @@ -26,6 +29,8 @@ class Subscribe extends Action ->inject('realtime') ->inject('register') ->inject('projectId') + ->inject('eventTailRegistry') + ->inject('dbForConsole') ->callback($this->action(...)); } @@ -39,6 +44,8 @@ class Subscribe extends Action Realtime $realtime, Registry $register, ?string $projectId, + EventTailRegistry $eventTailRegistry, + Database $dbForConsole, ): array { $roles = $realtime->connections[$connectionId]['roles'] ?? [Role::guests()->toString()]; $userId = $realtime->connections[$connectionId]['userId'] ?? ''; @@ -80,6 +87,43 @@ class Subscribe extends Action ); } + // Console live event tail: any `console.tail.` channel is authorized by + // team membership of the target project's owning team, then registered with its + // compiled filter so the worker's pub/sub loop can filter+sample before delivery. + $now = \microtime(true); + foreach ($parsedPayloads as $parsedPayload) { + foreach ($parsedPayload['convertedChannels'] as $channel) { + $targetProjectId = EventTailRegistry::projectFromChannel($channel); + if ($targetProjectId === null) { + continue; + } + + $teamId = $eventTailRegistry->cachedTeam($targetProjectId); + if ($teamId === null) { + $project = $dbForConsole->getAuthorization()->skip( + fn () => $dbForConsole->getDocument('projects', $targetProjectId) + ); + $teamId = (string) $project->getAttribute('teamId', ''); + $eventTailRegistry->cacheTeam($targetProjectId, $teamId); + } + + if ($teamId === '' || !\in_array(Role::team($teamId)->toString(), $roles, true)) { + throw new Exception( + Exception::REALTIME_POLICY_VIOLATION, + 'Not authorized to tail project "' . $targetProjectId . '".' + ); + } + + $eventTailRegistry->add( + $connectionId, + $parsedPayload['subscriptionId'], + $targetProjectId, + RuntimeQuery::compile($parsedPayload['queries']), + $now, + ); + } + } + $subscriptionsAfter = \count($realtime->getSubscriptionMetadata($connectionId)); $subscriptionDelta = $subscriptionsAfter - $subscriptionsBefore; $subscriptionsRequested = \count($parsedPayloads); From 5c45ec91b2b842b8ecc5a485cf701ed53f12a570 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sat, 6 Jun 2026 18:21:04 +0530 Subject: [PATCH 07/15] feat: integrate EventTailRegistry into Unsubscribe handler for tail subscription management --- src/Appwrite/Realtime/Message/Handlers/Unsubscribe.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Appwrite/Realtime/Message/Handlers/Unsubscribe.php b/src/Appwrite/Realtime/Message/Handlers/Unsubscribe.php index e1bc163f68..cba8aa4a3f 100644 --- a/src/Appwrite/Realtime/Message/Handlers/Unsubscribe.php +++ b/src/Appwrite/Realtime/Message/Handlers/Unsubscribe.php @@ -3,6 +3,7 @@ namespace Appwrite\Realtime\Message\Handlers; use Appwrite\Messaging\Adapter\Realtime; +use Appwrite\Realtime\EventTailRegistry; use Appwrite\Realtime\Message\Dispatcher; use Appwrite\Realtime\Message\Validators\UnsubscribePayload as UnsubscribePayloadValidator; use Utopia\Platform\Action; @@ -21,6 +22,7 @@ class Unsubscribe extends Action ->inject('connectionId') ->inject('realtime') ->inject('register') + ->inject('eventTailRegistry') ->callback($this->action(...)); } @@ -33,6 +35,7 @@ class Unsubscribe extends Action int $connectionId, Realtime $realtime, Registry $register, + EventTailRegistry $eventTailRegistry, ): array { $subscriptionsBefore = \count($realtime->getSubscriptionMetadata($connectionId)); @@ -43,6 +46,8 @@ class Unsubscribe extends Action 'subscriptionId' => $subscriptionId, 'removed' => $realtime->unsubscribeSubscription($connectionId, $subscriptionId), ]; + // No-op if it wasn't a tail subscription. + $eventTailRegistry->remove($subscriptionId); } $subscriptionsAfter = \count($realtime->getSubscriptionMetadata($connectionId)); From 61227fddc4e20d95756b2707bd1c93e99ae94bc7 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sat, 6 Jun 2026 18:21:11 +0530 Subject: [PATCH 08/15] feat: add unit tests for EventTailRegistry and console live event tail functionality --- ...altimeCustomClientQueryTestWithMessage.php | 227 +++++++++++ tests/unit/Realtime/EventTailRegistryTest.php | 366 ++++++++++++++++++ 2 files changed, 593 insertions(+) create mode 100644 tests/unit/Realtime/EventTailRegistryTest.php diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php b/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php index 13a7d549dc..3b1f8c6e54 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php @@ -13,6 +13,7 @@ use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use WebSocket\Client as WebSocketClient; +use WebSocket\ConnectionException; use WebSocket\TimeoutException; final class RealtimeCustomClientQueryTestWithMessage extends Scope @@ -810,4 +811,230 @@ final class RealtimeCustomClientQueryTestWithMessage extends Scope $client->close(); } + + /** + * Create a database + collection (Role::any) with a string attribute, BEFORE the + * tail connection opens, so the setup events don't appear in the tail. + * + * @return array{databaseId:string, collectionId:string} + */ + private function setupTailCollection(): array + { + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]; + + $database = $this->client->call(Client::METHOD_POST, '/databases', $headers, [ + 'databaseId' => ID::unique(), + 'name' => 'Tail DB', + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', $headers, [ + 'collectionId' => ID::unique(), + 'name' => 'Actors', + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', $headers, [ + 'key' => 'name', + 'size' => 256, + 'required' => true, + ]); + + $this->assertEventually(function () use ($databaseId, $collectionId, $headers) { + $attribute = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/name', $headers); + $this->assertEquals(200, $attribute['headers']['status-code']); + $this->assertEquals('available', $attribute['body']['status']); + }, 120000, 500); + + return ['databaseId' => $databaseId, 'collectionId' => $collectionId]; + } + + /** + * Open a console connection (root session) in message mode and subscribe to a + * `console.tail.` channel with optional filter queries. + * + * @param array $queries + */ + private function openConsoleTail(string $targetProjectId, array $queries = [], string $subscriptionId = 'tail-1'): WebSocketClient + { + $queryString = \http_build_query(['project' => 'console']); + $client = new WebSocketClient( + 'ws://appwrite.test/v1/realtime?' . $queryString, + [ + 'headers' => [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + ], + 'timeout' => 10, + ] + ); + + $connected = \json_decode($client->receive(), true); + $this->assertEquals('connected', $connected['type'] ?? null); + + $sub = ['channels' => ['console.tail.' . $targetProjectId], 'subscriptionId' => $subscriptionId]; + if (!empty($queries)) { + $sub['queries'] = $queries; + } + + $client->send(\json_encode(['type' => 'subscribe', 'data' => [$sub]])); + + $response = \json_decode($client->receive(), true); + $this->assertEquals('response', $response['type'] ?? null); + $this->assertEquals('subscribe', $response['data']['to'] ?? null); + $this->assertTrue($response['data']['success'] ?? false); + + return $client; + } + + /** + * Receive frames until a `console.tail` event arrives, returning its payload (an + * array of compact frames). Skips the periodic `console.tail.stats` counter frames. + * + * @return array> + */ + private function receiveTailFrames(WebSocketClient $client, int $maxFrames = 5): array + { + for ($i = 0; $i < $maxFrames; $i++) { + $message = \json_decode($client->receive(), true); + $this->assertEquals('event', $message['type']); + if (($message['data']['events'][0] ?? null) === 'console.tail') { + return $message['data']['payload']; + } + } + $this->fail('No console.tail event frame received.'); + } + + public function testConsoleTailReceivesCompactMetadata(): void + { + ['databaseId' => $databaseId, 'collectionId' => $collectionId] = $this->setupTailCollection(); + + $projectId = $this->getProject()['$id']; + $client = $this->openConsoleTail($projectId); + + $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'documentId' => ID::unique(), + 'data' => ['name' => 'Spiderman'], + ]); + $this->assertEquals(201, $document['headers']['status-code']); + $documentId = $document['body']['$id']; + + $frames = $this->receiveTailFrames($client); + + $match = null; + foreach ($frames as $frame) { + if (($frame['resourceId'] ?? null) === $documentId) { + $match = $frame; + break; + } + } + + $this->assertNotNull($match, 'document.create not present in tail: ' . \json_encode($frames)); + $this->assertSame('databases', $match['type']); + $this->assertSame('create', $match['action']); + $this->assertSame($databaseId, $match['databaseId']); + $this->assertSame($collectionId, $match['collectionId']); + // Compact: no full document body, and no scope ids from other resource types. + $this->assertArrayNotHasKey('name', $match); + $this->assertArrayNotHasKey('functionId', $match); + $this->assertArrayNotHasKey('bucketId', $match); + + $client->close(); + } + + public function testConsoleTailServerSideFilter(): void + { + ['databaseId' => $databaseId, 'collectionId' => $collectionId] = $this->setupTailCollection(); + + $projectId = $this->getProject()['$id']; + // Only deletes should reach this tail. + $client = $this->openConsoleTail($projectId, [Query::equal('action', ['delete'])->toString()]); + + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]; + + // create — must be filtered out (never reaches the token bucket). + $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers, [ + 'documentId' => ID::unique(), + 'data' => ['name' => 'Ironman'], + ]); + $this->assertEquals(201, $document['headers']['status-code']); + $documentId = $document['body']['$id']; + + // delete — must pass the filter. + $deleted = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $documentId, $headers); + $this->assertEquals(204, $deleted['headers']['status-code']); + + $frames = $this->receiveTailFrames($client); + + foreach ($frames as $frame) { + $this->assertSame('delete', $frame['action'], 'filter leaked a non-delete event: ' . \json_encode($frame)); + } + $deleteFrames = \array_filter($frames, fn ($f) => ($f['resourceId'] ?? null) === $documentId); + $this->assertNotEmpty($deleteFrames, 'expected the delete event in the tail'); + + $client->close(); + } + + public function testConsoleTailUnauthorizedProjectIsRejected(): void + { + $queryString = \http_build_query(['project' => 'console']); + $client = new WebSocketClient( + 'ws://appwrite.test/v1/realtime?' . $queryString, + [ + 'headers' => [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + ], + 'timeout' => 10, + ] + ); + + $connected = \json_decode($client->receive(), true); + $this->assertEquals('connected', $connected['type'] ?? null); + + // A project the console user's team does not own (does not exist) → policy violation. + $client->send(\json_encode([ + 'type' => 'subscribe', + 'data' => [['channels' => ['console.tail.nonexistentproject'], 'subscriptionId' => 'tail-x']], + ])); + + // A 1008 policy violation sends the error frame and then closes the connection. + // Depending on flush/close timing the client may read the error frame, or the + // socket may already be dead — both are valid rejections. What must NOT happen + // is a successful `response` subscription. + try { + $response = \json_decode($client->receive(), true); + $this->assertEquals('error', $response['type'] ?? null); + $this->assertEquals(1008, $response['data']['code'] ?? null); + } catch (ConnectionException $e) { + $this->addToAssertionCount(1); + } + + // close() can itself throw on an already-closed socket — best effort. + try { + $client->close(); + } catch (ConnectionException $e) { + // already closed by the server + } + } } diff --git a/tests/unit/Realtime/EventTailRegistryTest.php b/tests/unit/Realtime/EventTailRegistryTest.php new file mode 100644 index 0000000000..3f6160142d --- /dev/null +++ b/tests/unit/Realtime/EventTailRegistryTest.php @@ -0,0 +1,366 @@ +> $sent + */ + private function server(array &$sent): Server + { + // A stub (not a mock): we only capture sends, no expectations to verify. + $server = $this->createStub(Server::class); + $server->method('send')->willReturnCallback( + function (array $connections, string $message) use (&$sent): void { + $sent[] = json_decode($message, true); + } + ); + return $server; + } + + /** @param array $payload */ + private function compile(array $queries): array + { + return RuntimeQuery::compile($queries); + } + + // ---- Realtime::toTailMetadata ---- + + public function testTailMetadataDocumentEvent(): void + { + $event = [ + 'userId' => 'user-1', + 'data' => [ + 'events' => ['databases.main.collections.posts.documents.p1.create'], + 'timestamp' => '2026-06-06T00:00:00.000+00:00', + 'payload' => ['$id' => 'p1', '$databaseId' => 'main', '$collectionId' => 'posts', 'title' => 'hello'], + ], + ]; + + $meta = Realtime::toTailMetadata($event); + + $this->assertSame('databases.main.collections.posts.documents.p1.create', $meta['event']); + $this->assertSame('databases', $meta['type']); + $this->assertSame('create', $meta['action']); + $this->assertSame('user-1', $meta['userId']); + $this->assertSame('p1', $meta['resourceId']); + $this->assertSame('main', $meta['databaseId']); + $this->assertSame('posts', $meta['collectionId']); + // No full document body leaks through. + $this->assertArrayNotHasKey('title', $meta); + // Scope is type-specific: a databases event carries no bucket/function ids. + $this->assertArrayNotHasKey('bucketId', $meta); + $this->assertArrayNotHasKey('functionId', $meta); + } + + public function testTailMetadataScopeIsTypeSpecific(): void + { + $fn = Realtime::toTailMetadata([ + 'userId' => 'u1', + 'data' => [ + 'events' => ['functions.fn1.deployments.d1.create'], + 'payload' => ['$id' => 'd1', 'functionId' => 'fn1'], + ], + ]); + $this->assertSame('functions', $fn['type']); + $this->assertSame('fn1', $fn['functionId']); + $this->assertArrayNotHasKey('databaseId', $fn); + $this->assertArrayNotHasKey('bucketId', $fn); + + $file = Realtime::toTailMetadata([ + 'userId' => 'u1', + 'data' => [ + 'events' => ['buckets.b1.files.f1.create'], + 'payload' => ['$id' => 'f1', 'bucketId' => 'b1'], + ], + ]); + $this->assertSame('buckets', $file['type']); + $this->assertSame('b1', $file['bucketId']); + $this->assertArrayNotHasKey('functionId', $file); + $this->assertArrayNotHasKey('collectionId', $file); + + // A null scope id is omitted rather than emitted. + $fnNoScope = Realtime::toTailMetadata([ + 'data' => ['events' => ['functions.fn1.executions.e1.create'], 'payload' => ['$id' => 'e1']], + ]); + $this->assertArrayNotHasKey('functionId', $fnNoScope); + } + + public function testTailMetadataTablesUsesTableIdAsCollection(): void + { + $meta = Realtime::toTailMetadata([ + 'userId' => 'u1', + 'data' => [ + 'events' => ['databases.main.tables.t1.rows.r1.create'], + 'payload' => ['$id' => 'r1', '$databaseId' => 'main', '$tableId' => 't1'], + ], + ]); + $this->assertSame('databases', $meta['type']); + $this->assertSame('main', $meta['databaseId']); + $this->assertSame('t1', $meta['collectionId']); + } + + public function testTailMetadataAttributeTrailingAction(): void + { + // `users.U.update.email` — action is the second-to-last segment. + $meta = Realtime::toTailMetadata([ + 'userId' => 'u9', + 'data' => ['events' => ['users.u9.update.email'], 'payload' => ['$id' => 'u9']], + ]); + + $this->assertSame('users', $meta['type']); + $this->assertSame('update', $meta['action']); + $this->assertSame('u9', $meta['resourceId']); + } + + public function testTailMetadataHandlesMissingPayload(): void + { + $meta = Realtime::toTailMetadata(['data' => ['events' => ['functions.f1.executions.e1.delete']]]); + + $this->assertSame('functions', $meta['type']); + $this->assertSame('delete', $meta['action']); + $this->assertNull($meta['userId']); + $this->assertNull($meta['resourceId']); + } + + // ---- channel helpers ---- + + public function testChannelHelpers(): void + { + $this->assertSame('console.tail.proj1', EventTailRegistry::channel('proj1')); + $this->assertSame('proj1', EventTailRegistry::projectFromChannel('console.tail.proj1')); + $this->assertNull(EventTailRegistry::projectFromChannel('documents')); + $this->assertNull(EventTailRegistry::projectFromChannel('console.tail.')); + } + + // ---- filter THEN sample ---- + + public function testFilterIsAppliedBeforeSampling(): void + { + $registry = new EventTailRegistry(rate: 100); + $registry->add(1, 'sub-a', 'projX', $this->compile([Query::equal('type', ['databases'])]), 0.0); + + $registry->ingest('projX', ['type' => 'databases', 'action' => 'create'], 0.0); + $registry->ingest('projX', ['type' => 'buckets', 'action' => 'create'], 0.0); + + $sent = []; + $registry->flush($this->server($sent)); + + // Exactly one event frame, carrying only the databases frame; no stats frame + // because the filtered-out bucket event never consumed a token. + $this->assertCount(1, $sent); + $this->assertSame(['console.tail'], $sent[0]['data']['events']); + $this->assertCount(1, $sent[0]['data']['payload']); + $this->assertSame('databases', $sent[0]['data']['payload'][0]['type']); + } + + public function testTokenBucketDropsExcessAndEmitsCounter(): void + { + $registry = new EventTailRegistry(rate: 5); + $registry->add(1, 'sub-a', 'projX', $this->compile([]), 0.0); // selectAll + + for ($i = 0; $i < 20; $i++) { + $registry->ingest('projX', ['type' => 'databases', 'seq' => $i], 0.0); + } + + $sent = []; + $registry->flush($this->server($sent)); + + $eventFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail'])); + $statsFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail.stats'])); + + $this->assertCount(1, $eventFrames); + $this->assertCount(5, $eventFrames[0]['data']['payload'], 'rate=5 → only 5 delivered at the same instant'); + + $this->assertCount(1, $statsFrames); + $this->assertSame('tail.stats', $statsFrames[0]['data']['payload']['$type']); + $this->assertSame(5, $statsFrames[0]['data']['payload']['delivered']); + $this->assertSame(15, $statsFrames[0]['data']['payload']['dropped']); + // delivered + dropped accounts for every ingested event. + $this->assertSame(20, $statsFrames[0]['data']['payload']['delivered'] + $statsFrames[0]['data']['payload']['dropped']); + } + + public function testNoCounterFrameWhenNothingDropped(): void + { + $registry = new EventTailRegistry(rate: 100); + $registry->add(1, 'sub-a', 'projX', $this->compile([]), 0.0); + + $registry->ingest('projX', ['type' => 'databases'], 0.0); + + $sent = []; + $registry->flush($this->server($sent)); + + // Only the event frame — no stats frame because nothing was dropped. + $this->assertCount(1, $sent); + $this->assertSame(['console.tail'], $sent[0]['data']['events']); + } + + public function testDroppedAccumulatesAcrossIngestsInWindow(): void + { + $registry = new EventTailRegistry(rate: 2); + $registry->add(1, 'sub-a', 'projX', $this->compile([]), 0.0); + + // Three separate ingests at the same instant: 2 delivered, 1 dropped. + $registry->ingest('projX', ['type' => 'databases', 'seq' => 1], 0.0); + $registry->ingest('projX', ['type' => 'databases', 'seq' => 2], 0.0); + $registry->ingest('projX', ['type' => 'databases', 'seq' => 3], 0.0); + + $sent = []; + $registry->flush($this->server($sent)); + + $statsFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail.stats'])); + $this->assertCount(1, $statsFrames); + $this->assertSame(2, $statsFrames[0]['data']['payload']['delivered']); + $this->assertSame(1, $statsFrames[0]['data']['payload']['dropped']); + } + + public function testCounterResetsBetweenWindows(): void + { + $registry = new EventTailRegistry(rate: 1); + $registry->add(1, 'sub-a', 'projX', $this->compile([]), 0.0); + + // Window 1: 3 events at t=0 → 1 delivered, 2 dropped → stats frame emitted. + for ($i = 0; $i < 3; $i++) { + $registry->ingest('projX', ['type' => 'databases', 'seq' => $i], 0.0); + } + $sent = []; + $registry->flush($this->server($sent)); + $statsFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail.stats'])); + $this->assertCount(1, $statsFrames); + $this->assertSame(2, $statsFrames[0]['data']['payload']['dropped']); + + // Window 2: a single event well after refill → delivered, nothing dropped → no stats frame + // and the previous window's dropped count does not carry over. + $registry->ingest('projX', ['type' => 'databases'], 100.0); + $sent = []; + $registry->flush($this->server($sent)); + $statsFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail.stats'])); + $this->assertCount(0, $statsFrames); + } + + public function testLargeBufferIsChunkedAcrossFrames(): void + { + // rate high enough to admit all, batchMax forces splitting on flush. + $registry = new EventTailRegistry(rate: 100, batchMax: 2); + $registry->add(1, 'sub-a', 'projX', $this->compile([]), 0.0); + + for ($i = 0; $i < 5; $i++) { + $registry->ingest('projX', ['type' => 'databases', 'seq' => $i], 0.0); + } + + $sent = []; + $registry->flush($this->server($sent)); + + $eventFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail'])); + // 5 buffered frames, batchMax 2 → 3 websocket frames (2 + 2 + 1). + $this->assertCount(3, $eventFrames); + $this->assertCount(2, $eventFrames[0]['data']['payload']); + $this->assertCount(2, $eventFrames[1]['data']['payload']); + $this->assertCount(1, $eventFrames[2]['data']['payload']); + // No event is lost across the chunk boundaries. + $total = array_sum(array_map(fn ($f) => count($f['data']['payload']), $eventFrames)); + $this->assertSame(5, $total); + } + + public function testTokensRefillOverTime(): void + { + $registry = new EventTailRegistry(rate: 5); + $registry->add(1, 'sub-a', 'projX', $this->compile([]), 0.0); + + // Drain the bucket at t=0. + for ($i = 0; $i < 10; $i++) { + $registry->ingest('projX', ['type' => 'databases'], 0.0); + } + $sent = []; + $registry->flush($this->server($sent)); // resets the per-window counters + + // One second later, the bucket has refilled to the cap (rate=5). + for ($i = 0; $i < 10; $i++) { + $registry->ingest('projX', ['type' => 'databases'], 1.0); + } + $sent = []; + $registry->flush($this->server($sent)); + + $eventFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail'])); + $this->assertCount(5, $eventFrames[0]['data']['payload']); + } + + public function testFlushClearsBufferBetweenWindows(): void + { + $registry = new EventTailRegistry(rate: 100); + $registry->add(1, 'sub-a', 'projX', $this->compile([]), 0.0); + $registry->ingest('projX', ['type' => 'databases'], 0.0); + + $sent = []; + $registry->flush($this->server($sent)); + $this->assertCount(1, $sent); + + // Nothing new ingested → second flush sends nothing. + $sent = []; + $registry->flush($this->server($sent)); + $this->assertCount(0, $sent); + } + + public function testRemoveConnectionStopsDelivery(): void + { + $registry = new EventTailRegistry(rate: 100); + $registry->add(7, 'sub-a', 'projX', $this->compile([]), 0.0); + $registry->add(7, 'sub-b', 'projY', $this->compile([]), 0.0); + + $this->assertTrue($registry->isTailed('projX')); + $this->assertTrue($registry->isTailed('projY')); + + $registry->removeConnection(7); + + $this->assertFalse($registry->isTailed('projX')); + $this->assertFalse($registry->isTailed('projY')); + + $registry->ingest('projX', ['type' => 'databases'], 0.0); + $sent = []; + $registry->flush($this->server($sent)); + $this->assertCount(0, $sent); + } + + public function testRemoveSingleSubscription(): void + { + $registry = new EventTailRegistry(rate: 100); + $registry->add(7, 'sub-a', 'projX', $this->compile([]), 0.0); + $registry->add(7, 'sub-b', 'projX', $this->compile([]), 0.0); + + $registry->remove('sub-a'); + $this->assertTrue($registry->isTailed('projX'), 'sub-b still tails projX'); + + $registry->remove('sub-b'); + $this->assertFalse($registry->isTailed('projX')); + + // Removing an unknown subscription is a no-op. + $registry->remove('does-not-exist'); + $this->assertFalse($registry->isTailed('projX')); + } + + public function testTeamCache(): void + { + $registry = new EventTailRegistry(); + $this->assertNull($registry->cachedTeam('projX')); + $registry->cacheTeam('projX', 'team-1'); + $this->assertSame('team-1', $registry->cachedTeam('projX')); + } +} From 6b94c966b1cadb065df95a9362d367dfaec2e1c9 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sat, 6 Jun 2026 18:38:50 +0530 Subject: [PATCH 09/15] feat: refactor EventTailRegistry for improved subscription management and isolation --- src/Appwrite/Realtime/EventTailRegistry.php | 237 ++++++++++-------- .../Realtime/Message/Handlers/Subscribe.php | 29 ++- .../Realtime/Message/Handlers/Unsubscribe.php | 5 +- tests/unit/Realtime/EventTailRegistryTest.php | 129 +++++++++- 4 files changed, 272 insertions(+), 128 deletions(-) diff --git a/src/Appwrite/Realtime/EventTailRegistry.php b/src/Appwrite/Realtime/EventTailRegistry.php index a108de1d50..b2f0950038 100644 --- a/src/Appwrite/Realtime/EventTailRegistry.php +++ b/src/Appwrite/Realtime/EventTailRegistry.php @@ -5,29 +5,32 @@ namespace Appwrite\Realtime; use Appwrite\Utopia\Database\RuntimeQuery; use Utopia\WebSocket\Server; -// token bucket + buffers per subId of a project per worker +/** + * Per-worker token-bucket + buffers for console event-tail subscriptions. + * + * Each tail of a single project is one entry, keyed by (connId, subId, projectId). + * Subscription IDs are only unique WITHIN a connection, so the connection id is part + * of the key — subscriptions from different connections (or different projects under + * the same subscription id) never collide. A subscription that names several tail + * channels gets one entry per project under the same (connId, subId). + * + * Source of truth: $entries[connId][subId][projectId] => state. + * Hot-path index: $projectIndex[projectId][connId][subId] => true. + */ class EventTailRegistry { - /** @var array, * tokens:float, * lastRefill:float, * dropped:int, * delivered:int, * buffer:array> - * }> keyed by subscriptionId */ - private array $subs = []; + * }>>> connId => subId => projectId => state */ + private array $entries = []; - /** @var array> projectId => set of subIds (hot-path lookup) */ - private array $projectSubscriptionsMap = []; - - /** @var array> connId => set of subIds (cleanup on close) */ - private array $byConnection = []; - - /** @var array projectId => teamId (resolved once per worker at subscribe) */ - private array $teamCache = []; + /** @var array>> projectId => connId => set of subIds */ + private array $projectIndex = []; /** * @param int $rate tokens per second per subscription @@ -45,6 +48,10 @@ class EventTailRegistry return CONSOLE_TAIL_CHANNEL_PREFIX . '.' . $projectId; } + /** + * Extract the projectId from a `console.tail.` channel, or null if the + * channel isn't a tail channel. + */ public static function projectFromChannel(string $channel): ?string { $prefix = CONSOLE_TAIL_CHANNEL_PREFIX . '.'; @@ -55,26 +62,19 @@ class EventTailRegistry return $projectId === '' ? null : $projectId; } - public function cachedTeam(string $projectId): ?string - { - return $this->teamCache[$projectId] ?? null; - } - - public function cacheTeam(string $projectId, string $teamId): void - { - $this->teamCache[$projectId] = $teamId; - } - /** - * Register a tail subscription owned by this worker. + * Register a tail of one project under (connId, subId). Additive, mirroring the + * realtime adapter's subscribe(): adding a new channel never drops channels already + * associated with the subscription id. Re-adding the SAME (connId, subId, projectId) + * overwrites in place with fresh state — that's how a reused subscription id updates + * its filter without leaving a stale entry behind. Entries are dropped only by + * remove()/removeConnection() (i.e. unsubscribe or disconnect), matching the tree. * * @param array $filters result of RuntimeQuery::compile() */ public function add(int $connId, string $subId, string $projectId, array $filters, float $now): void { - $this->subs[$subId] = [ - 'connId' => $connId, - 'projectId' => $projectId, + $this->entries[$connId][$subId][$projectId] = [ 'filters' => $filters, 'tokens' => (float) $this->rate, 'lastRefill' => $now, @@ -82,110 +82,141 @@ class EventTailRegistry 'delivered' => 0, 'buffer' => [], ]; - $this->projectSubscriptionsMap[$projectId][$subId] = true; - $this->byConnection[$connId][$subId] = true; + $this->projectIndex[$projectId][$connId][$subId] = true; } - public function remove(string $subId): void + /** + * Remove every entry for a (connId, subId) — covers a subscription that tailed + * multiple projects. No-op for unknown subscriptions. + */ + public function remove(int $connId, string $subId): void { - $sub = $this->subs[$subId] ?? null; - if ($sub === null) { - return; + foreach (\array_keys($this->entries[$connId][$subId] ?? []) as $projectId) { + $this->detachFromProjectIndex((string) $projectId, $connId, $subId); } - unset($this->projectSubscriptionsMap[$sub['projectId']][$subId]); - if (empty($this->projectSubscriptionsMap[$sub['projectId']])) { - unset($this->projectSubscriptionsMap[$sub['projectId']]); + unset($this->entries[$connId][$subId]); + if (empty($this->entries[$connId])) { + unset($this->entries[$connId]); } - - unset($this->byConnection[$sub['connId']][$subId]); - if (empty($this->byConnection[$sub['connId']])) { - unset($this->byConnection[$sub['connId']]); - } - - unset($this->subs[$subId]); } public function removeConnection(int $connId): void { - foreach (\array_keys($this->byConnection[$connId] ?? []) as $subId) { - $this->remove($subId); + foreach (\array_keys($this->entries[$connId] ?? []) as $subId) { + $this->remove($connId, (string) $subId); } } + private function detachFromProjectIndex(string $projectId, int $connId, string $subId): void + { + unset($this->projectIndex[$projectId][$connId][$subId]); + if (empty($this->projectIndex[$projectId][$connId])) { + unset($this->projectIndex[$projectId][$connId]); + } + if (empty($this->projectIndex[$projectId])) { + unset($this->projectIndex[$projectId]); + } + } + + /** + * O(1) gate for the hot path: does THIS worker hold any tail for the project? + */ public function isTailed(string $projectId): bool { - return isset($this->projectSubscriptionsMap[$projectId]); + return !empty($this->projectIndex[$projectId]); } + /** + * For one decoded firehose event: filter FIRST, then sample into each matching + * entry's buffer. The sampling budget is therefore spent only on events that pass + * the filter. + * + * @param array $payload compact metadata from Realtime::toTailMetadata() + */ public function ingest(string $projectId, array $payload, float $now): void { - foreach (\array_keys($this->projectSubscriptionsMap[$projectId] ?? []) as $subId) { - $sub = &$this->subs[$subId]; + foreach (($this->projectIndex[$projectId] ?? []) as $connId => $subs) { + foreach (\array_keys($subs) as $subId) { + $entry = &$this->entries[$connId][$subId][$projectId]; - if (RuntimeQuery::filter($sub['filters'], $payload) === null) { - continue; + // 1) FILTER FIRST — reuse RuntimeQuery against the compact frame + if (RuntimeQuery::filter($entry['filters'], $payload) === null) { + unset($entry); + continue; + } + + // 2) THEN SAMPLE — lazy token-bucket refill + consume. + // tokens is a fractional value that refills with elapsed time, capped at rate. + $elapsed = \max(0.0, $now - $entry['lastRefill']); + $entry['tokens'] = \min((float) $this->rate, $entry['tokens'] + $elapsed * $this->rate); + $entry['lastRefill'] = $now; + + if ($entry['tokens'] >= 1.0) { + $entry['tokens'] -= 1.0; + $entry['buffer'][] = $payload; + $entry['delivered']++; + } else { + $entry['dropped']++; + } + + unset($entry); } - - // tokens will be always a fractional value lazily increasing with the rate - $elapsed = \max(0.0, $now - $sub['lastRefill']); - $sub['tokens'] = \min((float) $this->rate, $sub['tokens'] + $elapsed * $this->rate); - $sub['lastRefill'] = $now; - - if ($sub['tokens'] >= 1.0) { - $sub['tokens'] -= 1.0; - $sub['buffer'][] = $payload; - $sub['delivered']++; - } else { - $sub['dropped']++; - } - - unset($sub); } } + /** + * Flush all non-empty buffers as one (or more) batched websocket frame(s) per + * entry, plus a counter frame whenever events were dropped. Call from a Timer::tick + * inside onWorkerStart. + */ public function flush(Server $server): void { - foreach ($this->subs as $subId => &$sub) { - $channel = self::channel($sub['projectId']); + foreach ($this->entries as $connId => $bySub) { + foreach ($bySub as $subId => $byProject) { + foreach (\array_keys($byProject) as $projectId) { + $entry = &$this->entries[$connId][$subId][$projectId]; + $channel = self::channel((string) $projectId); - if (!empty($sub['buffer'])) { - // Chunk so a single frame stays under the websocket package max length. - foreach (\array_chunk($sub['buffer'], $this->batchMax) as $chunk) { - $server->send([$sub['connId']], (string) \json_encode([ - 'type' => 'event', - 'data' => [ - 'channels' => [$channel], - 'events' => ['console.tail'], - 'subscriptions' => [$subId], - 'payload' => $chunk, - ], - ])); + if (!empty($entry['buffer'])) { + // Chunk so a single frame stays under the websocket package max length. + foreach (\array_chunk($entry['buffer'], $this->batchMax) as $chunk) { + $server->send([$connId], (string) \json_encode([ + 'type' => 'event', + 'data' => [ + 'channels' => [$channel], + 'events' => ['console.tail'], + 'subscriptions' => [(string) $subId], + 'payload' => $chunk, + ], + ])); + } + $entry['buffer'] = []; + } + + // Counter frame — emit the true rate vs dropped count when sampling kicked in. + if ($entry['dropped'] > 0) { + $server->send([$connId], (string) \json_encode([ + 'type' => 'event', + 'data' => [ + 'channels' => [$channel], + 'events' => ['console.tail.stats'], + 'subscriptions' => [(string) $subId], + 'payload' => [ + '$type' => 'tail.stats', + 'delivered' => $entry['delivered'], + 'dropped' => $entry['dropped'], + ], + ], + ])); + } + + // Reset counters every flush window (~150ms). + $entry['delivered'] = 0; + $entry['dropped'] = 0; + unset($entry); } - $sub['buffer'] = []; } - - // Counter frame — emit the true rate vs dropped count when sampling kicked in. - if ($sub['dropped'] > 0) { - $server->send([$sub['connId']], (string) \json_encode([ - 'type' => 'event', - 'data' => [ - 'channels' => [$channel], - 'events' => ['console.tail.stats'], - 'subscriptions' => [$subId], - 'payload' => [ - '$type' => 'tail.stats', - 'delivered' => $sub['delivered'], - 'dropped' => $sub['dropped'], - ], - ], - ])); - } - - // Reset counters every flush window (~150ms). - $sub['delivered'] = 0; - $sub['dropped'] = 0; } - unset($sub); } } diff --git a/src/Appwrite/Realtime/Message/Handlers/Subscribe.php b/src/Appwrite/Realtime/Message/Handlers/Subscribe.php index 537dba8cdd..158277bbb4 100644 --- a/src/Appwrite/Realtime/Message/Handlers/Subscribe.php +++ b/src/Appwrite/Realtime/Message/Handlers/Subscribe.php @@ -90,22 +90,26 @@ class Subscribe extends Action // Console live event tail: any `console.tail.` channel is authorized by // team membership of the target project's owning team, then registered with its // compiled filter so the worker's pub/sub loop can filter+sample before delivery. + // + // Registration is additive, mirroring Realtime::subscribe(): a reused subscription + // id accumulates channels (it never drops the ones already subscribed), and the + // registry is keyed by (connId, subId, projectId) so re-subscribing the same + // channel overwrites its filter in place rather than leaving a stale entry. $now = \microtime(true); foreach ($parsedPayloads as $parsedPayload) { + $compiled = null; foreach ($parsedPayload['convertedChannels'] as $channel) { $targetProjectId = EventTailRegistry::projectFromChannel($channel); if ($targetProjectId === null) { continue; } - $teamId = $eventTailRegistry->cachedTeam($targetProjectId); - if ($teamId === null) { - $project = $dbForConsole->getAuthorization()->skip( - fn () => $dbForConsole->getDocument('projects', $targetProjectId) - ); - $teamId = (string) $project->getAttribute('teamId', ''); - $eventTailRegistry->cacheTeam($targetProjectId, $teamId); - } + // Resolve ownership fresh on every subscribe — projects can be transferred + // between teams, so a cached teamId could authorize the wrong team. + $project = $dbForConsole->getAuthorization()->skip( + fn () => $dbForConsole->getDocument('projects', $targetProjectId) + ); + $teamId = (string) $project->getAttribute('teamId', ''); if ($teamId === '' || !\in_array(Role::team($teamId)->toString(), $roles, true)) { throw new Exception( @@ -114,13 +118,8 @@ class Subscribe extends Action ); } - $eventTailRegistry->add( - $connectionId, - $parsedPayload['subscriptionId'], - $targetProjectId, - RuntimeQuery::compile($parsedPayload['queries']), - $now, - ); + $compiled ??= RuntimeQuery::compile($parsedPayload['queries']); + $eventTailRegistry->add($connectionId, $parsedPayload['subscriptionId'], $targetProjectId, $compiled, $now); } } diff --git a/src/Appwrite/Realtime/Message/Handlers/Unsubscribe.php b/src/Appwrite/Realtime/Message/Handlers/Unsubscribe.php index cba8aa4a3f..6e2ad5ea63 100644 --- a/src/Appwrite/Realtime/Message/Handlers/Unsubscribe.php +++ b/src/Appwrite/Realtime/Message/Handlers/Unsubscribe.php @@ -46,8 +46,9 @@ class Unsubscribe extends Action 'subscriptionId' => $subscriptionId, 'removed' => $realtime->unsubscribeSubscription($connectionId, $subscriptionId), ]; - // No-op if it wasn't a tail subscription. - $eventTailRegistry->remove($subscriptionId); + // No-op if it wasn't a tail subscription. Connection-scoped: subscription + // IDs are only unique within a connection. + $eventTailRegistry->remove($connectionId, $subscriptionId); } $subscriptionsAfter = \count($realtime->getSubscriptionMetadata($connectionId)); diff --git a/tests/unit/Realtime/EventTailRegistryTest.php b/tests/unit/Realtime/EventTailRegistryTest.php index 3f6160142d..e806e148cc 100644 --- a/tests/unit/Realtime/EventTailRegistryTest.php +++ b/tests/unit/Realtime/EventTailRegistryTest.php @@ -36,6 +36,23 @@ final class EventTailRegistryTest extends TestCase return $server; } + /** + * Like server(), but records the target connection id alongside each frame so tests + * can assert per-connection isolation. + * + * @param array}> $sent + */ + private function serverWithConn(array &$sent): Server + { + $server = $this->createStub(Server::class); + $server->method('send')->willReturnCallback( + function (array $connections, string $message) use (&$sent): void { + $sent[] = ['connId' => $connections[0] ?? null, 'frame' => json_decode($message, true)]; + } + ); + return $server; + } + /** @param array $payload */ private function compile(array $queries): array { @@ -345,22 +362,118 @@ final class EventTailRegistryTest extends TestCase $registry->add(7, 'sub-a', 'projX', $this->compile([]), 0.0); $registry->add(7, 'sub-b', 'projX', $this->compile([]), 0.0); - $registry->remove('sub-a'); + $registry->remove(7, 'sub-a'); $this->assertTrue($registry->isTailed('projX'), 'sub-b still tails projX'); - $registry->remove('sub-b'); + $registry->remove(7, 'sub-b'); $this->assertFalse($registry->isTailed('projX')); // Removing an unknown subscription is a no-op. - $registry->remove('does-not-exist'); + $registry->remove(7, 'does-not-exist'); $this->assertFalse($registry->isTailed('projX')); } - public function testTeamCache(): void + public function testSameSubscriptionIdOnDifferentConnectionsIsolated(): void { - $registry = new EventTailRegistry(); - $this->assertNull($registry->cachedTeam('projX')); - $registry->cacheTeam('projX', 'team-1'); - $this->assertSame('team-1', $registry->cachedTeam('projX')); + // Subscription IDs are only unique within a connection. Two connections both + // using 'dup' must stay fully isolated — no cross-project/connection leakage. + $registry = new EventTailRegistry(rate: 100); + $registry->add(1, 'dup', 'projA', $this->compile([]), 0.0); + $registry->add(2, 'dup', 'projB', $this->compile([]), 0.0); + + $registry->ingest('projA', ['type' => 'databases', 'resourceId' => 'a1'], 0.0); + $registry->ingest('projB', ['type' => 'databases', 'resourceId' => 'b1'], 0.0); + + $sent = []; + $registry->flush($this->serverWithConn($sent)); + + $byConn = []; + foreach ($sent as $s) { + $byConn[$s['connId']][] = $s['frame']['data']['channels'][0]; + } + // Connection 1 only ever receives projA's channel; connection 2 only projB's. + $this->assertSame(['console.tail.projA'], $byConn[1] ?? []); + $this->assertSame(['console.tail.projB'], $byConn[2] ?? []); + + // Removing one connection's 'dup' must not touch the other's. + $registry->remove(1, 'dup'); + $this->assertFalse($registry->isTailed('projA')); + $this->assertTrue($registry->isTailed('projB')); + } + + public function testMultipleProjectsUnderOneSubscription(): void + { + // A subscription naming several tail channels tails each project independently; + // none overwrites another, and removing the subscription clears them all. + $registry = new EventTailRegistry(rate: 100); + $registry->add(1, 'sub-multi', 'projA', $this->compile([]), 0.0); + $registry->add(1, 'sub-multi', 'projB', $this->compile([]), 0.0); + + $this->assertTrue($registry->isTailed('projA')); + $this->assertTrue($registry->isTailed('projB')); + + $registry->ingest('projA', ['type' => 'databases', 'resourceId' => 'a1'], 0.0); + $registry->ingest('projB', ['type' => 'databases', 'resourceId' => 'b1'], 0.0); + + $sent = []; + $registry->flush($this->serverWithConn($sent)); + + $channels = array_map(fn ($s) => $s['frame']['data']['channels'][0], $sent); + $this->assertContains('console.tail.projA', $channels); + $this->assertContains('console.tail.projB', $channels); + // Both frames carry the same (real) subscription id. + foreach ($sent as $s) { + $this->assertSame(['sub-multi'], $s['frame']['data']['subscriptions']); + } + + $registry->remove(1, 'sub-multi'); + $this->assertFalse($registry->isTailed('projA')); + $this->assertFalse($registry->isTailed('projB')); + } + + public function testReAddingSameChannelOverwritesFilterWithoutStaleEntry(): void + { + // Reusing a subscription id on the SAME tail channel updates its filter in place + // (keyed by connId/subId/projectId) — no stale entry, no duplicate delivery. + $registry = new EventTailRegistry(rate: 100); + $registry->add(1, 'x', 'projA', $this->compile([Query::equal('action', ['create'])]), 0.0); + $registry->add(1, 'x', 'projA', $this->compile([Query::equal('action', ['delete'])]), 0.0); + + $registry->ingest('projA', ['action' => 'create', 'resourceId' => 'c1'], 0.0); + $registry->ingest('projA', ['action' => 'delete', 'resourceId' => 'd1'], 0.0); + + $sent = []; + $registry->flush($this->server($sent)); + + $eventFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail'])); + // A lingering create-filter entry would surface the create and/or add a 2nd frame. + $this->assertCount(1, $eventFrames); + $this->assertCount(1, $eventFrames[0]['data']['payload']); + $this->assertSame('delete', $eventFrames[0]['data']['payload'][0]['action']); + $this->assertSame('d1', $eventFrames[0]['data']['payload'][0]['resourceId']); + } + + public function testAddingChannelPreservesExistingTail(): void + { + // Review scenario: subscribe id 'x' to tail A, later add tail B under the same id. + // Registration is additive (mirrors Realtime::subscribe()), so A must keep delivering. + $registry = new EventTailRegistry(rate: 100); + $registry->add(1, 'x', 'projA', $this->compile([]), 0.0); + + // ... later, the same subscription id also tails projB. + $registry->add(1, 'x', 'projB', $this->compile([]), 0.0); + + $this->assertTrue($registry->isTailed('projA'), 'projA must remain tailed after adding projB'); + $this->assertTrue($registry->isTailed('projB')); + + $registry->ingest('projA', ['type' => 'databases', 'resourceId' => 'a1'], 0.0); + $registry->ingest('projB', ['type' => 'databases', 'resourceId' => 'b1'], 0.0); + + $sent = []; + $registry->flush($this->serverWithConn($sent)); + + $channels = array_map(fn ($s) => $s['frame']['data']['channels'][0], $sent); + $this->assertContains('console.tail.projA', $channels, 'projA stopped delivering after projB was added'); + $this->assertContains('console.tail.projB', $channels); } } From 9ead3a65ecd845e8efea74d828ca92424ec517f5 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sun, 7 Jun 2026 12:36:44 +0530 Subject: [PATCH 10/15] feat: enhance Realtime event handling with improved scope extraction and authorization for subscriptions --- src/Appwrite/Messaging/Adapter/Realtime.php | 46 ++++++++++++----- .../Realtime/Message/Handlers/Subscribe.php | 49 ++++++++++--------- tests/unit/Realtime/EventTailRegistryTest.php | 40 +++++++++++++-- 3 files changed, 97 insertions(+), 38 deletions(-) diff --git a/src/Appwrite/Messaging/Adapter/Realtime.php b/src/Appwrite/Messaging/Adapter/Realtime.php index f7602f8c9d..91697be18a 100644 --- a/src/Appwrite/Messaging/Adapter/Realtime.php +++ b/src/Appwrite/Messaging/Adapter/Realtime.php @@ -534,20 +534,42 @@ class Realtime extends MessagingAdapter 'resourceId' => $payload['$id'] ?? null, ]; - // Attach only the scope ids that belong to this resource type. - $scope = match ($type) { - 'databases' => [ - 'databaseId' => $payload['$databaseId'] ?? null, - 'collectionId' => $payload['$collectionId'] ?? $payload['$tableId'] ?? null, - ], - 'buckets' => ['bucketId' => $payload['bucketId'] ?? null], - 'functions' => ['functionId' => $payload['functionId'] ?? null], - 'teams' => ['teamId' => $payload['teamId'] ?? null], - default => [], - }; + // Attach only the scope ids that belong to this resource type. Derived from the + // (concrete) event name rather than the payload: for a top-level resource event + // (e.g. `teams.T.create`) the payload's id lives at $id, not at a teamId field, + // so a payload lookup would drop the scope and silently break filters. The event + // name encodes the id at every level — parts[1] is the top-level resource id and + // parts[3] the collection/table id. + $scope = []; + switch ($type) { + case 'databases': + if (isset($parts[1])) { + $scope['databaseId'] = $parts[1]; + } + if (isset($parts[2], $parts[3]) && \in_array($parts[2], ['collections', 'tables'], true)) { + $scope['collectionId'] = $parts[3]; + } + break; + case 'buckets': + if (isset($parts[1])) { + $scope['bucketId'] = $parts[1]; + } + break; + case 'functions': + if (isset($parts[1])) { + $scope['functionId'] = $parts[1]; + } + break; + case 'teams': + if (isset($parts[1])) { + $scope['teamId'] = $parts[1]; + } + break; + } foreach ($scope as $key => $value) { - if ($value !== null) { + // Skip wildcard/empty segments (defensive: events[0] is concrete, but guard anyway). + if ($value !== '' && $value !== '*') { $metadata[$key] = $value; } } diff --git a/src/Appwrite/Realtime/Message/Handlers/Subscribe.php b/src/Appwrite/Realtime/Message/Handlers/Subscribe.php index 158277bbb4..b421aeb424 100644 --- a/src/Appwrite/Realtime/Message/Handlers/Subscribe.php +++ b/src/Appwrite/Realtime/Message/Handlers/Subscribe.php @@ -76,26 +76,14 @@ class Subscribe extends Action ]; } - foreach ($parsedPayloads as $parsedPayload) { - $realtime->subscribe( - $projectId, - $connectionId, - $parsedPayload['subscriptionId'], - $roles, - $parsedPayload['convertedChannels'], - $parsedPayload['queries'], - ); - } - - // Console live event tail: any `console.tail.` channel is authorized by - // team membership of the target project's owning team, then registered with its - // compiled filter so the worker's pub/sub loop can filter+sample before delivery. - // - // Registration is additive, mirroring Realtime::subscribe(): a reused subscription - // id accumulates channels (it never drops the ones already subscribed), and the - // registry is keyed by (connId, subId, projectId) so re-subscribing the same - // channel overwrites its filter in place rather than leaving a stale entry. + // Console live event tail: authorize every `console.tail.` channel + // FIRST — before any subscription state is mutated — so a batch mixing authorized + // and unauthorized tails can't leave partial state behind. Each tail is authorized + // by team membership of the target project's owning team, with ownership resolved + // fresh (projects can be transferred between teams, so a cached teamId could + // authorize the wrong team). $now = \microtime(true); + $pendingTails = []; foreach ($parsedPayloads as $parsedPayload) { $compiled = null; foreach ($parsedPayload['convertedChannels'] as $channel) { @@ -104,8 +92,6 @@ class Subscribe extends Action continue; } - // Resolve ownership fresh on every subscribe — projects can be transferred - // between teams, so a cached teamId could authorize the wrong team. $project = $dbForConsole->getAuthorization()->skip( fn () => $dbForConsole->getDocument('projects', $targetProjectId) ); @@ -119,10 +105,29 @@ class Subscribe extends Action } $compiled ??= RuntimeQuery::compile($parsedPayload['queries']); - $eventTailRegistry->add($connectionId, $parsedPayload['subscriptionId'], $targetProjectId, $compiled, $now); + $pendingTails[] = [$parsedPayload['subscriptionId'], $targetProjectId, $compiled]; } } + foreach ($parsedPayloads as $parsedPayload) { + $realtime->subscribe( + $projectId, + $connectionId, + $parsedPayload['subscriptionId'], + $roles, + $parsedPayload['convertedChannels'], + $parsedPayload['queries'], + ); + } + + // All tail channels passed authorization above — register them additively. The + // registry is keyed by (connId, subId, projectId): re-subscribing the same channel + // overwrites its filter in place, while a reused subscription id that names a new + // tail channel keeps the ones already registered (mirrors the tree). + foreach ($pendingTails as [$subId, $targetProjectId, $compiled]) { + $eventTailRegistry->add($connectionId, $subId, $targetProjectId, $compiled, $now); + } + $subscriptionsAfter = \count($realtime->getSubscriptionMetadata($connectionId)); $subscriptionDelta = $subscriptionsAfter - $subscriptionsBefore; $subscriptionsRequested = \count($parsedPayloads); diff --git a/tests/unit/Realtime/EventTailRegistryTest.php b/tests/unit/Realtime/EventTailRegistryTest.php index e806e148cc..97d23b5f5f 100644 --- a/tests/unit/Realtime/EventTailRegistryTest.php +++ b/tests/unit/Realtime/EventTailRegistryTest.php @@ -53,7 +53,7 @@ final class EventTailRegistryTest extends TestCase return $server; } - /** @param array $payload */ + /** @param array $queries */ private function compile(array $queries): array { return RuntimeQuery::compile($queries); @@ -114,11 +114,43 @@ final class EventTailRegistryTest extends TestCase $this->assertArrayNotHasKey('functionId', $file); $this->assertArrayNotHasKey('collectionId', $file); - // A null scope id is omitted rather than emitted. - $fnNoScope = Realtime::toTailMetadata([ + // Scope comes from the event name, so it's present even when the payload (here an + // execution doc) carries no functionId field — resourceId is the leaf ($id). + $exec = Realtime::toTailMetadata([ 'data' => ['events' => ['functions.fn1.executions.e1.create'], 'payload' => ['$id' => 'e1']], ]); - $this->assertArrayNotHasKey('functionId', $fnNoScope); + $this->assertSame('fn1', $exec['functionId']); + $this->assertSame('e1', $exec['resourceId']); + } + + public function testTailMetadataTopLevelResourceEventsCarryScope(): void + { + // Top-level resource events: the payload is the resource doc itself (id at $id, + // not at a teamId/bucketId/functionId field). Scope must still be present so a + // filter like equal('teamId', ['T']) matches the team's own create/update events. + $teamCreate = Realtime::toTailMetadata([ + 'data' => ['events' => ['teams.T.create'], 'payload' => ['$id' => 'T', 'name' => 'Acme']], + ]); + $this->assertSame('teams', $teamCreate['type']); + $this->assertSame('T', $teamCreate['teamId']); + $this->assertSame('T', $teamCreate['resourceId']); + + // Nested membership event: same teamId scope, leaf resourceId is the membership. + $membership = Realtime::toTailMetadata([ + 'data' => ['events' => ['teams.T.memberships.M.create'], 'payload' => ['$id' => 'M', 'teamId' => 'T']], + ]); + $this->assertSame('T', $membership['teamId']); + $this->assertSame('M', $membership['resourceId']); + + $bucketCreate = Realtime::toTailMetadata([ + 'data' => ['events' => ['buckets.B.create'], 'payload' => ['$id' => 'B']], + ]); + $this->assertSame('B', $bucketCreate['bucketId']); + + $functionCreate = Realtime::toTailMetadata([ + 'data' => ['events' => ['functions.F.create'], 'payload' => ['$id' => 'F']], + ]); + $this->assertSame('F', $functionCreate['functionId']); } public function testTailMetadataTablesUsesTableIdAsCollection(): void From 435b9c36f2c3f3516707569a9274f566bb47df65 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sun, 7 Jun 2026 13:31:25 +0530 Subject: [PATCH 11/15] added tests --- ...altimeCustomClientQueryTestWithMessage.php | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php b/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php index 3b1f8c6e54..c7841ebf16 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php @@ -1037,4 +1037,76 @@ final class RealtimeCustomClientQueryTestWithMessage extends Scope // already closed by the server } } + + public function testConsoleTailSeesEventsNotReadableByTheConsoleUser(): void + { + // The tail authorizes the *viewer* by team membership at subscribe time and then + // forwards every project event regardless of the resource's read ACL. A document + // readable only by some OTHER user (not `any`, not the console user) must still + // appear in the console tail. + $projectId = $this->getProject()['$id']; + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]; + + $database = $this->client->call(Client::METHOD_POST, '/databases', $headers, [ + 'databaseId' => ID::unique(), + 'name' => 'Tail ACL DB', + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // documentSecurity ON and NO read(any) at the collection level, so the only read + // grant comes from the document's own (restricted) permission. + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', $headers, [ + 'collectionId' => ID::unique(), + 'name' => 'Secrets', + 'documentSecurity' => true, + 'permissions' => [Permission::create(Role::any())], + ]); + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', $headers, [ + 'key' => 'name', + 'size' => 256, + 'required' => true, + ]); + $this->assertEventually(function () use ($databaseId, $collectionId, $headers) { + $attribute = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/name', $headers); + $this->assertEquals(200, $attribute['headers']['status-code']); + $this->assertEquals('available', $attribute['body']['status']); + }, 120000, 500); + + $client = $this->openConsoleTail($projectId); + + // Read restricted to a user that is NOT the console user → the console viewer + // could never read this document directly, yet the tail must surface its event. + $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers, [ + 'documentId' => ID::unique(), + 'data' => ['name' => 'top secret'], + 'permissions' => [Permission::read(Role::user('not-the-console-user'))], + ]); + $this->assertEquals(201, $document['headers']['status-code']); + $documentId = $document['body']['$id']; + + $frames = $this->receiveTailFrames($client); + + $match = null; + foreach ($frames as $frame) { + if (($frame['resourceId'] ?? null) === $documentId) { + $match = $frame; + break; + } + } + + $this->assertNotNull($match, 'restricted-read document.create not present in tail: ' . \json_encode($frames)); + $this->assertSame('databases', $match['type']); + $this->assertSame('create', $match['action']); + $this->assertArrayNotHasKey('name', $match); // still no document body + + $client->close(); + } } From 80d4f27caa1edf21522177f3cb1ecefdc395315a Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sun, 7 Jun 2026 13:33:47 +0530 Subject: [PATCH 12/15] removed redundant --- src/Appwrite/Realtime/EventTailRegistry.php | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/Appwrite/Realtime/EventTailRegistry.php b/src/Appwrite/Realtime/EventTailRegistry.php index b2f0950038..744ab30f25 100644 --- a/src/Appwrite/Realtime/EventTailRegistry.php +++ b/src/Appwrite/Realtime/EventTailRegistry.php @@ -9,13 +9,6 @@ use Utopia\WebSocket\Server; * Per-worker token-bucket + buffers for console event-tail subscriptions. * * Each tail of a single project is one entry, keyed by (connId, subId, projectId). - * Subscription IDs are only unique WITHIN a connection, so the connection id is part - * of the key — subscriptions from different connections (or different projects under - * the same subscription id) never collide. A subscription that names several tail - * channels gets one entry per project under the same (connId, subId). - * - * Source of truth: $entries[connId][subId][projectId] => state. - * Hot-path index: $projectIndex[projectId][connId][subId] => true. */ class EventTailRegistry { From 5e2007ead29f9eaaa285b435fc9a45b598745d7a Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 23 Jun 2026 17:16:49 +0530 Subject: [PATCH 13/15] linting --- app/realtime.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/realtime.php b/app/realtime.php index 91b55d1ec4..cd1d3bf83e 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -1455,7 +1455,7 @@ $server->onClose(function (int $connection) use ($realtime, $stats, $register, $ $eventTailRegistry->removeConnection($connection); } catch (\Throwable $th) { Console::error('Realtime onClose tail cleanup error: ' . $th->getMessage()); - Span::error($th); + Span::current()?->setError($th); } Span::add('realtime.success', $success); From acdeb28422e265cd6b6b60174d4216cc990bd7a1 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 24 Jun 2026 10:37:09 +0530 Subject: [PATCH 14/15] addressed greptile comments --- src/Appwrite/Messaging/Adapter/Realtime.php | 60 +++++++++++-------- .../Realtime/Message/Handlers/Subscribe.php | 28 ++++++++- ...altimeCustomClientQueryTestWithMessage.php | 47 ++++++++++++++- tests/unit/Realtime/EventTailRegistryTest.php | 27 +++++++++ 4 files changed, 133 insertions(+), 29 deletions(-) diff --git a/src/Appwrite/Messaging/Adapter/Realtime.php b/src/Appwrite/Messaging/Adapter/Realtime.php index 91697be18a..9cf5866d80 100644 --- a/src/Appwrite/Messaging/Adapter/Realtime.php +++ b/src/Appwrite/Messaging/Adapter/Realtime.php @@ -16,6 +16,11 @@ class Realtime extends MessagingAdapter { public const SUPPORTED_ACTIONS = ['create', 'update', 'upsert', 'delete']; + // The database family is published under several event prefixes (legacy `databases` + // plus the per-type projections). They share the same `databases...…` + // shape, so the tail normalizes them to `databases` for consistent metadata + filtering. + public const DATABASE_EVENT_PREFIXES = ['databases', 'tablesdb', 'documentsdb', 'vectorsdb']; + // Resources whose channels receive an action-suffixed sibling at publish time. // The suffix loop in fromPayload() treats any channel whose last OR second-to-last // segment matches an entry here as a candidate for `.{action}` suffixing. @@ -513,6 +518,12 @@ class Realtime extends MessagingAdapter $count = \count($parts); $type = $parts[0] ?? null; + // The database family is published under several prefixes (databases/tablesdb/ + // documentsdb/vectorsdb). `type` keeps the actual prefix (more informative), but + // scope ids are extracted for all of them so databaseId/collectionId filters work + // across the whole family. + $isDatabaseFamily = \in_array($type, self::DATABASE_EVENT_PREFIXES, true); + $action = null; if ($count >= 1 && \in_array($parts[$count - 1], self::SUPPORTED_ACTIONS, true)) { $action = $parts[$count - 1]; @@ -541,30 +552,31 @@ class Realtime extends MessagingAdapter // name encodes the id at every level — parts[1] is the top-level resource id and // parts[3] the collection/table id. $scope = []; - switch ($type) { - case 'databases': - if (isset($parts[1])) { - $scope['databaseId'] = $parts[1]; - } - if (isset($parts[2], $parts[3]) && \in_array($parts[2], ['collections', 'tables'], true)) { - $scope['collectionId'] = $parts[3]; - } - break; - case 'buckets': - if (isset($parts[1])) { - $scope['bucketId'] = $parts[1]; - } - break; - case 'functions': - if (isset($parts[1])) { - $scope['functionId'] = $parts[1]; - } - break; - case 'teams': - if (isset($parts[1])) { - $scope['teamId'] = $parts[1]; - } - break; + if ($isDatabaseFamily) { + if (isset($parts[1])) { + $scope['databaseId'] = $parts[1]; + } + if (isset($parts[2], $parts[3]) && \in_array($parts[2], ['collections', 'tables'], true)) { + $scope['collectionId'] = $parts[3]; + } + } else { + switch ($type) { + case 'buckets': + if (isset($parts[1])) { + $scope['bucketId'] = $parts[1]; + } + break; + case 'functions': + if (isset($parts[1])) { + $scope['functionId'] = $parts[1]; + } + break; + case 'teams': + if (isset($parts[1])) { + $scope['teamId'] = $parts[1]; + } + break; + } } foreach ($scope as $key => $value) { diff --git a/src/Appwrite/Realtime/Message/Handlers/Subscribe.php b/src/Appwrite/Realtime/Message/Handlers/Subscribe.php index b421aeb424..bb8177cc63 100644 --- a/src/Appwrite/Realtime/Message/Handlers/Subscribe.php +++ b/src/Appwrite/Realtime/Message/Handlers/Subscribe.php @@ -92,9 +92,31 @@ class Subscribe extends Action continue; } - $project = $dbForConsole->getAuthorization()->skip( - fn () => $dbForConsole->getDocument('projects', $targetProjectId) - ); + // Tail channels are console-only. A normal project websocket (even for a + // team member) must not tail — that would expose compact metadata for + // resources the user could not read through the project API. Only a + // connection to the `console` project may register a tail. + if ($projectId !== 'console') { + throw new Exception( + Exception::REALTIME_POLICY_VIOLATION, + 'Console live tail is only available on the console connection.' + ); + } + + // Resolve ownership fresh (projects can move between teams). Treat any + // lookup failure (missing/invalid project) as a policy violation so the + // connection is closed with 1008, consistent with an unauthorized tail — + // rather than leaking a raw not-found error and staying open. + try { + $project = $dbForConsole->getAuthorization()->skip( + fn () => $dbForConsole->getDocument('projects', $targetProjectId) + ); + } catch (\Throwable) { + throw new Exception( + Exception::REALTIME_POLICY_VIOLATION, + 'Not authorized to tail project "' . $targetProjectId . '".' + ); + } $teamId = (string) $project->getAttribute('teamId', ''); if ($teamId === '' || !\in_array(Role::team($teamId)->toString(), $roles, true)) { diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php b/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php index c7841ebf16..c0c47cf668 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php @@ -1026,14 +1026,14 @@ final class RealtimeCustomClientQueryTestWithMessage extends Scope $response = \json_decode($client->receive(), true); $this->assertEquals('error', $response['type'] ?? null); $this->assertEquals(1008, $response['data']['code'] ?? null); - } catch (ConnectionException $e) { + } catch (ConnectionException) { $this->addToAssertionCount(1); } // close() can itself throw on an already-closed socket — best effort. try { $client->close(); - } catch (ConnectionException $e) { + } catch (ConnectionException) { // already closed by the server } } @@ -1109,4 +1109,47 @@ final class RealtimeCustomClientQueryTestWithMessage extends Scope $client->close(); } + + public function testProjectConnectionCannotTail(): void + { + // A normal project websocket (not the console project) must not be able to tail, + // even for a team member — only the console connection may. Expect a 1008 reject. + $user = $this->getUser(); + $session = $user['session'] ?? ''; + $projectId = $this->getProject()['$id']; + + $queryString = \http_build_query(['project' => $projectId]); // NOT console + $client = new WebSocketClient( + 'ws://appwrite.test/v1/realtime?' . $queryString, + [ + 'headers' => [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ], + 'timeout' => 10, + ] + ); + + $connected = \json_decode($client->receive(), true); + $this->assertEquals('connected', $connected['type'] ?? null); + + $client->send(\json_encode([ + 'type' => 'subscribe', + 'data' => [['channels' => ['console.tail.' . $projectId], 'subscriptionId' => 'tail-proj']], + ])); + + try { + $response = \json_decode($client->receive(), true); + $this->assertEquals('error', $response['type'] ?? null); + $this->assertEquals(1008, $response['data']['code'] ?? null); + } catch (ConnectionException) { + $this->addToAssertionCount(1); + } + + try { + $client->close(); + } catch (ConnectionException) { + // already closed by the server + } + } } diff --git a/tests/unit/Realtime/EventTailRegistryTest.php b/tests/unit/Realtime/EventTailRegistryTest.php index 97d23b5f5f..6d3da66ba0 100644 --- a/tests/unit/Realtime/EventTailRegistryTest.php +++ b/tests/unit/Realtime/EventTailRegistryTest.php @@ -167,6 +167,33 @@ final class EventTailRegistryTest extends TestCase $this->assertSame('t1', $meta['collectionId']); } + public function testTailMetadataExtractsScopeForAllDatabasePrefixes(): void + { + // The database family is published under several prefixes. `type` keeps the + // actual prefix, and databaseId/collectionId are extracted for every one of them + // (so scope filters work across the whole family regardless of the variant). + foreach (['databases', 'tablesdb', 'documentsdb', 'vectorsdb'] as $prefix) { + $meta = Realtime::toTailMetadata([ + 'userId' => 'u1', + 'data' => [ + 'events' => ["{$prefix}.main.collections.posts.documents.p1.create"], + 'payload' => ['$id' => 'p1'], + ], + ]); + $this->assertSame($prefix, $meta['type'], 'type preserves the actual prefix'); + $this->assertSame('main', $meta['databaseId']); + $this->assertSame('posts', $meta['collectionId']); + } + + // `tables` sub-resource under a projected prefix still maps to collectionId. + $rows = Realtime::toTailMetadata([ + 'data' => ['events' => ['tablesdb.main.tables.t1.rows.r1.create'], 'payload' => ['$id' => 'r1']], + ]); + $this->assertSame('tablesdb', $rows['type']); + $this->assertSame('main', $rows['databaseId']); + $this->assertSame('t1', $rows['collectionId']); + } + public function testTailMetadataAttributeTrailingAction(): void { // `users.U.update.email` — action is the second-to-last segment. From 1952079fee2bc655fe1aa3c4d520e9842972a061 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 24 Jun 2026 11:12:45 +0530 Subject: [PATCH 15/15] addressed greptile comments --- app/realtime.php | 26 ++++++++-- src/Appwrite/Realtime/EventTailRegistry.php | 50 +++++++++++++++---- .../Realtime/Message/Handlers/Subscribe.php | 11 ++-- tests/unit/Realtime/EventTailRegistryTest.php | 43 ++++++++++++++++ 4 files changed, 114 insertions(+), 16 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index cd1d3bf83e..b2b7e5fa46 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -727,6 +727,11 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, if ($subscriptionDelta !== 0) { $register->get('telemetry.workerSubscriptionCounter')->add($subscriptionDelta, $register->get('telemetry.workerAttributes')); } + + // Tail entries live outside the subscription tree, so the rebuild + // above doesn't touch them. Drop any whose authorizing team role + // the connection no longer holds (membership revoked / project moved). + $eventTailRegistry->revalidateConnection($connection, $roles); } } } @@ -815,9 +820,24 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, // whether it had regular subscribers ($total). The firehose is published // unconditionally, so a project tailed from the console with no other // realtime clients must still be observed here. - $src = $event['project'] ?? ''; - if ($src !== '' && $src !== 'console' && $eventTailRegistry->isTailed($src)) { - $eventTailRegistry->ingest($src, Realtime::toTailMetadata($event), \microtime(true)); + // + // Most events carry the owning project in $event['project']. But some + // project-owned events (schema changes, deployments, rules, migrations, + // project settings) are rerouted by fromPayload() to project='console'; + // for those the real project id lives in the `projects.` channel. + // Resolve that so the tail still receives them. + $tailProjectId = $event['project'] ?? ''; + if ($tailProjectId === 'console') { + $tailProjectId = ''; + foreach (($event['data']['channels'] ?? []) as $channel) { + if (\str_starts_with($channel, 'projects.')) { + $tailProjectId = \substr($channel, \strlen('projects.')); + break; + } + } + } + if ($tailProjectId !== '' && $eventTailRegistry->isTailed($tailProjectId)) { + $eventTailRegistry->ingest($tailProjectId, Realtime::toTailMetadata($event), \microtime(true)); } }); } catch (Throwable $th) { diff --git a/src/Appwrite/Realtime/EventTailRegistry.php b/src/Appwrite/Realtime/EventTailRegistry.php index 744ab30f25..2a35666efd 100644 --- a/src/Appwrite/Realtime/EventTailRegistry.php +++ b/src/Appwrite/Realtime/EventTailRegistry.php @@ -13,6 +13,7 @@ use Utopia\WebSocket\Server; class EventTailRegistry { /** @var array, * tokens:float, * lastRefill:float, @@ -25,12 +26,18 @@ class EventTailRegistry /** @var array>> projectId => connId => set of subIds */ private array $projectIndex = []; + private int $rate; + + private int $batchMax; + /** * @param int $rate tokens per second per subscription * @param int $batchMax max frames per flushed websocket frame (kept under setPackageMaxLength) */ - public function __construct(private int $rate = 50, private int $batchMax = 200) + public function __construct(int $rate = 50, int $batchMax = 200) { + $this->rate = \max(1, $rate); + $this->batchMax = \max(1, $batchMax); } /** @@ -64,10 +71,12 @@ class EventTailRegistry * remove()/removeConnection() (i.e. unsubscribe or disconnect), matching the tree. * * @param array $filters result of RuntimeQuery::compile() + * @param string $role the team role this tail was authorized with */ - public function add(int $connId, string $subId, string $projectId, array $filters, float $now): void + public function add(int $connId, string $subId, string $projectId, array $filters, float $now, string $role = ''): void { $this->entries[$connId][$subId][$projectId] = [ + 'role' => $role, 'filters' => $filters, 'tokens' => (float) $this->rate, 'lastRefill' => $now, @@ -85,12 +94,7 @@ class EventTailRegistry public function remove(int $connId, string $subId): void { foreach (\array_keys($this->entries[$connId][$subId] ?? []) as $projectId) { - $this->detachFromProjectIndex((string) $projectId, $connId, $subId); - } - - unset($this->entries[$connId][$subId]); - if (empty($this->entries[$connId])) { - unset($this->entries[$connId]); + $this->detachEntry($connId, $subId, (string) $projectId); } } @@ -101,7 +105,27 @@ class EventTailRegistry } } - private function detachFromProjectIndex(string $projectId, int $connId, string $subId): void + /** + * Drop any tail whose authorizing team role the connection no longer holds. Called + * from the permissions-changed path (which already recomputes a connection's roles + * and rebuilds its tree) so a membership revocation or project transfer stops the + * tail mid-connection instead of streaming until disconnect. + * + * @param array $roles the connection's current roles + */ + public function revalidateConnection(int $connId, array $roles): void + { + foreach (($this->entries[$connId] ?? []) as $subId => $byProject) { + foreach ($byProject as $projectId => $entry) { + $role = $entry['role']; + if ($role !== '' && !\in_array($role, $roles, true)) { + $this->detachEntry($connId, (string) $subId, (string) $projectId); + } + } + } + } + + private function detachEntry(int $connId, string $subId, string $projectId): void { unset($this->projectIndex[$projectId][$connId][$subId]); if (empty($this->projectIndex[$projectId][$connId])) { @@ -110,6 +134,14 @@ class EventTailRegistry if (empty($this->projectIndex[$projectId])) { unset($this->projectIndex[$projectId]); } + + unset($this->entries[$connId][$subId][$projectId]); + if (empty($this->entries[$connId][$subId])) { + unset($this->entries[$connId][$subId]); + } + if (empty($this->entries[$connId])) { + unset($this->entries[$connId]); + } } /** diff --git a/src/Appwrite/Realtime/Message/Handlers/Subscribe.php b/src/Appwrite/Realtime/Message/Handlers/Subscribe.php index bb8177cc63..013e6bf864 100644 --- a/src/Appwrite/Realtime/Message/Handlers/Subscribe.php +++ b/src/Appwrite/Realtime/Message/Handlers/Subscribe.php @@ -118,8 +118,9 @@ class Subscribe extends Action ); } $teamId = (string) $project->getAttribute('teamId', ''); + $requiredRole = Role::team($teamId)->toString(); - if ($teamId === '' || !\in_array(Role::team($teamId)->toString(), $roles, true)) { + if ($teamId === '' || !\in_array($requiredRole, $roles, true)) { throw new Exception( Exception::REALTIME_POLICY_VIOLATION, 'Not authorized to tail project "' . $targetProjectId . '".' @@ -127,7 +128,9 @@ class Subscribe extends Action } $compiled ??= RuntimeQuery::compile($parsedPayload['queries']); - $pendingTails[] = [$parsedPayload['subscriptionId'], $targetProjectId, $compiled]; + // Carry the authorizing role so the tail can be revoked if the viewer + // later loses team membership (or the project is transferred). + $pendingTails[] = [$parsedPayload['subscriptionId'], $targetProjectId, $compiled, $requiredRole]; } } @@ -146,8 +149,8 @@ class Subscribe extends Action // registry is keyed by (connId, subId, projectId): re-subscribing the same channel // overwrites its filter in place, while a reused subscription id that names a new // tail channel keeps the ones already registered (mirrors the tree). - foreach ($pendingTails as [$subId, $targetProjectId, $compiled]) { - $eventTailRegistry->add($connectionId, $subId, $targetProjectId, $compiled, $now); + foreach ($pendingTails as [$subId, $targetProjectId, $compiled, $role]) { + $eventTailRegistry->add($connectionId, $subId, $targetProjectId, $compiled, $now, $role); } $subscriptionsAfter = \count($realtime->getSubscriptionMetadata($connectionId)); diff --git a/tests/unit/Realtime/EventTailRegistryTest.php b/tests/unit/Realtime/EventTailRegistryTest.php index 6d3da66ba0..f41c8e1365 100644 --- a/tests/unit/Realtime/EventTailRegistryTest.php +++ b/tests/unit/Realtime/EventTailRegistryTest.php @@ -535,4 +535,47 @@ final class EventTailRegistryTest extends TestCase $this->assertContains('console.tail.projA', $channels, 'projA stopped delivering after projB was added'); $this->assertContains('console.tail.projB', $channels); } + + public function testRateClampedToAtLeastOne(): void + { + // A misconfigured non-positive rate must not disable the tail entirely. + foreach ([0, -5] as $badRate) { + $registry = new EventTailRegistry(rate: $badRate); + $registry->add(1, 'x', 'projA', $this->compile([]), 0.0); + $registry->ingest('projA', ['type' => 'databases'], 0.0); + + $sent = []; + $registry->flush($this->server($sent)); + + $eventFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail'])); + $this->assertCount(1, $eventFrames, "rate {$badRate} must clamp to >=1 so events still flow"); + $this->assertCount(1, $eventFrames[0]['data']['payload']); + } + } + + public function testRevalidateConnectionDropsTailsWithRevokedRole(): void + { + $registry = new EventTailRegistry(rate: 100); + $registry->add(1, 'x', 'projA', $this->compile([]), 0.0, 'team:teamA'); + $registry->add(1, 'y', 'projB', $this->compile([]), 0.0, 'team:teamB'); + $this->assertTrue($registry->isTailed('projA')); + $this->assertTrue($registry->isTailed('projB')); + + // The connection keeps teamB membership but loses teamA. + $registry->revalidateConnection(1, ['user:u1', 'team:teamB']); + + $this->assertFalse($registry->isTailed('projA'), 'tail for the revoked team must be dropped'); + $this->assertTrue($registry->isTailed('projB'), 'tail for the still-held team must remain'); + } + + public function testRevalidateConnectionKeepsEntriesWithoutRole(): void + { + // Entries registered without an authorizing role are never pruned (defensive). + $registry = new EventTailRegistry(rate: 100); + $registry->add(1, 'x', 'projA', $this->compile([]), 0.0); + + $registry->revalidateConnection(1, ['user:u1']); + + $this->assertTrue($registry->isTailed('projA')); + } }