diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php b/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php index 13a7d549dc..3b1f8c6e54 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php @@ -13,6 +13,7 @@ use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; use WebSocket\Client as WebSocketClient; +use WebSocket\ConnectionException; use WebSocket\TimeoutException; final class RealtimeCustomClientQueryTestWithMessage extends Scope @@ -810,4 +811,230 @@ final class RealtimeCustomClientQueryTestWithMessage extends Scope $client->close(); } + + /** + * Create a database + collection (Role::any) with a string attribute, BEFORE the + * tail connection opens, so the setup events don't appear in the tail. + * + * @return array{databaseId:string, collectionId:string} + */ + private function setupTailCollection(): array + { + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]; + + $database = $this->client->call(Client::METHOD_POST, '/databases', $headers, [ + 'databaseId' => ID::unique(), + 'name' => 'Tail DB', + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', $headers, [ + 'collectionId' => ID::unique(), + 'name' => 'Actors', + 'permissions' => [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', $headers, [ + 'key' => 'name', + 'size' => 256, + 'required' => true, + ]); + + $this->assertEventually(function () use ($databaseId, $collectionId, $headers) { + $attribute = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/name', $headers); + $this->assertEquals(200, $attribute['headers']['status-code']); + $this->assertEquals('available', $attribute['body']['status']); + }, 120000, 500); + + return ['databaseId' => $databaseId, 'collectionId' => $collectionId]; + } + + /** + * Open a console connection (root session) in message mode and subscribe to a + * `console.tail.` channel with optional filter queries. + * + * @param array $queries + */ + private function openConsoleTail(string $targetProjectId, array $queries = [], string $subscriptionId = 'tail-1'): WebSocketClient + { + $queryString = \http_build_query(['project' => 'console']); + $client = new WebSocketClient( + 'ws://appwrite.test/v1/realtime?' . $queryString, + [ + 'headers' => [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + ], + 'timeout' => 10, + ] + ); + + $connected = \json_decode($client->receive(), true); + $this->assertEquals('connected', $connected['type'] ?? null); + + $sub = ['channels' => ['console.tail.' . $targetProjectId], 'subscriptionId' => $subscriptionId]; + if (!empty($queries)) { + $sub['queries'] = $queries; + } + + $client->send(\json_encode(['type' => 'subscribe', 'data' => [$sub]])); + + $response = \json_decode($client->receive(), true); + $this->assertEquals('response', $response['type'] ?? null); + $this->assertEquals('subscribe', $response['data']['to'] ?? null); + $this->assertTrue($response['data']['success'] ?? false); + + return $client; + } + + /** + * Receive frames until a `console.tail` event arrives, returning its payload (an + * array of compact frames). Skips the periodic `console.tail.stats` counter frames. + * + * @return array> + */ + private function receiveTailFrames(WebSocketClient $client, int $maxFrames = 5): array + { + for ($i = 0; $i < $maxFrames; $i++) { + $message = \json_decode($client->receive(), true); + $this->assertEquals('event', $message['type']); + if (($message['data']['events'][0] ?? null) === 'console.tail') { + return $message['data']['payload']; + } + } + $this->fail('No console.tail event frame received.'); + } + + public function testConsoleTailReceivesCompactMetadata(): void + { + ['databaseId' => $databaseId, 'collectionId' => $collectionId] = $this->setupTailCollection(); + + $projectId = $this->getProject()['$id']; + $client = $this->openConsoleTail($projectId); + + $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'documentId' => ID::unique(), + 'data' => ['name' => 'Spiderman'], + ]); + $this->assertEquals(201, $document['headers']['status-code']); + $documentId = $document['body']['$id']; + + $frames = $this->receiveTailFrames($client); + + $match = null; + foreach ($frames as $frame) { + if (($frame['resourceId'] ?? null) === $documentId) { + $match = $frame; + break; + } + } + + $this->assertNotNull($match, 'document.create not present in tail: ' . \json_encode($frames)); + $this->assertSame('databases', $match['type']); + $this->assertSame('create', $match['action']); + $this->assertSame($databaseId, $match['databaseId']); + $this->assertSame($collectionId, $match['collectionId']); + // Compact: no full document body, and no scope ids from other resource types. + $this->assertArrayNotHasKey('name', $match); + $this->assertArrayNotHasKey('functionId', $match); + $this->assertArrayNotHasKey('bucketId', $match); + + $client->close(); + } + + public function testConsoleTailServerSideFilter(): void + { + ['databaseId' => $databaseId, 'collectionId' => $collectionId] = $this->setupTailCollection(); + + $projectId = $this->getProject()['$id']; + // Only deletes should reach this tail. + $client = $this->openConsoleTail($projectId, [Query::equal('action', ['delete'])->toString()]); + + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]; + + // create — must be filtered out (never reaches the token bucket). + $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers, [ + 'documentId' => ID::unique(), + 'data' => ['name' => 'Ironman'], + ]); + $this->assertEquals(201, $document['headers']['status-code']); + $documentId = $document['body']['$id']; + + // delete — must pass the filter. + $deleted = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $documentId, $headers); + $this->assertEquals(204, $deleted['headers']['status-code']); + + $frames = $this->receiveTailFrames($client); + + foreach ($frames as $frame) { + $this->assertSame('delete', $frame['action'], 'filter leaked a non-delete event: ' . \json_encode($frame)); + } + $deleteFrames = \array_filter($frames, fn ($f) => ($f['resourceId'] ?? null) === $documentId); + $this->assertNotEmpty($deleteFrames, 'expected the delete event in the tail'); + + $client->close(); + } + + public function testConsoleTailUnauthorizedProjectIsRejected(): void + { + $queryString = \http_build_query(['project' => 'console']); + $client = new WebSocketClient( + 'ws://appwrite.test/v1/realtime?' . $queryString, + [ + 'headers' => [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + ], + 'timeout' => 10, + ] + ); + + $connected = \json_decode($client->receive(), true); + $this->assertEquals('connected', $connected['type'] ?? null); + + // A project the console user's team does not own (does not exist) → policy violation. + $client->send(\json_encode([ + 'type' => 'subscribe', + 'data' => [['channels' => ['console.tail.nonexistentproject'], 'subscriptionId' => 'tail-x']], + ])); + + // A 1008 policy violation sends the error frame and then closes the connection. + // Depending on flush/close timing the client may read the error frame, or the + // socket may already be dead — both are valid rejections. What must NOT happen + // is a successful `response` subscription. + try { + $response = \json_decode($client->receive(), true); + $this->assertEquals('error', $response['type'] ?? null); + $this->assertEquals(1008, $response['data']['code'] ?? null); + } catch (ConnectionException $e) { + $this->addToAssertionCount(1); + } + + // close() can itself throw on an already-closed socket — best effort. + try { + $client->close(); + } catch (ConnectionException $e) { + // already closed by the server + } + } } diff --git a/tests/unit/Realtime/EventTailRegistryTest.php b/tests/unit/Realtime/EventTailRegistryTest.php new file mode 100644 index 0000000000..3f6160142d --- /dev/null +++ b/tests/unit/Realtime/EventTailRegistryTest.php @@ -0,0 +1,366 @@ +> $sent + */ + private function server(array &$sent): Server + { + // A stub (not a mock): we only capture sends, no expectations to verify. + $server = $this->createStub(Server::class); + $server->method('send')->willReturnCallback( + function (array $connections, string $message) use (&$sent): void { + $sent[] = json_decode($message, true); + } + ); + return $server; + } + + /** @param array $payload */ + private function compile(array $queries): array + { + return RuntimeQuery::compile($queries); + } + + // ---- Realtime::toTailMetadata ---- + + public function testTailMetadataDocumentEvent(): void + { + $event = [ + 'userId' => 'user-1', + 'data' => [ + 'events' => ['databases.main.collections.posts.documents.p1.create'], + 'timestamp' => '2026-06-06T00:00:00.000+00:00', + 'payload' => ['$id' => 'p1', '$databaseId' => 'main', '$collectionId' => 'posts', 'title' => 'hello'], + ], + ]; + + $meta = Realtime::toTailMetadata($event); + + $this->assertSame('databases.main.collections.posts.documents.p1.create', $meta['event']); + $this->assertSame('databases', $meta['type']); + $this->assertSame('create', $meta['action']); + $this->assertSame('user-1', $meta['userId']); + $this->assertSame('p1', $meta['resourceId']); + $this->assertSame('main', $meta['databaseId']); + $this->assertSame('posts', $meta['collectionId']); + // No full document body leaks through. + $this->assertArrayNotHasKey('title', $meta); + // Scope is type-specific: a databases event carries no bucket/function ids. + $this->assertArrayNotHasKey('bucketId', $meta); + $this->assertArrayNotHasKey('functionId', $meta); + } + + public function testTailMetadataScopeIsTypeSpecific(): void + { + $fn = Realtime::toTailMetadata([ + 'userId' => 'u1', + 'data' => [ + 'events' => ['functions.fn1.deployments.d1.create'], + 'payload' => ['$id' => 'd1', 'functionId' => 'fn1'], + ], + ]); + $this->assertSame('functions', $fn['type']); + $this->assertSame('fn1', $fn['functionId']); + $this->assertArrayNotHasKey('databaseId', $fn); + $this->assertArrayNotHasKey('bucketId', $fn); + + $file = Realtime::toTailMetadata([ + 'userId' => 'u1', + 'data' => [ + 'events' => ['buckets.b1.files.f1.create'], + 'payload' => ['$id' => 'f1', 'bucketId' => 'b1'], + ], + ]); + $this->assertSame('buckets', $file['type']); + $this->assertSame('b1', $file['bucketId']); + $this->assertArrayNotHasKey('functionId', $file); + $this->assertArrayNotHasKey('collectionId', $file); + + // A null scope id is omitted rather than emitted. + $fnNoScope = Realtime::toTailMetadata([ + 'data' => ['events' => ['functions.fn1.executions.e1.create'], 'payload' => ['$id' => 'e1']], + ]); + $this->assertArrayNotHasKey('functionId', $fnNoScope); + } + + public function testTailMetadataTablesUsesTableIdAsCollection(): void + { + $meta = Realtime::toTailMetadata([ + 'userId' => 'u1', + 'data' => [ + 'events' => ['databases.main.tables.t1.rows.r1.create'], + 'payload' => ['$id' => 'r1', '$databaseId' => 'main', '$tableId' => 't1'], + ], + ]); + $this->assertSame('databases', $meta['type']); + $this->assertSame('main', $meta['databaseId']); + $this->assertSame('t1', $meta['collectionId']); + } + + public function testTailMetadataAttributeTrailingAction(): void + { + // `users.U.update.email` — action is the second-to-last segment. + $meta = Realtime::toTailMetadata([ + 'userId' => 'u9', + 'data' => ['events' => ['users.u9.update.email'], 'payload' => ['$id' => 'u9']], + ]); + + $this->assertSame('users', $meta['type']); + $this->assertSame('update', $meta['action']); + $this->assertSame('u9', $meta['resourceId']); + } + + public function testTailMetadataHandlesMissingPayload(): void + { + $meta = Realtime::toTailMetadata(['data' => ['events' => ['functions.f1.executions.e1.delete']]]); + + $this->assertSame('functions', $meta['type']); + $this->assertSame('delete', $meta['action']); + $this->assertNull($meta['userId']); + $this->assertNull($meta['resourceId']); + } + + // ---- channel helpers ---- + + public function testChannelHelpers(): void + { + $this->assertSame('console.tail.proj1', EventTailRegistry::channel('proj1')); + $this->assertSame('proj1', EventTailRegistry::projectFromChannel('console.tail.proj1')); + $this->assertNull(EventTailRegistry::projectFromChannel('documents')); + $this->assertNull(EventTailRegistry::projectFromChannel('console.tail.')); + } + + // ---- filter THEN sample ---- + + public function testFilterIsAppliedBeforeSampling(): void + { + $registry = new EventTailRegistry(rate: 100); + $registry->add(1, 'sub-a', 'projX', $this->compile([Query::equal('type', ['databases'])]), 0.0); + + $registry->ingest('projX', ['type' => 'databases', 'action' => 'create'], 0.0); + $registry->ingest('projX', ['type' => 'buckets', 'action' => 'create'], 0.0); + + $sent = []; + $registry->flush($this->server($sent)); + + // Exactly one event frame, carrying only the databases frame; no stats frame + // because the filtered-out bucket event never consumed a token. + $this->assertCount(1, $sent); + $this->assertSame(['console.tail'], $sent[0]['data']['events']); + $this->assertCount(1, $sent[0]['data']['payload']); + $this->assertSame('databases', $sent[0]['data']['payload'][0]['type']); + } + + public function testTokenBucketDropsExcessAndEmitsCounter(): void + { + $registry = new EventTailRegistry(rate: 5); + $registry->add(1, 'sub-a', 'projX', $this->compile([]), 0.0); // selectAll + + for ($i = 0; $i < 20; $i++) { + $registry->ingest('projX', ['type' => 'databases', 'seq' => $i], 0.0); + } + + $sent = []; + $registry->flush($this->server($sent)); + + $eventFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail'])); + $statsFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail.stats'])); + + $this->assertCount(1, $eventFrames); + $this->assertCount(5, $eventFrames[0]['data']['payload'], 'rate=5 → only 5 delivered at the same instant'); + + $this->assertCount(1, $statsFrames); + $this->assertSame('tail.stats', $statsFrames[0]['data']['payload']['$type']); + $this->assertSame(5, $statsFrames[0]['data']['payload']['delivered']); + $this->assertSame(15, $statsFrames[0]['data']['payload']['dropped']); + // delivered + dropped accounts for every ingested event. + $this->assertSame(20, $statsFrames[0]['data']['payload']['delivered'] + $statsFrames[0]['data']['payload']['dropped']); + } + + public function testNoCounterFrameWhenNothingDropped(): void + { + $registry = new EventTailRegistry(rate: 100); + $registry->add(1, 'sub-a', 'projX', $this->compile([]), 0.0); + + $registry->ingest('projX', ['type' => 'databases'], 0.0); + + $sent = []; + $registry->flush($this->server($sent)); + + // Only the event frame — no stats frame because nothing was dropped. + $this->assertCount(1, $sent); + $this->assertSame(['console.tail'], $sent[0]['data']['events']); + } + + public function testDroppedAccumulatesAcrossIngestsInWindow(): void + { + $registry = new EventTailRegistry(rate: 2); + $registry->add(1, 'sub-a', 'projX', $this->compile([]), 0.0); + + // Three separate ingests at the same instant: 2 delivered, 1 dropped. + $registry->ingest('projX', ['type' => 'databases', 'seq' => 1], 0.0); + $registry->ingest('projX', ['type' => 'databases', 'seq' => 2], 0.0); + $registry->ingest('projX', ['type' => 'databases', 'seq' => 3], 0.0); + + $sent = []; + $registry->flush($this->server($sent)); + + $statsFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail.stats'])); + $this->assertCount(1, $statsFrames); + $this->assertSame(2, $statsFrames[0]['data']['payload']['delivered']); + $this->assertSame(1, $statsFrames[0]['data']['payload']['dropped']); + } + + public function testCounterResetsBetweenWindows(): void + { + $registry = new EventTailRegistry(rate: 1); + $registry->add(1, 'sub-a', 'projX', $this->compile([]), 0.0); + + // Window 1: 3 events at t=0 → 1 delivered, 2 dropped → stats frame emitted. + for ($i = 0; $i < 3; $i++) { + $registry->ingest('projX', ['type' => 'databases', 'seq' => $i], 0.0); + } + $sent = []; + $registry->flush($this->server($sent)); + $statsFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail.stats'])); + $this->assertCount(1, $statsFrames); + $this->assertSame(2, $statsFrames[0]['data']['payload']['dropped']); + + // Window 2: a single event well after refill → delivered, nothing dropped → no stats frame + // and the previous window's dropped count does not carry over. + $registry->ingest('projX', ['type' => 'databases'], 100.0); + $sent = []; + $registry->flush($this->server($sent)); + $statsFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail.stats'])); + $this->assertCount(0, $statsFrames); + } + + public function testLargeBufferIsChunkedAcrossFrames(): void + { + // rate high enough to admit all, batchMax forces splitting on flush. + $registry = new EventTailRegistry(rate: 100, batchMax: 2); + $registry->add(1, 'sub-a', 'projX', $this->compile([]), 0.0); + + for ($i = 0; $i < 5; $i++) { + $registry->ingest('projX', ['type' => 'databases', 'seq' => $i], 0.0); + } + + $sent = []; + $registry->flush($this->server($sent)); + + $eventFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail'])); + // 5 buffered frames, batchMax 2 → 3 websocket frames (2 + 2 + 1). + $this->assertCount(3, $eventFrames); + $this->assertCount(2, $eventFrames[0]['data']['payload']); + $this->assertCount(2, $eventFrames[1]['data']['payload']); + $this->assertCount(1, $eventFrames[2]['data']['payload']); + // No event is lost across the chunk boundaries. + $total = array_sum(array_map(fn ($f) => count($f['data']['payload']), $eventFrames)); + $this->assertSame(5, $total); + } + + public function testTokensRefillOverTime(): void + { + $registry = new EventTailRegistry(rate: 5); + $registry->add(1, 'sub-a', 'projX', $this->compile([]), 0.0); + + // Drain the bucket at t=0. + for ($i = 0; $i < 10; $i++) { + $registry->ingest('projX', ['type' => 'databases'], 0.0); + } + $sent = []; + $registry->flush($this->server($sent)); // resets the per-window counters + + // One second later, the bucket has refilled to the cap (rate=5). + for ($i = 0; $i < 10; $i++) { + $registry->ingest('projX', ['type' => 'databases'], 1.0); + } + $sent = []; + $registry->flush($this->server($sent)); + + $eventFrames = array_values(array_filter($sent, fn ($f) => $f['data']['events'] === ['console.tail'])); + $this->assertCount(5, $eventFrames[0]['data']['payload']); + } + + public function testFlushClearsBufferBetweenWindows(): void + { + $registry = new EventTailRegistry(rate: 100); + $registry->add(1, 'sub-a', 'projX', $this->compile([]), 0.0); + $registry->ingest('projX', ['type' => 'databases'], 0.0); + + $sent = []; + $registry->flush($this->server($sent)); + $this->assertCount(1, $sent); + + // Nothing new ingested → second flush sends nothing. + $sent = []; + $registry->flush($this->server($sent)); + $this->assertCount(0, $sent); + } + + public function testRemoveConnectionStopsDelivery(): void + { + $registry = new EventTailRegistry(rate: 100); + $registry->add(7, 'sub-a', 'projX', $this->compile([]), 0.0); + $registry->add(7, 'sub-b', 'projY', $this->compile([]), 0.0); + + $this->assertTrue($registry->isTailed('projX')); + $this->assertTrue($registry->isTailed('projY')); + + $registry->removeConnection(7); + + $this->assertFalse($registry->isTailed('projX')); + $this->assertFalse($registry->isTailed('projY')); + + $registry->ingest('projX', ['type' => 'databases'], 0.0); + $sent = []; + $registry->flush($this->server($sent)); + $this->assertCount(0, $sent); + } + + public function testRemoveSingleSubscription(): void + { + $registry = new EventTailRegistry(rate: 100); + $registry->add(7, 'sub-a', 'projX', $this->compile([]), 0.0); + $registry->add(7, 'sub-b', 'projX', $this->compile([]), 0.0); + + $registry->remove('sub-a'); + $this->assertTrue($registry->isTailed('projX'), 'sub-b still tails projX'); + + $registry->remove('sub-b'); + $this->assertFalse($registry->isTailed('projX')); + + // Removing an unknown subscription is a no-op. + $registry->remove('does-not-exist'); + $this->assertFalse($registry->isTailed('projX')); + } + + public function testTeamCache(): void + { + $registry = new EventTailRegistry(); + $this->assertNull($registry->cachedTeam('projX')); + $registry->cacheTeam('projX', 'team-1'); + $this->assertSame('team-1', $registry->cachedTeam('projX')); + } +}