refactor(mqtt): consume the utopia-php/mqtt Server/Handler deep-module API

Collapse the per-packet handlers, Dispatcher and Response into a single
Appwrite\Mqtt\Handler that implements the library Handler interface; the
Server now owns framing, dispatch, encoding, keep-alive and the subscription
store. Resolve broker resources (cache, console/project DB, redis, plan)
through the container instead of global functions. Honour each topic's
expiry as the replay-cursor TTL (PUBACK preserves it with a 0-ttl save
rather than shrinking it to the default hour), and clamp delivery QoS to the
subscriber's granted QoS. Drop the adapter unit test whose registry/store
behaviour now lives in the library.
This commit is contained in:
ArnabChatterjee20k committed 2026-09-17 21:15:29 +05:30
1 parent 7015b55c5d
commit e8932bed7d
16 files changed
+500 -1320

No files matched your search

+6 -4
View File
@@ -13,25 +13,27 @@ use Utopia\DI\Container;
use Utopia\System\System;
return function (Container $container): void {
$getProject = function (string $projectId, Authorization $authorization): Document {
$getProject = function (string $projectId, Authorization $authorization) use ($container): Document {
if ($projectId === '' || $projectId === 'console') {
return new Document(Config::getParam('console'));
}
$dbForPlatform = getConsoleDB();
$getConsoleDB = $container->get('getConsoleDB');
$dbForPlatform = $getConsoleDB();
$dbForPlatform->setAuthorization($authorization);
return $authorization->skip(fn () => $dbForPlatform->getDocument('projects', $projectId));
};
$getUser = function (Document $project, string $authMethod, string $credential, Authorization $authorization): User {
$getUser = function (Document $project, string $authMethod, string $credential, Authorization $authorization) use ($container): User {
if ($project->isEmpty() || $credential === '') {
return new User([]);
}
$authorization->setDefaultStatus(true);
$dbForProject = getProjectDB($project);
$getProjectDB = $container->get('getProjectDB');
$dbForProject = $getProjectDB($project);
$dbForProject->setAuthorization($authorization);
switch ($authMethod) {
+116 -277
View File
@@ -1,20 +1,11 @@
<?php
use Appwrite\Messaging\Adapter\Mqtt;
use Appwrite\Mqtt\Dispatcher;
use Appwrite\Mqtt\Handlers\Auth as AuthHandler;
use Appwrite\Mqtt\Handlers\Connect as ConnectHandler;
use Appwrite\Mqtt\Handlers\Disconnect as DisconnectHandler;
use Appwrite\Mqtt\Handlers\Ping as PingHandler;
use Appwrite\Mqtt\Handlers\Puback as PubackHandler;
use Appwrite\Mqtt\Handlers\Subscribe as SubscribeHandler;
use Appwrite\Mqtt\Handlers\Unsubscribe as UnsubscribeHandler;
use Appwrite\Mqtt\Response;
use Appwrite\Mqtt\Handler;
use Appwrite\PubSub\Adapter\Pool as PubSubPool;
use Appwrite\Utopia\Database\Documents\User;
use Swoole\Coroutine;
use Swoole\Runtime;
use Swoole\Timer;
use Utopia\Cache\Adapter\Pool as CachePool;
use Utopia\Cache\Adapter\Sharding;
use Utopia\Cache\Cache;
@@ -26,10 +17,6 @@ use Utopia\Database\Document;
use Utopia\DI\Container;
use Utopia\DSN\DSN;
use Utopia\Mqtt\Adapter;
use Utopia\Mqtt\Keepalive;
use Utopia\Mqtt\Packet;
use Utopia\Mqtt\Packet\V3;
use Utopia\Mqtt\Packet\V5;
use Utopia\Mqtt\Server;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
@@ -55,170 +42,129 @@ if (!$container->has('pools')) {
}, ['register']);
}
if (!function_exists('getCache')) {
function getCache(): Cache
{
$ctx = Coroutine::getContext();
$container->set('getCache', fn () => function () use ($register): Cache {
$ctx = Coroutine::getContext();
if (isset($ctx['cache'])) {
return $ctx['cache'];
}
global $register;
/** @var Group $pools */
$pools = $register->get('pools');
$adapters = [];
foreach (Config::getParam('pools-cache', []) as $value) {
$adapters[] = new CachePool($pools->get($value));
}
return $ctx['cache'] = new Cache(new Sharding($adapters));
if (isset($ctx['cache'])) {
return $ctx['cache'];
}
}
if (!function_exists('getConsoleDB')) {
function getConsoleDB(): Database
{
$ctx = Coroutine::getContext();
/** @var Group $pools */
$pools = $register->get('pools');
if (isset($ctx['dbForPlatform'])) {
return $ctx['dbForPlatform'];
}
global $register;
/** @var Group $pools */
$pools = $register->get('pools');
$adapter = new DatabasePool($pools->get('console'));
$database = new Database($adapter, getCache());
$database
->setDatabase(APP_DATABASE)
->setNamespace('_console')
->setMetadata('host', \gethostname())
->setMetadata('project', '_console');
$database->setDocumentType('users', User::class);
return $ctx['dbForPlatform'] = $database;
$adapters = [];
foreach (Config::getParam('pools-cache', []) as $value) {
$adapters[] = new CachePool($pools->get($value));
}
}
if (!function_exists('getProjectDB')) {
function getProjectDB(Document $project): Database
{
$ctx = Coroutine::getContext();
return $ctx['cache'] = new Cache(new Sharding($adapters));
}, []);
if (!isset($ctx['dbForProject'])) {
$ctx['dbForProject'] = [];
}
$container->set('getConsoleDB', fn () => function () use ($register, $container): Database {
$ctx = Coroutine::getContext();
if (isset($ctx['dbForProject'][$project->getSequence()])) {
return $ctx['dbForProject'][$project->getSequence()];
}
if (isset($ctx['dbForPlatform'])) {
return $ctx['dbForPlatform'];
}
if ($project->isEmpty() || $project->getId() === 'console') {
return getConsoleDB();
}
$getCache = $container->get('getCache');
global $register;
/** @var Group $pools */
$pools = $register->get('pools');
/** @var Group $pools */
$pools = $register->get('pools');
$adapter = new DatabasePool($pools->get('console'));
$database = new Database($adapter, $getCache());
$database
->setDatabase(APP_DATABASE)
->setNamespace('_console')
->setMetadata('host', \gethostname())
->setMetadata('project', '_console');
$database->setDocumentType('users', User::class);
try {
$dsn = new DSN($project->getAttribute('database'));
} catch (\InvalidArgumentException) {
$dsn = new DSN('mysql://' . $project->getAttribute('database'));
}
return $ctx['dbForPlatform'] = $database;
}, []);
$adapter = new DatabasePool($pools->get($dsn->getHost()));
$database = new Database($adapter, getCache());
$container->set('getProjectDB', fn () => function (Document $project) use ($register, $container): Database {
$ctx = Coroutine::getContext();
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (!isset($ctx['dbForProject'])) {
$ctx['dbForProject'] = [];
}
if (\in_array($dsn->getHost(), $sharedTables)) {
$projectCollections = Config::getParam('collections', [])['projects'] ?? [];
$globalCollections = array_keys($projectCollections);
$globalCollections[] = 'audit';
if (isset($ctx['dbForProject'][$project->getSequence()])) {
return $ctx['dbForProject'][$project->getSequence()];
}
$database
->setSharedTables(true)
->setGlobalCollections($globalCollections)
->setTenant($project->getSequence())
->setNamespace($dsn->getParam('namespace'));
} else {
$database
->setSharedTables(false)
->setTenant(null)
->setNamespace('_' . $project->getSequence());
}
if ($project->isEmpty() || $project->getId() === 'console') {
$getConsoleDB = $container->get('getConsoleDB');
return $getConsoleDB();
}
$getCache = $container->get('getCache');
/** @var Group $pools */
$pools = $register->get('pools');
try {
$dsn = new DSN($project->getAttribute('database'));
} catch (\InvalidArgumentException) {
$dsn = new DSN('mysql://' . $project->getAttribute('database'));
}
$adapter = new DatabasePool($pools->get($dsn->getHost()));
$database = new Database($adapter, $getCache());
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
if (\in_array($dsn->getHost(), $sharedTables)) {
$projectCollections = Config::getParam('collections', [])['projects'] ?? [];
$globalCollections = array_keys($projectCollections);
$globalCollections[] = 'audit';
$database
->setDatabase(APP_DATABASE)
->setMetadata('host', \gethostname())
->setMetadata('project', $project->getId());
$database->setDocumentType('users', User::class);
return $ctx['dbForProject'][$project->getSequence()] = $database;
->setSharedTables(true)
->setGlobalCollections($globalCollections)
->setTenant($project->getSequence())
->setNamespace($dsn->getParam('namespace'));
} else {
$database
->setSharedTables(false)
->setTenant(null)
->setNamespace('_' . $project->getSequence());
}
}
if (!function_exists('getPlanForUser')) {
function getPlanForUser(Document $project, string $userId): int
{
return (int) System::getEnv('_APP_MQTT_REPLAY_DEPTH', '5');
$database
->setDatabase(APP_DATABASE)
->setMetadata('host', \gethostname())
->setMetadata('project', $project->getId());
$database->setDocumentType('users', User::class);
return $ctx['dbForProject'][$project->getSequence()] = $database;
}, []);
$container->set('getPlanForUser', fn () => fn (Document $project, string $userId): int => (int) System::getEnv('_APP_MQTT_REPLAY_DEPTH', '5'), []);
$container->set('getRedis', fn () => function (): \Redis {
$ctx = Coroutine::getContext();
if (isset($ctx['redis'])) {
return $ctx['redis'];
}
}
if (!function_exists('getCache')) {
function getCache(): Cache
{
$ctx = Coroutine::getContext();
$host = System::getEnv('_APP_REDIS_HOST', 'localhost');
$port = System::getEnv('_APP_REDIS_PORT', 6379);
$pass = System::getEnv('_APP_REDIS_PASS', '');
if (isset($ctx['cache'])) {
return $ctx['cache'];
}
global $register;
$pools = $register->get('pools'); /** @var Group $pools */
$list = Config::getParam('pools-cache', []);
$adapters = [];
foreach ($list as $value) {
$adapters[] = new CachePool($pools->get($value));
}
return $ctx['cache'] = new Cache(new Sharding($adapters));
$redis = new \Redis();
@$redis->pconnect($host, (int)$port);
if ($pass) {
$redis->auth($pass);
}
}
$redis->setOption(\Redis::OPT_READ_TIMEOUT, -1);
if (!function_exists('getRedis')) {
function getRedis(): \Redis
{
$ctx = Coroutine::getContext();
if (isset($ctx['redis'])) {
return $ctx['redis'];
}
$host = System::getEnv('_APP_REDIS_HOST', 'localhost');
$port = System::getEnv('_APP_REDIS_PORT', 6379);
$pass = System::getEnv('_APP_REDIS_PASS', '');
$redis = new \Redis();
@$redis->pconnect($host, (int)$port);
if ($pass) {
$redis->auth($pass);
}
$redis->setOption(\Redis::OPT_READ_TIMEOUT, -1);
return $ctx['redis'] = $redis;
}
}
return $ctx['redis'] = $redis;
}, []);
// Register the CONNECT authenticator and per-SUBSCRIBE authorizer on the global container
// (see app/init/mqtt/connection.php). The CONNECT/SUBSCRIBE handlers inject them by name,
@@ -228,49 +174,25 @@ $registerConnectionResources($container);
/** @var \Utopia\Telemetry\Adapter $telemetry */
$telemetry = $container->get('telemetry');
$adapter = new Adapter\Swoole(host: '0.0.0.0', port: 1883);
$adapter->setPackageMaxLength((int) System::getEnv('_APP_MQTT_MAX_PACKET_SIZE', '64000'));
$server = new Server($adapter);
$server->error(fn (\Throwable $error, string $action) => Console::error("MQTT {$action} error: " . $error->getMessage()));
$adapter = new Adapter\Swoole([
new Adapter\Swoole\Tcp('0.0.0.0', 1883, (int) System::getEnv('_APP_MQTT_MAX_PACKET_SIZE', '64000')),
], workers: 1);
$mqtt = new Mqtt($telemetry, new PubSubPool($register->get('pools')->get('pubsub')));
$dispatcher = new Dispatcher([
new ConnectHandler(),
new SubscribeHandler(),
new UnsubscribeHandler(),
new PubackHandler(),
new AuthHandler(),
new PingHandler(),
new DisconnectHandler(),
]);
// The broker owns framing, decoding, dispatch, per-version encoding, packet ids, the QoS
// handshake, keep-alive reaping and subscription matching. Appwrite policy lives in the Handler.
$handler = new Handler($container, $mqtt);
$server = new Server($adapter, $handler);
$server->setTelemetry($telemetry); // broker owns connection/packet/subscription metrics
$server->onStart(fn () => print("MQTT broker started\n"));
$server->error(fn (\Throwable $error, string $action) => Console::error("MQTT {$action} error: " . $error->getMessage()));
$server->onWorkerStart(function (int $workerId) use ($server, $mqtt, $register): void {
// Keep-alive reaper: every INTERVAL seconds drain the wheel's due buckets and close any
// client silent past its deadline (keepAlive x MULTIPLIER). A survivor whose deadline was
// pushed forward by recent traffic is rescheduled instead. onClose balances the gauge and
// forgets the fd. Per worker, since connections and the wheel are per worker.
Timer::tick($mqtt->keepAlive->interval * 1000, function () use ($server, $mqtt): void {
$now = microtime(true);
foreach ($mqtt->keepAlive->drain((int) $now) as $fd) {
$connection = $mqtt->connections[$fd] ?? null;
if ($connection === null || $connection->keepAlive <= 0) {
continue;
}
if ($connection->expiresAt <= $now) {
$server->close($fd);
} else {
$connection->wheelSlot = $mqtt->keepAlive->schedule($fd, $connection->expiresAt);
}
}
});
go(function () use ($server, $mqtt, $register): void {
// Server-initiated delivery: bridge the Redis 'mqtt' firehose to this worker's local subscribers.
// Appwrite clients never PUBLISH; messages are produced by the Messaging worker onto the channel.
$server->onWorkerStart(function (int $workerId) use ($server, $handler, $mqtt, $register): void {
go(function () use ($server, $handler, $mqtt, $register): void {
$attempts = 0;
while ($attempts < 300) {
try {
@@ -280,7 +202,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $mqtt, $register):
$attempts = 0;
}
$pubsub->subscribe(['mqtt'], function (mixed $redis, string $channel, string $payload) use ($server, $mqtt): void {
$pubsub->subscribe(['mqtt'], function (mixed $redis, string $channel, string $payload) use ($server, $handler, $mqtt): void {
$event = json_decode($payload, true);
if (!\is_array($event)) {
return;
@@ -292,43 +214,17 @@ $server->onWorkerStart(function (int $workerId) use ($server, $mqtt, $register):
$sequence = (int) ($event['sequence'] ?? 0);
$message = base64_decode((string) ($event['payload'] ?? ''));
// The broker is the sender on this hop (fan-out to subscribers), so the
// span is marked is_broker to separate it from client-originated packets.
$span = Span::init('mqtt.deliver');
$span->set('project.id', $projectId);
$span->set('mqtt.topic', $topic);
$span->set('mqtt.qos', $qos);
$span->set('mqtt.is_broker', true);
$subscribers = $mqtt->getSubscribers($projectId, $topic);
$span->set('mqtt.subscribers', count($subscribers));
$delivered = $handler->deliver($server, $projectId, $topic, $message, $qos, $sequence);
if ($subscribers === []) {
// No local subscriber on this worker; with multiple workers this
// counts per-worker rather than as a global drop.
$span->set('mqtt.subscribers', $delivered);
if ($delivered === 0) {
$mqtt->messagesDropped->add(1, ['reason' => 'no_subscriber']);
$span->finish();
return;
}
foreach ($subscribers as $fd => $grantedQos) {
$subscriber = $mqtt->connections[$fd] ?? null;
if ($subscriber === null) {
continue;
}
$effectiveQos = min($qos, $grantedQos);
$packetId = $subscriber->nextPacketId();
$publish = $subscriber->protocol >= 5
? V5::publish($topic, $message, $effectiveQos, $packetId)
: V3::publish($topic, $message, $effectiveQos, $packetId);
$server->send($fd, $publish);
$mqtt->bytesSent->add(\strlen($publish));
$mqtt->messagesDelivered->add(1, ['qos' => $effectiveQos]);
// Hold QoS 1 deliveries until the subscriber's PUBACK matches them back.
if ($effectiveQos === 1) {
$subscriber->track($packetId, $topic, $sequence);
}
}
$span->finish();
@@ -344,61 +240,4 @@ $server->onWorkerStart(function (int $workerId) use ($server, $mqtt, $register):
});
});
$server->onReceive(function (int $fd, string $data) use (
$server,
$mqtt,
$dispatcher,
$container
): void {
// The whole packet lifecycle is wrapped: a malformed packet or a handler failure must
// drop only this connection, never bubble up and take down the worker (and every other
// connection it holds).
$span = null;
try {
$packet = Packet::parse($data);
$connection = $mqtt->open($fd);
$mqtt->bytesReceived->add(\strlen($data));
$span = Span::init('mqtt.' . $packet->name());
$span->set('mqtt.fd', $fd);
$span->set('mqtt.is_broker', false); // an inbound packet from a client
$span->set('mqtt.bytes', \strlen($data));
$response = new Response($server, $fd, $mqtt->bytesSent);
// authenticator/authorizer are inherited from the global container (registered above).
$packetContainer = new Container($container);
$packetContainer->set('mqtt', fn () => $mqtt);
$packetContainer->set('connection', fn () => $connection);
$packetContainer->set('packet', fn () => $packet);
$packetContainer->set('response', fn () => $response);
$dispatcher->dispatch($packetContainer, $packet->type);
// Every inbound packet is liveness: push the deadline forward (O(1), no wheel touch).
// The wheel is seeded once, when CONNECT establishes the interval; later packets only
// move the deadline and the reaper reschedules lazily when it visits the slot.
$connection->updateExpiresAt(microtime(true), $mqtt->keepAlive->multiplier);
if ($packet->type === Packet::CONNECT && $connection->active && $connection->keepAlive > 0) {
$connection->wheelSlot = $mqtt->keepAlive->schedule($fd, $connection->expiresAt);
}
// The identity, project and clean-start flag are populated by the CONNECT
// handler during dispatch, so record them afterwards rather than as defaults.
$span->set('project.id', $connection->prefix);
$span->set('user.id', $connection->identity['userId'] ?? '');
$span->set('mqtt.clean_start', $connection->cleanStart);
$span->finish();
} catch (\Throwable $error) {
Console::error('MQTT packet error on fd ' . $fd . ': ' . $error->getMessage());
$span?->finish(error: $error);
$server->close($fd);
}
});
$server->onClose(function (int $fd) use ($mqtt): void {
$mqtt->close($fd);
});
$server->start();
Generated
+6 -5
View File
@@ -4149,16 +4149,17 @@
"source": {
"type": "git",
"url": "https://github.com/utopia-php/mqtt.git",
"reference": "a4dafdebd5634ab64ebb08ad5930b603c5fc5d02"
"reference": "53ff974d0a179f07035241a8b8df0ca1b15a2ff5"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/mqtt/zipball/a4dafdebd5634ab64ebb08ad5930b603c5fc5d02",
"reference": "a4dafdebd5634ab64ebb08ad5930b603c5fc5d02",
"url": "https://api.github.com/repos/utopia-php/mqtt/zipball/53ff974d0a179f07035241a8b8df0ca1b15a2ff5",
"reference": "53ff974d0a179f07035241a8b8df0ca1b15a2ff5",
"shasum": ""
},
"require": {
"php": ">=8.1"
"php": ">=8.1",
"utopia-php/telemetry": "^0.4"
},
"require-dev": {
"laravel/pint": "^1.15 <1.21",
@@ -4207,7 +4208,7 @@
"source": "https://github.com/utopia-php/mqtt/tree/dev",
"issues": "https://github.com/utopia-php/mqtt/issues"
},
"time": "2026-09-15T07:09:28+00:00"
"time": "2026-09-17T15:41:04+00:00"
},
{
"name": "utopia-php/openapi",
+16 -132
View File
@@ -4,146 +4,47 @@ namespace Appwrite\Messaging\Adapter;
use Appwrite\Messaging\Adapter as MessagingAdapter;
use Appwrite\PubSub\Adapter as PubSub;
use Utopia\Mqtt\Connection;
use Utopia\Mqtt\Keepalive;
use Utopia\Mqtt\Packet;
use Utopia\Mqtt\Subscription\Store;
use Utopia\Telemetry\Adapter as Telemetry;
use Utopia\Telemetry\Counter;
use Utopia\Telemetry\Histogram;
use Utopia\Telemetry\UpDownCounter;
/**
* The MQTT producer: publishes messages onto the 'mqtt' pub/sub channel for the broker workers
* to fan out, and owns the broker telemetry instruments. Connection state, keep-alive and the
* subscription index now live inside the broker (utopia-php/mqtt Server), so they are gone here.
*/
class Mqtt extends MessagingAdapter
{
private const CHANNEL = 'mqtt';
/**
* Connection registry.
*
* [FD] -> Connection { protocol, prefix, identity, packetId, active }
*
* @var array<int, Connection>
*/
public array $connections = [];
// Telemetry instruments the broker records connection and delivery outcomes on
// (accepted/rejected, granted/denied, delivered/dropped). A no-op telemetry adapter
// yields no-op instruments, so the broker runs untelemetered under raw protocol tests.
public readonly Counter $connectionsOpened;
public readonly UpDownCounter $connectionsActive;
public readonly Counter $subscriptions;
public readonly Counter $messagesPublished;
public readonly Counter $messagesDelivered;
public readonly Counter $messagesDropped;
public readonly Counter $messagesAcked;
public readonly Counter $pubacksReceived;
public readonly Counter $reauth;
public readonly Counter $bytesReceived;
public readonly Counter $bytesSent;
public readonly Histogram $authDuration;
public readonly Histogram $connectionDuration;
public readonly Histogram $messageSize;
/** Keep-alive reaper wheel: connections register their deadline here, the tick drains it. */
public readonly Keepalive $keepAlive;
private ?Store $subscriptionStore = null;
public function __construct(Telemetry $telemetry, private readonly PubSub $pubsub)
{
$this->connectionsOpened = $telemetry->createCounter('mqtt.connections.opened');
$this->connectionsActive = $telemetry->createUpDownCounter('mqtt.connections.active');
$this->subscriptions = $telemetry->createCounter('mqtt.subscriptions');
$this->messagesPublished = $telemetry->createCounter('mqtt.messages.published');
$this->messagesDelivered = $telemetry->createCounter('mqtt.messages.delivered');
$this->messagesDropped = $telemetry->createCounter('mqtt.messages.dropped');
$this->messagesAcked = $telemetry->createCounter('mqtt.messages.acked');
$this->pubacksReceived = $telemetry->createCounter('mqtt.puback.received');
$this->reauth = $telemetry->createCounter('mqtt.reauth');
$this->bytesReceived = $telemetry->createCounter('mqtt.bytes.received', 'By');
$this->bytesSent = $telemetry->createCounter('mqtt.bytes.sent', 'By');
$this->authDuration = $telemetry->createHistogram('mqtt.auth.duration', 's');
$this->connectionDuration = $telemetry->createHistogram('mqtt.connection.duration', 's');
$this->messageSize = $telemetry->createHistogram('mqtt.message.size', 'By');
$this->keepAlive = new Keepalive();
}
/**
* The subscription index, created lazily so the adapter owns it rather than
* receiving it — the in-memory tree has no external dependency to wire in.
*/
private function subscriptionStore(): Store
{
return $this->subscriptionStore ??= new Store();
}
/** Get or create the connection state for a file descriptor. */
public function open(int $fd): Connection
{
return $this->connections[$fd] ??= new Connection($fd);
}
/**
* Drop a connection: balance the active gauge, remove every subscription, and
* forget its state. The transport-level socket close is handled by the caller.
*/
public function close(int $fd): void
{
$connection = $this->connections[$fd] ?? null;
if ($connection !== null) {
if ($connection->active) {
$this->connectionsActive->add(-1);
$this->connectionDuration->record(microtime(true) - $connection->openedAt);
}
$this->keepAlive->remove($fd);
}
$this->unsubscribe($fd);
unset($this->connections[$fd]);
}
/**
* Subscribe a connection to topic filters. $identifier is the fd, $channels the
* topic filters; roles, queries and $subscriptionId have no MQTT meaning and are
* ignored (the store keys by topic). The granted QoS is 1 and the user id comes
* from the resolved identity so the subscription store can scale per client.
*
* @param string $subscriptionId ignored (part of the Messaging adapter contract)
* @param array<int, string> $roles ignored
* @param array<int, string> $channels topic filters
* @param array<int, mixed> $queryGroup ignored
* @param int $qos granted QoS stored per subscription (delivery is clamped to it)
*/
public function subscribe(string $projectId, mixed $identifier, string $subscriptionId, array $roles, array $channels, array $queryGroup = [], int $qos = Packet::QOS_1): void
{
$userId = $this->connections[$identifier]->identity['userId'] ?? '';
foreach ($channels as $topic) {
$this->subscriptionStore()->subscribe(
$projectId,
$userId,
$topic,
$identifier,
$qos,
);
}
}
/** Remove every subscription for a connection (used on close). */
public function unsubscribe(mixed $identifier): void
{
$this->subscriptionStore()->close($identifier);
}
/**
* Publish a message onto the 'mqtt' pub/sub channel for fan-out. Every broker
* worker listens on that channel and delivers to its own local subscribers, so a
* publish reaches subscribers regardless of which worker holds them — the MQTT
* analogue of Realtime::send publishing to the realtime firehose.
*
* MQTT carries raw bytes, so the message and its inbound QoS ride in $options
* ($options['payload'], $options['qos']) rather than the array $payload;
* $channels holds the topics. The payload is base64-encoded to survive the JSON
* envelope.
* Publish a message onto the 'mqtt' pub/sub channel for fan-out. Every broker worker listens
* on that channel and delivers to its own local subscribers. The binary MQTT payload and its
* QoS ride in $options ($options['payload'], $options['qos'], $options['sequence']); $channels
* holds the topics. The payload is base64-encoded to survive the JSON envelope.
*
* @param array<mixed> $payload unused for MQTT (payload is binary, see $options)
* @param array<int, string> $events ignored
@@ -171,36 +72,19 @@ class Mqtt extends MessagingAdapter
}
}
/** Remove a single topic subscription (MQTT UNSUBSCRIBE). */
public function unsubscribeSubscription(int $fd, string $topic): void
{
$this->subscriptionStore()->unsubscribe($topic, $fd);
}
/**
* The fds subscribed to a topic in a project, each mapped to its granted QoS.
* Part of the messaging-adapter contract but unused for MQTT: subscriptions are owned by the
* broker (utopia-php/mqtt Server records them from the SUBACK the Handler returns).
*
* @return array<int, int> fd => granted QoS
* @param array<int, string> $roles
* @param array<int, string> $channels
* @param array<int, mixed> $queryGroup
*/
public function getSubscribers(string $projectId, string $topic): array
public function subscribe(string $projectId, mixed $identifier, string $subscriptionId, array $roles, array $channels, array $queryGroup = []): void
{
return $this->subscriptionStore()->getSubscribers($projectId, $topic);
}
public function hasSubscriber(string $projectId, string $topic): bool
public function unsubscribe(mixed $identifier): void
{
return $this->subscriptionStore()->getSubscribers($projectId, $topic) !== [];
}
/**
* The subscription-store record for an fd: its project, user, and subscriptions,
* or null when the fd holds none. This is the store's copy (the prefix/userId
* captured at subscribe time), not the live Connection object from open().
*
* @return array{prefix: string, userId: string, subs: array<string, int>}|null
*/
public function getConnection(int $fd): ?array
{
return $this->subscriptionStore()->getConnection($fd);
}
}
-61
View File
@@ -1,61 +0,0 @@
<?php
namespace Appwrite\Mqtt;
use Appwrite\Extend\Exception;
use Utopia\DI\Container;
use Utopia\Platform\Action;
/**
* Routes a decoded MQTT packet to the handler registered for its control-packet
* type, resolving the handler's injections from the per-packet container, keyed on
* the binary packet type.
*
* The handler set is fixed at construction and never mutated afterwards: there is no
* addHandler/removeHandler. In a coroutine broker a single dispatcher is shared across
* every connection and packet, so a mutable registry would leak handler state across
* requests and invite ad-hoc reconfiguration; an immutable table keeps dispatch a pure
* lookup.
*/
class Dispatcher
{
public const LABEL_TYPE = 'packetType';
/** @var array<int, Action> */
private readonly array $handlers;
/**
* @param array<int, Action> $handlers each labelled with its packet type (self::LABEL_TYPE)
*/
public function __construct(array $handlers)
{
$table = [];
foreach ($handlers as $handler) {
$type = $handler->getLabels()[self::LABEL_TYPE]
?? throw new Exception(Exception::GENERAL_SERVER_ERROR, 'MQTT packet handler is missing the packetType label.');
$table[$type] = $handler;
}
$this->handlers = $table;
}
/**
* Dispatch to the handler for this packet type. Unhandled types (e.g. PUBACK)
* are ignored, matching a broker that acknowledges only what it must.
*/
public function dispatch(Container $container, int $type): void
{
$handler = $this->handlers[$type] ?? null;
if ($handler === null) {
return;
}
$args = [];
foreach ($handler->getOptions() as $option) {
$args[] = $container->get($option['name']);
}
($handler->getCallback())(...$args);
}
}
+343
View File
@@ -0,0 +1,343 @@
<?php
namespace Appwrite\Mqtt;
use Appwrite\Messaging\Adapter\Mqtt;
use Appwrite\Utopia\Database\Documents\User;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Query;
use Utopia\DI\Container;
use Utopia\Mqtt\Connection;
use Utopia\Mqtt\Handler as MqttHandler;
use Utopia\Mqtt\Packet;
use Utopia\Mqtt\Packet\Auth;
use Utopia\Mqtt\Packet\Connack;
use Utopia\Mqtt\Packet\Connect;
use Utopia\Mqtt\Packet\Disconnect;
use Utopia\Mqtt\Packet\Puback;
use Utopia\Mqtt\Packet\Publish;
use Utopia\Mqtt\Packet\Suback;
use Utopia\Mqtt\Packet\Subscribe;
use Utopia\Mqtt\Packet\Unsuback;
use Utopia\Mqtt\Packet\Unsubscribe;
use Utopia\Mqtt\Server;
use Utopia\Span\Span;
use Utopia\System\System;
class Handler implements MqttHandler
{
private const REPLAY_TTL = 3600;
public function __construct(
private readonly Container $container,
private readonly Mqtt $mqtt,
) {
}
public function onConnect(Connect $connect, Connection $connection): Connack|Auth
{
$authMethod = $connect->authMethod ?? '';
$connection->prefix = $connect->userProperties()['projectId'] ?? '';
Span::add('project.id', $connection->prefix);
Span::add('mqtt.clean_start', $connect->cleanStart);
Span::add('mqtt.auth_method', $authMethod);
$authenticator = $this->container->get('authenticator');
$start = microtime(true);
$identity = $authenticator($connection->prefix, $authMethod, $connect->authData ?? '');
$this->mqtt->authDuration->record(microtime(true) - $start);
if ($identity === []) {
Span::add('mqtt.result', 'rejected');
return Connack::refuse(Connack::NOT_AUTHORIZED);
}
$connection->identity = $identity;
Span::add('user.id', $identity['userId'] ?? '');
// Rate-limit CONNECT per user (keyed on userId until the infra surfaces client IP).
if (System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') {
$getRedis = $this->container->get('getRedis');
$timeLimit = new TimeLimitRedis('mqtt:connect:{userId}', 128, 60, $getRedis());
$timeLimit->setParam('{userId}', $identity['userId'] ?? '');
if ((new Abuse($timeLimit))->check()) {
Span::add('mqtt.result', 'abuse');
return Connack::refuse(Connack::QUOTA_EXCEEDED);
}
}
// The per-device session anchor: the client id, or an account-level fallback so reconnects
// resume the same session (the offline-replay cursor is keyed on it).
$clientId = $connect->clientId;
if ($clientId === '') {
$clientId = 'custom_' . $connection->prefix . '_' . ($identity['userId'] ?? '');
}
$connection->setClientId($clientId);
Span::add('mqtt.client_id', $connection->getClientId());
return Connack::accept();
}
public function onAuthenticate(Auth $auth, Connection $connection): Connack|Auth|Disconnect
{
$projectId = $auth->userProperties()['projectId'] ?? '';
$identity = [];
if ($projectId !== '' && $projectId === $connection->prefix) {
$authenticator = $this->container->get('authenticator');
$start = microtime(true);
$identity = $authenticator($projectId, $auth->method, $auth->data);
$this->mqtt->authDuration->record(microtime(true) - $start);
}
// Reauth only refreshes the credential for the identity resolved at CONNECT — it must not
// switch user or project, which would leak the previous user's subscriptions.
if ($identity === [] || ($identity['userId'] ?? '') !== ($connection->identity['userId'] ?? '')) {
$this->mqtt->reauth->add(1, ['result' => 'rejected']);
Span::add('mqtt.result', 'rejected');
return Disconnect::refuse(Disconnect::NOT_AUTHORIZED);
}
$connection->identity = $identity;
$this->mqtt->reauth->add(1, ['result' => 'success']);
Span::add('mqtt.result', 'reauthenticated');
return Auth::success($auth->method);
}
public function onSubscribe(Subscribe $subscribe, Connection $connection): Suback
{
$authorizer = $this->container->get('authorizer');
$identity = $connection->identity;
$projectDB = $this->getProjectDB($connection->prefix);
$authorization = $projectDB->getAuthorization();
/** @var User $user */
$user = $authorization->skip(fn () => $projectDB->getDocument('users', $identity['userId'] ?? ''));
$roles = $user->getRoles($authorization);
$allowed = [];
foreach ($subscribe->filters() as $filter) {
if (!isset($allowed[$filter->topic]) && ($authorizer === null || $authorizer($identity, $filter->topic))) {
$allowed[$filter->topic] = true;
}
}
$topicsById = [];
if ($allowed !== []) {
$documents = $authorization->skip(fn () => $projectDB->find('topics', [
Query::equal('$id', array_keys($allowed)),
Query::limit(\count($allowed)),
]));
foreach ($documents as $document) {
$topicsById[$document->getId()] = $document;
}
}
$suback = new Suback();
$grantedTopics = [];
foreach ($subscribe->filters() as $filter) {
Span::add('mqtt.topic', $filter->topic);
$document = $topicsById[$filter->topic] ?? null;
if (!isset($allowed[$filter->topic]) || $document === null) {
$suback->deny();
continue;
}
if (!$this->authorizedForTopic($document->getAttribute('subscribe', []) ?? [], $roles)) {
$suback->deny();
continue;
}
$topicQos = $document->getAttribute('qos');
$grantedQos = min($filter->qos, $topicQos === null ? Packet::QOS_1 : (int) $topicQos);
$suback->grant($grantedQos);
$grantedTopics[$filter->topic] = [$document, $grantedQos];
}
if ($grantedTopics !== []) {
$this->replayBacklog($connection, $projectDB, $grantedTopics);
}
return $suback;
}
public function onUnsubscribe(Unsubscribe $unsubscribe, Connection $connection): Unsuback
{
// The broker has already dropped the filters from its subscription index.
$unsuback = new Unsuback();
foreach ($unsubscribe->filters() as $filter) {
$unsuback->success();
}
return $unsuback;
}
/**
* Appwrite is server -> client push: clients never PUBLISH, so an inbound one is ignored.
* Server-initiated fan-out goes through deliver() instead.
*
* @param iterable<array{0: Connection, 1: int}> $subscribers
*/
public function onPublish(Publish $publish, Connection $connection, iterable $subscribers): void
{
}
/**
* Fan a message out to a topic's local subscribers, each at the QoS granted to that
* subscription (clamped by the message QoS). Returns the number delivered.
*/
public function deliver(Server $server, string $projectId, string $topic, string $message, int $qos, int $sequence): int
{
$delivered = 0;
foreach ($server->subscribers($projectId, $topic) as [$connection, $grantedQos]) {
$deliveryQos = min($qos, $grantedQos);
$connection->publish($topic, $message, qos: $deliveryQos, sequence: $sequence);
$this->mqtt->messagesDelivered->add(1, ['qos' => $deliveryQos]);
$delivered++;
}
return $delivered;
}
public function onPuback(Puback $puback, Connection $connection): void
{
$delivery = $connection->acknowledge($puback->packetId);
if ($delivery === null) {
return;
}
Span::add('mqtt.topic', $delivery['topic']);
Span::add('mqtt.sequence', $delivery['sequence']);
$this->mqtt->messagesAcked->add(1);
$this->mqtt->pubacksReceived->add(1);
$getCache = $this->container->get('getCache');
$cache = $getCache();
$key = $this->cursorKey($connection);
$topic = $delivery['topic'];
$cursor = (int) $delivery['cursor'];
$current = $cache->loadMany($key, self::REPLAY_TTL, [$topic]);
if ($cursor > (int) ($current[$topic]['sequence'] ?? -1)) {
$cache->saveMany($key, [$topic => ['sequence' => $cursor]], 0);
}
}
public function onDisconnect(?Disconnect $disconnect, Connection $connection): void
{
// The broker already removed the connection, its subscriptions and keep-alive slot, and
// records the active-connections gauge itself. Record the connection lifetime for
// accepted sessions.
if (($connection->identity['userId'] ?? '') !== '') {
$this->mqtt->connectionDuration->record(microtime(true) - $connection->openedAt);
}
}
/**
* Whether the subscriber's roles satisfy a topic's subscribe roles. No configured roles (or an
* explicit `any`) means the topic is open to everyone.
*
* @param array<int, string> $topicRoles
* @param array<int, string> $userRoles
*/
private function authorizedForTopic(array $topicRoles, array $userRoles): bool
{
if ($topicRoles === [] || \in_array(Role::any()->toString(), $topicRoles, true)) {
return true;
}
return \array_intersect($topicRoles, $userRoles) !== [];
}
/**
* Replay the offline backlog for freshly granted QoS 1 topics: resume each topic's cursor,
* then re-deliver stored messages via the connection (DUP flagged), tagging each with its
* durable sequence so a later PUBACK maps back.
*
* @param array<string, array{0: Document, 1: int}> $grantedTopics filter => [document, grantedQos]
*/
private function replayBacklog(Connection $connection, Database $projectDB, array $grantedTopics): void
{
$getCache = $this->container->get('getCache');
$getPlanForUser = $this->container->get('getPlanForUser');
$cache = $getCache();
$key = $this->cursorKey($connection);
$maxDepth = max(1, $getPlanForUser(new Document(['$id' => $connection->prefix]), $connection->identity['userId'] ?? ''));
$expiry = 0;
foreach ($grantedTopics as [$document]) {
$expiry = max($expiry, (int) $document->getAttribute('expiry', 0));
}
if ($expiry <= 0) {
$expiry = self::REPLAY_TTL;
}
if ($connection->cleanStart) {
$cache->purge($key);
}
$cursors = $cache->loadMany($key, $expiry);
$persist = [];
foreach ($grantedTopics as $filter => [$document, $grantedQos]) {
$tail = (int) $document->getAttribute('sequence', 0);
$from = (int) ($cursors[$filter]['sequence'] ?? $tail);
$persist[$filter] = ['sequence' => $from];
if ($grantedQos < 1 || $tail <= $from) {
continue;
}
$connection->resume($filter, $from);
$start = max($from + 1, $tail - $maxDepth + 1);
$messages = $projectDB->getAuthorization()->skip(fn () => $projectDB->find('appwritePushLedger', [
Query::equal('topic', [$filter]),
Query::greaterThanEqual('sequence', $start),
Query::orderAsc('sequence'),
Query::limit($maxDepth),
]));
foreach ($messages as $message) {
$stored = $message->getAttribute('data');
$data = \is_string($stored) ? $stored : (string) json_encode($stored);
$connection->publish($filter, $data, qos: 1, dup: true, sequence: (int) $message->getAttribute('sequence'));
}
}
if ($persist !== []) {
$cache->saveMany($key, $persist, $expiry);
}
}
private function cursorKey(Connection $connection): string
{
return 'appwrite:push:cursor:' . $connection->prefix
. ':' . ($connection->identity['userId'] ?? '')
. ':' . $connection->getClientId();
}
private function getProjectDB(string $projectId): Database
{
$getConsoleDB = $this->container->get('getConsoleDB');
$getProjectDB = $this->container->get('getProjectDB');
$consoleDB = $getConsoleDB();
$project = $consoleDB->getAuthorization()->skip(fn () => $consoleDB->getDocument('projects', $projectId));
return $getProjectDB($project);
}
}
-86
View File
@@ -1,86 +0,0 @@
<?php
namespace Appwrite\Mqtt\Handlers;
use Appwrite\Messaging\Adapter\Mqtt;
use Appwrite\Mqtt\Dispatcher;
use Appwrite\Mqtt\Response;
use Utopia\Mqtt\Connection;
use Utopia\Mqtt\Packet;
use Utopia\Mqtt\Packet\V5;
use Utopia\Mqtt\Properties;
use Utopia\Mqtt\Property;
use Utopia\Platform\Action;
use Utopia\Span\Span;
class Auth extends Action
{
public function __construct()
{
$this
->desc('Re-authenticate a live connection with a fresh credential')
->label(Dispatcher::LABEL_TYPE, Packet::AUTH)
->inject('mqtt')
->inject('authenticator')
->inject('connection')
->inject('packet')
->inject('response')
->callback($this->action(...));
}
/**
* @param (callable(string, string, string): array<string, string>)|null $authenticator
*/
public function action(Mqtt $mqtt, ?callable $authenticator, Connection $connection, Packet $packet, Response $response): void
{
$body = $packet->body;
$method = '';
$data = '';
$userProperties = [];
if ($body !== '') {
// $body[0] is the reason code (0x19 re-authenticate);
// from offset 1 everything is the MQTT 5.0 property block.
[$properties] = Properties::parse($body, 1);
$method = (string) ($properties->get(Property::AUTHENTICATION_METHOD) ?? '');
$data = (string) ($properties->get(Property::AUTHENTICATION_DATA) ?? '');
$userProperties = $properties->user();
}
// The AUTH packet re-sends projectId as a User Property.
if ($authenticator !== null) {
$projectId = $userProperties['projectId'] ?? '';
$identity = [];
if ($projectId !== '' && $projectId === $connection->prefix) {
$start = microtime(true);
$identity = $authenticator($projectId, $method, $data);
$mqtt->authDuration->record(microtime(true) - $start);
}
// Reauth only refreshes the credential for the identity resolved at CONNECT — it
// must not switch the user (or project). Allowing a switch would keep the previous
// user's subscriptions on this connection, leaking their fan-out to the new user
// and breaking PUBACK ownership checks. A mismatch (or failure) drops the connection.
if ($identity === [] || ($identity['userId'] ?? '') !== ($connection->identity['userId'] ?? '')) {
$mqtt->reauth->add(1, ['result' => 'rejected']);
Span::add('mqtt.result', 'rejected');
$response->send(V5::disconnect(V5::REASON_NOT_AUTHORIZED));
$response->close();
return;
}
$connection->identity = $identity;
$mqtt->reauth->add(1, ['result' => 'success']);
Span::add('mqtt.result', 'reauthenticated');
}
// Acknowledge success on the live connection with an AUTH packet that echoes
// the authentication method back.
$properties = new Properties();
if ($method !== '') {
$properties->add(new Property(Property::AUTHENTICATION_METHOD, $method));
}
$response->send(V5::auth(V5::AUTH_SUCCESS, $properties));
}
}
-130
View File
@@ -1,130 +0,0 @@
<?php
namespace Appwrite\Mqtt\Handlers;
use Appwrite\Messaging\Adapter\Mqtt;
use Appwrite\Mqtt\Dispatcher;
use Appwrite\Mqtt\Response;
use Utopia\Abuse\Abuse;
use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis;
use Utopia\Mqtt\Connection;
use Utopia\Mqtt\Packet;
use Utopia\Mqtt\Packet\V3;
use Utopia\Mqtt\Packet\V5;
use Utopia\Mqtt\Properties;
use Utopia\Mqtt\Property;
use Utopia\Platform\Action;
use Utopia\Span\Span;
use Utopia\System\System;
class Connect extends Action
{
public function __construct()
{
$this
->desc('Authenticate a CONNECT and open the session')
->label(Dispatcher::LABEL_TYPE, Packet::CONNECT)
->inject('authenticator')
->inject('mqtt')
->inject('connection')
->inject('packet')
->inject('response')
->callback($this->action(...));
}
/**
* @param (callable(string, string, string): array<string, string>)|null $authenticator
*/
public function action(?callable $authenticator, Mqtt $mqtt, Connection $connection, Packet $packet, Response $response): void
{
$body = $packet->body;
$offset = 0;
[, $offset] = Packet::readString($body, $offset); // protocol name
$level = ord($body[$offset]);
$connection->protocol = $level;
$offset += 1; // protocol level
$offset += 1; // connect flags
[$keepAlive, $offset] = Packet::readInt16($body, $offset); // keep alive (seconds)
$connection->keepAlive = $keepAlive;
$connection->cleanStart = Packet::isCleanStart($body);
Span::add('mqtt.clean_start', $connection->cleanStart);
// MQTT 3.1.1 carries no property block; enhanced auth and metadata are 5.0 only.
$authMethod = '';
$authData = '';
$user = [];
if ($level >= 5) {
[$properties] = Properties::parse($body, $offset);
$authMethod = (string) ($properties->get(Property::AUTHENTICATION_METHOD) ?? '');
$authData = (string) ($properties->get(Property::AUTHENTICATION_DATA) ?? '');
$user = $properties->user();
}
$projectId = $user['projectId'] ?? '';
$connection->prefix = $projectId;
Span::add('project.id', $projectId);
Span::add('mqtt.auth_method', $authMethod);
if ($authenticator !== null) {
$start = microtime(true);
$identity = $authenticator($projectId, $authMethod, $authData);
$duration = microtime(true) - $start;
$mqtt->authDuration->record($duration);
Span::add('mqtt.auth.duration', $duration);
if ($identity === []) {
$mqtt->connectionsOpened->add(1, ['auth_method' => $authMethod, 'result' => 'rejected']);
Span::add('mqtt.result', 'rejected');
$response->send($this->connack($level, false));
$response->close();
return;
}
$connection->identity = $identity;
Span::add('user.id', $identity['userId'] ?? '');
// Rate-limit CONNECT per user.
// TODO: in future this will be IP based; currently keyed on userId because the
// infra does not surface the client IP to the broker.
if (System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') {
$timeLimit = new TimeLimitRedis('mqtt:connect:{userId}', 128, 60, getRedis());
$timeLimit->setParam('{userId}', $identity['userId'] ?? '');
if ((new Abuse($timeLimit))->check()) {
$mqtt->connectionsOpened->add(1, ['auth_method' => $authMethod, 'result' => 'abuse']);
Span::add('mqtt.result', 'abuse');
$response->send($this->connack($level, false));
$response->close();
return;
}
}
}
// The per-device session anchor: the client-supplied id, or an account-level fallback so
// reconnects resume the same session (the offline-replay cursor is keyed on it). Resolved
// here rather than in the library, which only server-assigns a generic id for an empty one.
$clientId = Packet::getClientId($body);
if ($clientId === '') {
$clientId = 'custom_' . $connection->prefix . '_' . ($connection->identity['userId'] ?? '');
}
$connection->setClientId($clientId);
Span::add('mqtt.client_id', $connection->getClientId());
$mqtt->connectionsOpened->add(1, ['auth_method' => $authMethod, 'result' => 'accepted']);
$mqtt->connectionsActive->add(1);
$connection->active = true;
$response->send($this->connack($level, true));
}
/** CONNACK, with the acknowledgement code in each version's own vocabulary. */
private function connack(int $level, bool $accepted): string
{
if ($level >= 5) {
return V5::connack($accepted ? V5::REASON_SUCCESS : V5::REASON_NOT_AUTHORIZED);
}
return V3::connack($accepted ? V3::RETURN_ACCEPTED : V3::RETURN_NOT_AUTHORIZED);
}
}
-25
View File
@@ -1,25 +0,0 @@
<?php
namespace Appwrite\Mqtt\Handlers;
use Appwrite\Mqtt\Dispatcher;
use Appwrite\Mqtt\Response;
use Utopia\Mqtt\Packet;
use Utopia\Platform\Action;
class Disconnect extends Action
{
public function __construct()
{
$this
->desc('Close the connection on client request')
->label(Dispatcher::LABEL_TYPE, Packet::DISCONNECT)
->inject('response')
->callback($this->action(...));
}
public function action(Response $response): void
{
$response->close();
}
}
-25
View File
@@ -1,25 +0,0 @@
<?php
namespace Appwrite\Mqtt\Handlers;
use Appwrite\Mqtt\Dispatcher;
use Appwrite\Mqtt\Response;
use Utopia\Mqtt\Packet;
use Utopia\Platform\Action;
class Ping extends Action
{
public function __construct()
{
$this
->desc('Reply to a client heartbeat')
->label(Dispatcher::LABEL_TYPE, Packet::PINGREQ)
->inject('response')
->callback($this->action(...));
}
public function action(Response $response): void
{
$response->send(Packet::pingresp());
}
}
-75
View File
@@ -1,75 +0,0 @@
<?php
namespace Appwrite\Mqtt\Handlers;
use Appwrite\Messaging\Adapter\Mqtt;
use Appwrite\Mqtt\Dispatcher;
use Appwrite\Mqtt\Response;
use Utopia\Mqtt\Connection;
use Utopia\Mqtt\Packet;
use Utopia\Mqtt\Packet\V5;
use Utopia\Platform\Action;
use Utopia\Span\Span;
/**
* A subscriber's PUBACK for a QoS 1 message the broker delivered. This is the only
* event that advances that subscriber's cursor: the ack proves the subscriber holds
* that sequence, so replay on reconnect can skip it. A cursor never advances on
* publish, only here, which is how an offline client keeps its backlog.
*/
class Puback extends Action
{
public function __construct()
{
$this
->desc('Acknowledge a delivered QoS 1 message and advance the subscriber cursor')
->label(Dispatcher::LABEL_TYPE, Packet::PUBACK)
->inject('mqtt')
->inject('connection')
->inject('packet')
->inject('response')
->callback($this->action(...));
}
public function action(Mqtt $mqtt, Connection $connection, Packet $packet, Response $response): void
{
$body = $packet->body;
if (\strlen($body) < 2) {
return;
}
$connectionMetdata = $mqtt->getConnection($connection->fd);
if ($connectionMetdata['userId'] !== $connection->identity['userId'] || $connectionMetdata['prefix'] !== $connection->prefix) {
$response->send(V5::disconnect(V5::REASON_NOT_AUTHORIZED));
$response->close();
return;
}
// PUBACK opens with the two-byte packet id in both versions.
[$packetId] = Packet::readInt16($body, 0);
$delivery = $connection->acknowledge($packetId);
if ($delivery === null) {
return;
}
Span::add('mqtt.topic', $delivery['topic']);
Span::add('mqtt.sequence', $delivery['sequence']);
$mqtt->messagesAcked->add(1);
$mqtt->pubacksReceived->add(1);
// Save how far the client has caught up (the "cursor" from acknowledge), not the
// sequence it just acked — so an out-of-order ack can't skip a still-unacked
// message. Only ever move the cursor forward, so two acks at once can't rewind it.
$cache = getCache();
$cursorKey = 'appwrite:push:cursor:' . $connection->prefix . ':' . $connection->identity['userId'] . ':' . $connection->getClientId();
$topic = $delivery['topic'];
$cursor = (int) $delivery['cursor'];
$current = $cache->loadMany($cursorKey, 3600, [$topic]);
if ($cursor > (int) ($current[$topic]['sequence'] ?? -1)) {
$cache->saveMany($cursorKey, [$topic => ['sequence' => $cursor]], 3600);
}
}
}
-238
View File
@@ -1,238 +0,0 @@
<?php
namespace Appwrite\Mqtt\Handlers;
use Appwrite\Messaging\Adapter\Mqtt;
use Appwrite\Mqtt\Dispatcher;
use Appwrite\Mqtt\Response;
use Appwrite\Utopia\Database\Documents\User;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Query;
use Utopia\Mqtt\Connection;
use Utopia\Mqtt\Packet;
use Utopia\Mqtt\Packet\V3;
use Utopia\Mqtt\Packet\V5;
use Utopia\Mqtt\Properties;
use Utopia\Platform\Action;
use Utopia\Span\Span;
class Subscribe extends Action
{
public function __construct()
{
$this
->desc('Subscribe to topic filters, enforcing the ACL')
->label(Dispatcher::LABEL_TYPE, Packet::SUBSCRIBE)
->inject('authorizer')
->inject('mqtt')
->inject('connection')
->inject('packet')
->inject('response')
->callback($this->action(...));
}
/**
* @param (callable(array<string, string>, string): bool)|null $authorizer
*/
public function action(?callable $authorizer, Mqtt $mqtt, Connection $connection, Packet $packet, Response $response): void
{
$body = $packet->body;
$offset = 0;
$packetId = substr($body, 0, 2);
$offset += 2;
// MQTT 5.0 carries a property block before the filters; skip it to reach them.
if ($connection->protocol >= 5) {
$offset = Properties::skip($body, $offset);
}
$identity = $connection->identity;
$denied = $connection->protocol >= 5 ? V5::REASON_NOT_AUTHORIZED : V3::SUBSCRIBE_FAILURE;
$consoleDatabase = getConsoleDB();
$project = $consoleDatabase->getAuthorization()->skip(fn () => $consoleDatabase->getDocument('projects', $connection->prefix));
$projectDB = getProjectDB($project);
// The subscriber's roles, resolved once, to enforce each topic's subscribe ACL below.
$authorization = $projectDB->getAuthorization();
/** @var User $user */
$user = $authorization->skip(fn () => $projectDB->getDocument('users', $connection->identity['userId'] ?? ''));
$roles = $user->getRoles($authorization);
// Parse every (filter, requested QoS) in order; the SUBACK carries one reason code
// per filter in this order. Bits 0-1 of the subscription options byte are the max QoS.
$filters = [];
while ($offset < strlen($body)) {
[$filter, $offset] = Packet::readString($body, $offset);
$filters[] = [$filter, isset($body[$offset]) ? \ord($body[$offset]) & 0x03 : 0];
$offset += 1;
}
$allowed = [];
foreach ($filters as [$filter]) {
if (!isset($allowed[$filter]) && ($authorizer === null || $authorizer($identity, $filter))) {
$allowed[$filter] = true;
}
}
$topicsById = [];
if ($allowed !== []) {
$documents = $projectDB->getAuthorization()->skip(fn () => $projectDB->find('topics', [
Query::equal('$id', array_keys($allowed)),
Query::limit(\count($allowed)),
]));
foreach ($documents as $document) {
$topicsById[$document->getId()] = $document;
}
}
$granted = '';
$topicDocuments = []; // filter => topic document, reused for replay below
$grantedQosByTopic = []; // filter => granted QoS, so replay honours it below
foreach ($filters as [$filter, $requestedQos]) {
Span::add('mqtt.topic', $filter);
if (!isset($allowed[$filter])) {
$granted .= chr($denied);
$mqtt->subscriptions->add(1, ['result' => 'denied']);
continue;
}
// A filter is granted only if it maps to an existing project topic.
$topicDocument = $topicsById[$filter] ?? null;
if ($topicDocument === null) {
$granted .= chr($denied);
$mqtt->subscriptions->add(1, ['result' => 'unknown']);
continue;
}
// The topic's subscribe roles gate who may subscribe.
if (!$this->authorizedForTopic($topicDocument->getAttribute('subscribe', []) ?? [], $roles)) {
$granted .= chr($denied);
$mqtt->subscriptions->add(1, ['result' => 'forbidden']);
continue;
}
// Requested max QoS, capped by the topic's configured QoS (null = no cap, broker max 1).
$topicQos = $topicDocument->getAttribute('qos');
$grantedQos = min($requestedQos, $topicQos === null ? Packet::QOS_1 : (int) $topicQos);
$mqtt->subscribe($connection->prefix, $connection->fd, '', [], [$filter], [], $grantedQos);
$granted .= chr($grantedQos);
$mqtt->subscriptions->add(1, ['result' => 'granted']);
$topicDocuments[$filter] = $topicDocument;
$grantedQosByTopic[$filter] = $grantedQos;
}
$response->send(
$connection->protocol >= 5
? V5::suback($packetId, $granted)
: V3::suback($packetId, $granted),
);
if ($topicDocuments === []) {
return;
}
$topics = array_keys($topicDocuments);
$expiry = 0;
foreach ($topicDocuments as $topicDocument) {
$expiry = max($expiry, (int) $topicDocument->getAttribute('expiry', 0));
}
if ($expiry <= 0) {
$expiry = 3600;
}
// Replay depth comes from the user's org plan (self-hosted default; see getPlanForUser)
$maxDepth = max(1, getPlanForUser($project, $connection->identity['userId'] ?? ''));
$cache = getCache();
$cursorKey = 'appwrite:push:cursor:' . $connection->prefix . ':' . $connection->identity['userId'] . ':' . $connection->getClientId();
// A clean-start session discards any persisted cursor before resuming.
if ($connection->cleanStart) {
$cache->purge($cursorKey);
}
$cursors = $cache->loadMany($cursorKey, $expiry);
$known = array_keys($cursors);
$newTopics = array_values(array_diff($topics, $known));
$resumeTopics = array_values(array_intersect($topics, $known));
// Current tail (topics.sequence) per subscribed topic: the seed for new topics and
// the upper bound for replay. Reuses the documents fetched during the grant loop.
$tails = [];
foreach ($topicDocuments as $topic => $topicDocument) {
$tails[$topic] = (int) $topicDocument->getAttribute('sequence', 0);
}
$persist = [];
foreach ($newTopics as $topic) {
$persist[$topic] = ['sequence' => $tails[$topic] ?? 0];
}
foreach ($resumeTopics as $topic) {
$persist[$topic] = $cursors[$topic];
}
foreach ($resumeTopics as $topic) {
if (($grantedQosByTopic[$topic] ?? Packet::QOS_1) < Packet::QOS_1) {
continue;
}
$from = (int) ($cursors[$topic]['sequence'] ?? 0);
$tail = $tails[$topic] ?? $from;
if ($tail <= $from) {
continue;
}
$start = max($from + 1, $tail - $maxDepth + 1);
$messages = $projectDB->getAuthorization()->skip(fn () => $projectDB->find('appwritePushLedger', [
Query::equal('topic', [$topic]),
Query::greaterThanEqual('sequence', $start),
Query::orderAsc('sequence'),
Query::limit($maxDepth),
]));
foreach ($messages as $message) {
$packetId = $connection->nextPacketId();
// The ledger `data` (json filter) already holds the encoded envelope
// string the live path publishes; re-encoding it would double-encode.
$stored = $message->getAttribute('data');
$data = \is_string($stored) ? $stored : (string) json_encode($stored);
$publish = $connection->protocol >= 5
? V5::publish($topic, $data, Packet::QOS_1, $packetId, dup: true)
: V3::publish($topic, $data, Packet::QOS_1, $packetId, dup: true);
$response->send($publish);
$connection->track($packetId, $topic, (int) $message->getAttribute('sequence'));
}
}
if ($persist !== []) {
$cache->saveMany($cursorKey, $persist, $expiry);
}
}
/**
* Whether the subscriber's roles satisfy a topic's subscribe roles. No configured roles
* means the topic is open to everyone, as does an explicit `any` role.
*
* @param array<int, string> $topicRoles the topic's `subscribe` roles
* @param array<int, string> $userRoles the subscriber's resolved roles
*/
private function authorizedForTopic(array $topicRoles, array $userRoles): bool
{
if ($topicRoles === []) {
return true;
}
if (\in_array(Role::any()->toString(), $topicRoles, true)) {
return true;
}
return \array_intersect($topicRoles, $userRoles) !== [];
}
}
@@ -1,55 +0,0 @@
<?php
namespace Appwrite\Mqtt\Handlers;
use Appwrite\Messaging\Adapter\Mqtt;
use Appwrite\Mqtt\Dispatcher;
use Appwrite\Mqtt\Response;
use Utopia\Mqtt\Connection;
use Utopia\Mqtt\Packet;
use Utopia\Mqtt\Packet\V3;
use Utopia\Mqtt\Packet\V5;
use Utopia\Mqtt\Properties;
use Utopia\Platform\Action;
class Unsubscribe extends Action
{
public function __construct()
{
$this
->desc('Remove topic-filter subscriptions')
->label(Dispatcher::LABEL_TYPE, Packet::UNSUBSCRIBE)
->inject('mqtt')
->inject('connection')
->inject('packet')
->inject('response')
->callback($this->action(...));
}
public function action(Mqtt $mqtt, Connection $connection, Packet $packet, Response $response): void
{
$body = $packet->body;
$offset = 0;
$packetId = substr($body, 0, 2);
$offset += 2;
// MQTT 5.0 carries a property block before the filters; skip it to reach them.
if ($connection->protocol >= 5) {
$offset = Properties::skip($body, $offset);
}
$count = 0;
while ($offset < strlen($body)) {
[$filter, $offset] = Packet::readString($body, $offset);
$mqtt->unsubscribeSubscription($connection->fd, $filter);
$count++;
}
// 5.0 carries a Success reason code per filter; 3.1.1 UNSUBACK is bare.
$response->send(
$connection->protocol >= 5
? V5::unsuback($packetId, $count)
: V3::unsuback($packetId),
);
}
}
-39
View File
@@ -1,39 +0,0 @@
<?php
namespace Appwrite\Mqtt;
use Utopia\Mqtt\Server;
use Utopia\Telemetry\Counter;
/**
* Writes MQTT packets back to a single connection. Each packet handler receives this
* typed handle instead of a raw callback, so the reply surface is an explicit
* send()/close() contract rather than an untyped closure whose shape every call site
* has to infer.
*/
class Response
{
public function __construct(
private readonly Server $server,
private readonly int $fd,
private readonly Counter $bytesSent,
) {
}
/** Write one encoded MQTT packet back to the connection; an empty packet is a no-op. */
public function send(string $packet): void
{
if ($packet === '') {
return;
}
$this->server->send($this->fd, $packet);
$this->bytesSent->add(\strlen($packet));
}
/** Close the connection. */
public function close(): void
{
$this->server->close($this->fd);
}
}
+13 -4
View File
@@ -23,6 +23,9 @@ final class MqttSubscriber
private int $packetId = 0;
/** @var Packet[] PUBLISH packets read before the SUBACK (replay), drained by consume(). */
private array $pending = [];
public function __construct(
private readonly string $host,
private readonly int $port,
@@ -89,7 +92,9 @@ final class MqttSubscriber
return $codes;
}
// Ignore any early PUBLISH replay queued before the SUBACK; consume() reads those.
if ($packet->type === Packet::PUBLISH) {
$this->pending[] = $packet; // replay delivered before the SUBACK; consume() drains it
}
}
}
@@ -107,9 +112,13 @@ final class MqttSubscriber
$received = [];
while (\count($received) < $limit) {
$packet = $this->readPacket($deadline);
if ($packet === null) {
break; // timeout or disconnect
if ($this->pending !== []) {
$packet = \array_shift($this->pending);
} else {
$packet = $this->readPacket($deadline);
if ($packet === null) {
break; // timeout or disconnect
}
}
if ($packet->type !== Packet::PUBLISH) {
continue;
@@ -1,164 +0,0 @@
<?php
declare(strict_types=1);
namespace Tests\Unit\Appwrite\Messaging\Adapter;
use Appwrite\Messaging\Adapter\Mqtt;
use Appwrite\PubSub\Adapter as PubSub;
use PHPUnit\Framework\Attributes\DataProvider;
use PHPUnit\Framework\TestCase;
use Utopia\Mqtt\Connection;
use Utopia\Telemetry\Adapter\None as NoTelemetry;
/**
* Unit tests for the MQTT push adapter's own behaviour: the connection registry
* lifecycle (open/close) and the subscribe/unsubscribe/getSubscribers surface it
* layers over the subscription store — topic matching, QoS, project isolation.
*
* send() publishes onto the Redis 'mqtt' channel and needs a live pool, so it is
* covered by e2e rather than here; these tests never touch pub/sub or the transport.
*/
final class MqttTest extends TestCase
{
private function adapter(): Mqtt
{
// send() (the only path that publishes) is covered by e2e, so a null-object
// pub/sub is enough for the registry and subscription-store behaviour tested here.
$pubsub = new class () implements PubSub {
public function ping($message = null): bool
{
return true;
}
public function subscribe($channels, $callback)
{
}
public function publish($channel, $message)
{
}
};
return new Mqtt(new NoTelemetry(), $pubsub);
}
/** open() the fd, then subscribe it to $topic (the store keys by topic). */
private function join(Mqtt $mqtt, string $projectId, int $fd, string $topic): void
{
$mqtt->open($fd);
$mqtt->subscribe($projectId, $fd, '', [], [$topic]);
}
public function testOpenCreatesAndReusesConnection(): void
{
$mqtt = $this->adapter();
$connection = $mqtt->open(7);
$this->assertInstanceOf(Connection::class, $connection);
$this->assertSame(7, $connection->fd);
$this->assertSame($connection, $mqtt->open(7), 'open() must return the same instance for the same fd');
$this->assertArrayHasKey(7, $mqtt->connections);
}
public function testSubscribeExposesSubscriberWithGrantedQos(): void
{
$mqtt = $this->adapter();
$this->join($mqtt, 'project-a', 11, 'appwrite/push/user-1');
$this->assertSame([11 => 1], $mqtt->getSubscribers('project-a', 'appwrite/push/user-1'));
$this->assertTrue($mqtt->hasSubscriber('project-a', 'appwrite/push/user-1'));
$this->assertFalse($mqtt->hasSubscriber('project-a', 'appwrite/push/user-2'));
}
/**
* Topic-filter matching flows through to the store: + is one level, # is this
* level onward, an exact filter matches only itself.
*
* @return \Iterator<string, array{string, string, bool}>
*/
public static function wildcardProvider(): \Iterator
{
yield 'exact match' => ['test/hello', 'test/hello', true];
yield 'exact miss' => ['test/hello', 'test/world', false];
yield '+ one level' => ['test/+', 'test/hello', true];
yield '+ not multi level' => ['test/+', 'test/a/b', false];
yield '# multi level' => ['test/#', 'test/a/b', true];
yield '# matches parent' => ['test/#', 'test', true];
yield '# not sibling' => ['test/#', 'other', false];
}
#[DataProvider('wildcardProvider')]
public function testTopicFilterMatching(string $filter, string $publishTopic, bool $shouldMatch): void
{
$mqtt = $this->adapter();
$this->join($mqtt, 'project-a', 21, $filter);
$subscribers = $mqtt->getSubscribers('project-a', $publishTopic);
$this->assertSame($shouldMatch, isset($subscribers[21]));
}
public function testSubscribersAreIsolatedPerProject(): void
{
$mqtt = $this->adapter();
$this->join($mqtt, 'project-a', 31, 'shared/topic');
$this->join($mqtt, 'project-b', 32, 'shared/topic');
$this->assertSame([31 => 1], $mqtt->getSubscribers('project-a', 'shared/topic'));
$this->assertSame([32 => 1], $mqtt->getSubscribers('project-b', 'shared/topic'));
}
public function testUnsubscribeSubscriptionRemovesOnlyThatFilter(): void
{
$mqtt = $this->adapter();
$mqtt->open(41);
$mqtt->subscribe('project-a', 41, '', [], ['orders/new']);
$mqtt->subscribe('project-a', 41, '', [], ['alerts/all']);
$mqtt->unsubscribeSubscription(41, 'orders/new');
$this->assertFalse($mqtt->hasSubscriber('project-a', 'orders/new'));
$this->assertSame([41 => 1], $mqtt->getSubscribers('project-a', 'alerts/all'));
}
public function testUnsubscribeByTopic(): void
{
$mqtt = $this->adapter();
$this->join($mqtt, 'project-a', 51, 'news/tech');
$mqtt->unsubscribeSubscription(51, 'news/tech');
$this->assertFalse($mqtt->hasSubscriber('project-a', 'news/tech'));
}
public function testUnsubscribeRemovesEveryFilterForConnection(): void
{
$mqtt = $this->adapter();
$mqtt->open(61);
$mqtt->subscribe('project-a', 61, 'a', [], ['a/one']);
$mqtt->subscribe('project-a', 61, 'b', [], ['b/two']);
$mqtt->unsubscribe(61);
$this->assertFalse($mqtt->hasSubscriber('project-a', 'a/one'));
$this->assertFalse($mqtt->hasSubscriber('project-a', 'b/two'));
}
public function testCloseRemovesSubscriptionsAndForgetsConnection(): void
{
$mqtt = $this->adapter();
$connection = $mqtt->open(71);
$connection->active = true;
$mqtt->subscribe('project-a', 71, 'sub', [], ['live/feed']);
$mqtt->close(71);
$this->assertArrayNotHasKey(71, $mqtt->connections);
$this->assertFalse($mqtt->hasSubscriber('project-a', 'live/feed'));
$this->assertNotSame($connection, $mqtt->open(71), 'a reopened fd is a fresh connection');
}
}