mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Merge pull request #13372 from appwrite/feat-mqtt-push-poc
Appwrite Native Push Broker
This commit is contained in:
30 files changed
+3184
-6
No files matched your search
@@ -60,6 +60,7 @@ _APP_DB_PASS=password
|
||||
_APP_DB_ROOT_PASS=rootsecretpassword
|
||||
_APP_DATABASE_SHARED_NAMESPACE=
|
||||
_APP_DATABASE_SHARED_TABLES=
|
||||
_APP_MQTT_REPLAY_DEPTH=5
|
||||
_APP_DB_ADAPTER_DOCUMENTSDB=mongodb
|
||||
_APP_DB_HOST_DOCUMENTSDB=mongodb
|
||||
_APP_DB_PORT_DOCUMENTSDB=27017
|
||||
|
||||
@@ -240,6 +240,7 @@ jobs:
|
||||
{ name: 'Project', runner: runner4, functional: true, processes: null },
|
||||
{ name: 'Organization', runner: runner4, functional: true, processes: null },
|
||||
{ name: 'Presences', runner: runner4, functional: true, processes: null },
|
||||
{ name: 'Mqtt', runner: runner4, functional: false, processes: 1 },
|
||||
];
|
||||
core.setOutput('services', JSON.stringify(services));
|
||||
|
||||
|
||||
@@ -72,6 +72,7 @@ RUN chmod +x /usr/local/bin/doctor && \
|
||||
chmod +x /usr/local/bin/migrate && \
|
||||
chmod +x /usr/local/bin/stats-resources && \
|
||||
chmod +x /usr/local/bin/realtime && \
|
||||
chmod +x /usr/local/bin/mqtt && \
|
||||
chmod +x /usr/local/bin/schedule && \
|
||||
chmod +x /usr/local/bin/schedule-functions && \
|
||||
chmod +x /usr/local/bin/schedule-executions && \
|
||||
|
||||
@@ -2158,6 +2158,94 @@ return [
|
||||
],
|
||||
],
|
||||
|
||||
'appwritePushLedger' => [
|
||||
'$collection' => ID::custom(DATABASE::METADATA),
|
||||
'$id' => ID::custom('appwritePushLedger'),
|
||||
'name' => 'MQTT Messages',
|
||||
'attributes' => [
|
||||
[
|
||||
'$id' => ID::custom('topic'),
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => Database::LENGTH_KEY,
|
||||
'signed' => true,
|
||||
'required' => true,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => ID::custom('data'),
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => 65535,
|
||||
'signed' => true,
|
||||
'required' => true,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => ['json'],
|
||||
],
|
||||
[
|
||||
'$id' => ID::custom('messageId'),
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => Database::LENGTH_KEY,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
[
|
||||
'$id' => ID::custom('messageInternalId'),
|
||||
'type' => Database::VAR_STRING,
|
||||
'format' => '',
|
||||
'size' => Database::LENGTH_KEY,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
// The per-topic sequence, copied from the topic counter at insert time. qos and
|
||||
// expiry are topic settings now (see the topics collection), not per message.
|
||||
[
|
||||
'$id' => ID::custom('sequence'),
|
||||
'type' => Database::VAR_INTEGER,
|
||||
'format' => '',
|
||||
'size' => 0,
|
||||
'signed' => true,
|
||||
'required' => true,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
],
|
||||
'indexes' => [
|
||||
[
|
||||
'$id' => ID::custom('_key_topic_sequence'),
|
||||
'type' => Database::INDEX_KEY,
|
||||
'attributes' => ['topic', 'sequence'],
|
||||
'lengths' => [],
|
||||
'orders' => [Database::ORDER_ASC, Database::ORDER_ASC],
|
||||
],
|
||||
[
|
||||
'$id' => ID::custom('_key_messageInternalId'),
|
||||
'type' => Database::INDEX_KEY,
|
||||
'attributes' => ['messageInternalId'],
|
||||
'lengths' => [],
|
||||
'orders' => [Database::ORDER_ASC],
|
||||
],
|
||||
[
|
||||
'$id' => ID::custom('_key_message_topic'),
|
||||
'type' => Database::INDEX_UNIQUE,
|
||||
'attributes' => ['messageId', 'topic'],
|
||||
'lengths' => [],
|
||||
'orders' => [Database::ORDER_ASC, Database::ORDER_ASC],
|
||||
],
|
||||
],
|
||||
],
|
||||
|
||||
'topics' => [
|
||||
'$collection' => ID::custom(DATABASE::METADATA),
|
||||
'$id' => ID::custom('topics'),
|
||||
@@ -2240,6 +2328,46 @@ return [
|
||||
'array' => false,
|
||||
'filters' => ['topicSearch'],
|
||||
],
|
||||
// monotonic message counter for this topic. It is the tail sequence, so
|
||||
// a client's backlog depth is this minus the client's cursor.
|
||||
[
|
||||
'$id' => ID::custom('sequence'),
|
||||
'type' => Database::VAR_INTEGER,
|
||||
'format' => '',
|
||||
'size' => 0,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => 0,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
// MQTT quality of service for delivery on this topic: 0 is fire-and-forget
|
||||
// (only clients connected at publish time), 1 persists each message and replays
|
||||
// it when a client reconnects unacknowledged. null lets the subscriber choose.
|
||||
[
|
||||
'$id' => ID::custom('qos'),
|
||||
'type' => Database::VAR_INTEGER,
|
||||
'format' => '',
|
||||
'size' => 0,
|
||||
'signed' => true,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
// message retention in seconds capped at 7 days. A ledger message expires
|
||||
// this long after it is written.
|
||||
[
|
||||
'$id' => ID::custom('expiry'),
|
||||
'type' => Database::VAR_INTEGER,
|
||||
'format' => '',
|
||||
'size' => 0,
|
||||
'signed' => false,
|
||||
'required' => false,
|
||||
'default' => null,
|
||||
'array' => false,
|
||||
'filters' => [],
|
||||
],
|
||||
],
|
||||
|
||||
'indexes' => [
|
||||
|
||||
@@ -334,6 +334,8 @@ const FUNCTION_ALLOWLIST_HEADERS_RESPONSE = ['content-type', 'content-length'];
|
||||
const MESSAGE_TYPE_EMAIL = 'email';
|
||||
const MESSAGE_TYPE_SMS = 'sms';
|
||||
const MESSAGE_TYPE_PUSH = 'push';
|
||||
// Message providers
|
||||
const MESSAGE_PROVIDER_APPWRITE = 'appwrite';
|
||||
// Notification types
|
||||
const NOTIFICATION_TYPE_EMAIL = MESSAGE_TYPE_EMAIL;
|
||||
const NOTIFICATION_TYPE_SMS = MESSAGE_TYPE_SMS;
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
<?php
|
||||
|
||||
use Ahc\Jwt\JWT;
|
||||
use Ahc\Jwt\JWTException;
|
||||
use Appwrite\Utopia\Database\Documents\User;
|
||||
use Utopia\Auth\Hashes\Sha;
|
||||
use Utopia\Auth\Proofs\Token;
|
||||
use Utopia\Auth\Store;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\DI\Container;
|
||||
use Utopia\System\System;
|
||||
|
||||
return function (Container $container): void {
|
||||
$getProject = function (string $projectId, Authorization $authorization) use ($container): Document {
|
||||
if ($projectId === '' || $projectId === 'console') {
|
||||
return new Document(Config::getParam('console'));
|
||||
}
|
||||
|
||||
$getConsoleDB = $container->get('getConsoleDB');
|
||||
$dbForPlatform = $getConsoleDB();
|
||||
$dbForPlatform->setAuthorization($authorization);
|
||||
|
||||
return $authorization->skip(fn () => $dbForPlatform->getDocument('projects', $projectId));
|
||||
};
|
||||
|
||||
$getUser = function (Document $project, string $authMethod, string $credential, Authorization $authorization) use ($container): User {
|
||||
if ($project->isEmpty() || $credential === '') {
|
||||
return new User([]);
|
||||
}
|
||||
|
||||
$authorization->setDefaultStatus(true);
|
||||
|
||||
$getProjectDB = $container->get('getProjectDB');
|
||||
$dbForProject = $getProjectDB($project);
|
||||
$dbForProject->setAuthorization($authorization);
|
||||
|
||||
switch ($authMethod) {
|
||||
case 'appwrite-jwt':
|
||||
$jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 3600, 0);
|
||||
|
||||
try {
|
||||
$payload = $jwt->decode($credential);
|
||||
} catch (JWTException) {
|
||||
return new User([]);
|
||||
}
|
||||
|
||||
$userId = $payload['userId'] ?? '';
|
||||
$sessionId = $payload['sessionId'] ?? '';
|
||||
|
||||
/** @var User $user */
|
||||
$user = $dbForProject->getDocument('users', $userId);
|
||||
|
||||
if (
|
||||
$user->isEmpty()
|
||||
|| $user->getAttribute('status', true) === false // blocked account
|
||||
|| ($sessionId !== '' && !$user->sessionActive($sessionId))
|
||||
) {
|
||||
return new User([]);
|
||||
}
|
||||
|
||||
return $user;
|
||||
|
||||
case 'appwrite-session':
|
||||
$store = new Store();
|
||||
$store->decode($credential);
|
||||
|
||||
$proofForToken = new Token();
|
||||
$proofForToken->setHash(new Sha());
|
||||
|
||||
/** @var User $user */
|
||||
$user = $dbForProject->getDocument('users', $store->getProperty('id', ''));
|
||||
|
||||
if (
|
||||
$user->isEmpty()
|
||||
|| $user->getAttribute('status', true) === false // blocked account
|
||||
|| !$user->sessionVerify($store->getProperty('secret', ''), $proofForToken)
|
||||
) {
|
||||
return new User([]);
|
||||
}
|
||||
|
||||
return $user;
|
||||
|
||||
default:
|
||||
return new User([]);
|
||||
}
|
||||
};
|
||||
|
||||
$container->set('authenticator', function () use ($getProject, $getUser): callable {
|
||||
return function (string $projectId, string $authMethod, string $credential) use ($getProject, $getUser): array {
|
||||
$authorization = new Authorization();
|
||||
|
||||
$project = $getProject($projectId, $authorization);
|
||||
if ($project->isEmpty()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$user = $getUser($project, $authMethod, $credential, $authorization);
|
||||
if ($user->isEmpty()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
'projectId' => $project->getId(),
|
||||
'userId' => $user->getId(),
|
||||
];
|
||||
};
|
||||
}, []);
|
||||
|
||||
$container->set('authorizer', function () use ($getProject, $container): callable {
|
||||
return function (array $identity, string $topic) use ($getProject, $container): bool {
|
||||
$userId = $identity['userId'] ?? '';
|
||||
$projectId = $identity['projectId'] ?? '';
|
||||
if ($userId === '' || $projectId === '') {
|
||||
return false;
|
||||
}
|
||||
|
||||
$authorization = new Authorization();
|
||||
|
||||
$project = $getProject($projectId, $authorization);
|
||||
if ($project->isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$getProjectDB = $container->get('getProjectDB');
|
||||
$dbForProject = $getProjectDB($project);
|
||||
$dbForProject->setAuthorization($authorization);
|
||||
$user = $authorization->skip(fn () => $dbForProject->getDocument('users', $userId));
|
||||
|
||||
// status: true = enabled, false = blocked.
|
||||
return !$user->isEmpty() && $user->getAttribute('status', true) !== false;
|
||||
};
|
||||
}, []);
|
||||
};
|
||||
+245
@@ -0,0 +1,245 @@
|
||||
<?php
|
||||
|
||||
use Appwrite\Messaging\Adapter\Mqtt;
|
||||
use Appwrite\Mqtt\Handler;
|
||||
use Appwrite\PubSub\Adapter\Pool as PubSubPool;
|
||||
use Appwrite\Utopia\Database\Documents\User;
|
||||
use Swoole\Coroutine;
|
||||
use Swoole\Runtime;
|
||||
use Utopia\Cache\Adapter\Pool as CachePool;
|
||||
use Utopia\Cache\Adapter\Sharding;
|
||||
use Utopia\Cache\Cache;
|
||||
use Utopia\Config\Config;
|
||||
use Utopia\Console;
|
||||
use Utopia\Database\Adapter\Pool as DatabasePool;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\DI\Container;
|
||||
use Utopia\DSN\DSN;
|
||||
use Utopia\Mqtt\Adapter;
|
||||
use Utopia\Mqtt\Server;
|
||||
use Utopia\Pools\Group;
|
||||
use Utopia\Registry\Registry;
|
||||
use Utopia\Span\Span;
|
||||
use Utopia\System\System;
|
||||
|
||||
require_once __DIR__ . '/init.php';
|
||||
|
||||
require_once __DIR__ . '/init/span.php';
|
||||
|
||||
/** @var Registry $register */
|
||||
$register = $GLOBALS['register'] ?? throw new \RuntimeException('Registry not initialized');
|
||||
|
||||
$registerConnectionResources ??= require __DIR__ . '/init/mqtt/connection.php';
|
||||
|
||||
Runtime::enableCoroutine(SWOOLE_HOOK_ALL);
|
||||
|
||||
global $container;
|
||||
|
||||
if (!$container->has('pools')) {
|
||||
$container->set('pools', function ($register) {
|
||||
return $register->get('pools');
|
||||
}, ['register']);
|
||||
}
|
||||
|
||||
$container->set('getCache', fn () => function () use ($register): Cache {
|
||||
$ctx = Coroutine::getContext();
|
||||
|
||||
if (isset($ctx['cache'])) {
|
||||
return $ctx['cache'];
|
||||
}
|
||||
|
||||
/** @var Group $pools */
|
||||
$pools = $register->get('pools');
|
||||
|
||||
$adapters = [];
|
||||
foreach (Config::getParam('pools-cache', []) as $value) {
|
||||
$adapters[] = new CachePool($pools->get($value));
|
||||
}
|
||||
|
||||
return $ctx['cache'] = new Cache(new Sharding($adapters));
|
||||
}, []);
|
||||
|
||||
$container->set('getConsoleDB', fn () => function () use ($register, $container): Database {
|
||||
$ctx = Coroutine::getContext();
|
||||
|
||||
if (isset($ctx['dbForPlatform'])) {
|
||||
return $ctx['dbForPlatform'];
|
||||
}
|
||||
|
||||
$getCache = $container->get('getCache');
|
||||
|
||||
/** @var Group $pools */
|
||||
$pools = $register->get('pools');
|
||||
|
||||
$adapter = new DatabasePool($pools->get('console'));
|
||||
$database = new Database($adapter, $getCache());
|
||||
$database
|
||||
->setDatabase(APP_DATABASE)
|
||||
->setNamespace('_console')
|
||||
->setMetadata('host', \gethostname())
|
||||
->setMetadata('project', '_console');
|
||||
$database->setDocumentType('users', User::class);
|
||||
|
||||
return $ctx['dbForPlatform'] = $database;
|
||||
}, []);
|
||||
|
||||
$container->set('getProjectDB', fn () => function (Document $project) use ($register, $container): Database {
|
||||
$ctx = Coroutine::getContext();
|
||||
|
||||
if (!isset($ctx['dbForProject'])) {
|
||||
$ctx['dbForProject'] = [];
|
||||
}
|
||||
|
||||
if (isset($ctx['dbForProject'][$project->getSequence()])) {
|
||||
return $ctx['dbForProject'][$project->getSequence()];
|
||||
}
|
||||
|
||||
if ($project->isEmpty() || $project->getId() === 'console') {
|
||||
$getConsoleDB = $container->get('getConsoleDB');
|
||||
|
||||
return $getConsoleDB();
|
||||
}
|
||||
|
||||
$getCache = $container->get('getCache');
|
||||
|
||||
/** @var Group $pools */
|
||||
$pools = $register->get('pools');
|
||||
|
||||
try {
|
||||
$dsn = new DSN($project->getAttribute('database'));
|
||||
} catch (\InvalidArgumentException) {
|
||||
$dsn = new DSN('mysql://' . $project->getAttribute('database'));
|
||||
}
|
||||
|
||||
$adapter = new DatabasePool($pools->get($dsn->getHost()));
|
||||
$database = new Database($adapter, $getCache());
|
||||
|
||||
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
|
||||
|
||||
if (\in_array($dsn->getHost(), $sharedTables)) {
|
||||
$projectCollections = Config::getParam('collections', [])['projects'] ?? [];
|
||||
$globalCollections = array_keys($projectCollections);
|
||||
$globalCollections[] = 'audit';
|
||||
|
||||
$database
|
||||
->setSharedTables(true)
|
||||
->setGlobalCollections($globalCollections)
|
||||
->setTenant($project->getSequence())
|
||||
->setNamespace($dsn->getParam('namespace'));
|
||||
} else {
|
||||
$database
|
||||
->setSharedTables(false)
|
||||
->setTenant(null)
|
||||
->setNamespace('_' . $project->getSequence());
|
||||
}
|
||||
|
||||
$database
|
||||
->setDatabase(APP_DATABASE)
|
||||
->setMetadata('host', \gethostname())
|
||||
->setMetadata('project', $project->getId());
|
||||
$database->setDocumentType('users', User::class);
|
||||
|
||||
return $ctx['dbForProject'][$project->getSequence()] = $database;
|
||||
}, []);
|
||||
|
||||
$container->set('getPlanForUser', fn () => fn (Document $project, string $userId): int => (int) System::getEnv('_APP_MQTT_REPLAY_DEPTH', '5'), []);
|
||||
|
||||
$container->set('getRedis', fn () => function (): \Redis {
|
||||
$ctx = Coroutine::getContext();
|
||||
|
||||
if (isset($ctx['redis'])) {
|
||||
return $ctx['redis'];
|
||||
}
|
||||
|
||||
$host = System::getEnv('_APP_REDIS_HOST', 'localhost');
|
||||
$port = System::getEnv('_APP_REDIS_PORT', 6379);
|
||||
$pass = System::getEnv('_APP_REDIS_PASS', '');
|
||||
|
||||
$redis = new \Redis();
|
||||
@$redis->pconnect($host, (int)$port);
|
||||
if ($pass) {
|
||||
$redis->auth($pass);
|
||||
}
|
||||
$redis->setOption(\Redis::OPT_READ_TIMEOUT, -1);
|
||||
|
||||
return $ctx['redis'] = $redis;
|
||||
}, []);
|
||||
|
||||
// Register the CONNECT authenticator and per-SUBSCRIBE authorizer on the global container
|
||||
// (see app/init/mqtt/connection.php). The CONNECT/SUBSCRIBE handlers inject them by name,
|
||||
// resolved through the packet container that inherits from this one.
|
||||
$registerConnectionResources($container);
|
||||
|
||||
/** @var \Utopia\Telemetry\Adapter $telemetry */
|
||||
$telemetry = $container->get('telemetry');
|
||||
|
||||
$maxPacketSize = (int) System::getEnv('_APP_MQTT_MAX_PACKET_SIZE', '64000');
|
||||
$adapter = new Adapter\Swoole([
|
||||
new Adapter\Swoole\Tcp('0.0.0.0', 1883, $maxPacketSize),
|
||||
new Adapter\Swoole\WebSocket('0.0.0.0', (int) System::getEnv('_APP_MQTT_WS_PORT', '8083'), $maxPacketSize),
|
||||
], workers: 1);
|
||||
|
||||
$mqtt = new Mqtt($telemetry, new PubSubPool($register->get('pools')->get('pubsub')));
|
||||
|
||||
// The broker owns framing, decoding, dispatch, per-version encoding, packet ids, the QoS
|
||||
// handshake, keep-alive reaping and subscription matching. Appwrite policy lives in the Handler.
|
||||
$handler = new Handler($container, $mqtt, $container->get('getCache'), $container->get('getPlanForUser'));
|
||||
$server = new Server($adapter, $handler);
|
||||
$server->setTelemetry($telemetry); // broker owns connection/packet/subscription metrics
|
||||
|
||||
$server->onStart(fn () => print("MQTT broker started\n"));
|
||||
$server->error(fn (\Throwable $error, string $action) => Console::error("MQTT {$action} error: " . $error->getMessage()));
|
||||
|
||||
// Server-initiated delivery: bridge the Redis 'mqtt' firehose to this worker's local subscribers.
|
||||
// Appwrite clients never PUBLISH; messages are produced by the Messaging worker onto the channel.
|
||||
$server->onWorkerStart(function (int $workerId) use ($server, $handler, $mqtt, $register): void {
|
||||
go(function () use ($server, $handler, $mqtt, $register): void {
|
||||
$attempts = 0;
|
||||
while ($attempts < 60) {
|
||||
try {
|
||||
$pubsub = new PubSubPool($register->get('pools')->get('pubsub'));
|
||||
|
||||
if ($pubsub->ping(true)) {
|
||||
$attempts = 0;
|
||||
}
|
||||
|
||||
$pubsub->subscribe(['mqtt'], function (mixed $redis, string $channel, string $payload) use ($server, $handler, $mqtt): void {
|
||||
$event = json_decode($payload, true);
|
||||
if (!\is_array($event)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$projectId = (string) ($event['project'] ?? '');
|
||||
$topic = (string) ($event['topic'] ?? '');
|
||||
$qos = (int) ($event['qos'] ?? 0);
|
||||
$sequence = (int) ($event['sequence'] ?? 0);
|
||||
$message = base64_decode((string) ($event['payload'] ?? ''));
|
||||
|
||||
$span = Span::init('mqtt.deliver');
|
||||
$span->set('project.id', $projectId);
|
||||
$span->set('mqtt.topic', $topic);
|
||||
$span->set('mqtt.qos', $qos);
|
||||
$span->set('mqtt.is_broker', true);
|
||||
|
||||
$delivered = $handler->deliver($server, $projectId, $topic, $message, $qos, $sequence);
|
||||
|
||||
$span->set('mqtt.subscribers', $delivered);
|
||||
if ($delivered === 0) {
|
||||
$mqtt->messagesDropped->add(1, ['reason' => 'no_subscriber']);
|
||||
}
|
||||
|
||||
$span->finish();
|
||||
});
|
||||
} catch (\Throwable $error) {
|
||||
$attempts++;
|
||||
Console::error('MQTT pub/sub connection error: ' . $error->getMessage());
|
||||
sleep(DATABASE_RECONNECT_SLEEP);
|
||||
}
|
||||
}
|
||||
|
||||
Console::error('Failed to maintain MQTT pub/sub subscription');
|
||||
});
|
||||
});
|
||||
|
||||
$server->start();
|
||||
@@ -57,6 +57,10 @@
|
||||
{
|
||||
"type": "vcs",
|
||||
"url": "https://github.com/utopia-php/vcs"
|
||||
},
|
||||
{
|
||||
"type": "vcs",
|
||||
"url": "https://github.com/utopia-php/mqtt"
|
||||
}
|
||||
],
|
||||
"require": {
|
||||
@@ -99,6 +103,7 @@
|
||||
"utopia-php/locale": "0.8.*",
|
||||
"utopia-php/messaging": "^2.1",
|
||||
"utopia-php/migration": "^2.0.0",
|
||||
"utopia-php/mqtt": "0.0.*",
|
||||
"utopia-php/platform": "1.0.0-rc20",
|
||||
"utopia-php/pools": "2.*",
|
||||
"utopia-php/span": "^4.2",
|
||||
|
||||
Generated
+68
-1
@@ -4,7 +4,7 @@
|
||||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "067e4c5b691b20b06213bc3c9382a6c9",
|
||||
"content-hash": "8a4455dd9986b24d5ab9788f1d97a2bf",
|
||||
"packages": [
|
||||
{
|
||||
"name": "adhocore/jwt",
|
||||
@@ -4144,6 +4144,73 @@
|
||||
},
|
||||
"time": "2026-09-09T05:42:37+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/mqtt",
|
||||
"version": "0.0.1",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/utopia-php/mqtt.git",
|
||||
"reference": "d43d7e7f4b0c0854c5fd16c7ef085284d84b0d69"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/utopia-php/mqtt/zipball/d43d7e7f4b0c0854c5fd16c7ef085284d84b0d69",
|
||||
"reference": "d43d7e7f4b0c0854c5fd16c7ef085284d84b0d69",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=8.2",
|
||||
"utopia-php/telemetry": "^0.4"
|
||||
},
|
||||
"require-dev": {
|
||||
"laravel/pint": "^1.15 <1.21",
|
||||
"phpstan/phpstan": "^1.12",
|
||||
"phpunit/phpunit": "^10.5",
|
||||
"swoole/ide-helper": "5.1.2"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Utopia\\Mqtt\\": "src/Mqtt"
|
||||
}
|
||||
},
|
||||
"autoload-dev": {
|
||||
"psr-4": {
|
||||
"Utopia\\Mqtt\\Tests\\": "tests"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"test": [
|
||||
"./vendor/bin/phpunit"
|
||||
],
|
||||
"lint": [
|
||||
"./vendor/bin/pint --test --preset=psr12 src tests"
|
||||
],
|
||||
"format": [
|
||||
"./vendor/bin/pint --preset=psr12 src tests"
|
||||
],
|
||||
"check": [
|
||||
"./vendor/bin/phpstan analyse --level max src tests"
|
||||
]
|
||||
},
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"description": "A simple abstraction for building MQTT 5.0 brokers.",
|
||||
"keywords": [
|
||||
"broker",
|
||||
"framework",
|
||||
"mqtt",
|
||||
"php",
|
||||
"upf",
|
||||
"utopia"
|
||||
],
|
||||
"support": {
|
||||
"source": "https://github.com/utopia-php/mqtt/tree/0.0.1",
|
||||
"issues": "https://github.com/utopia-php/mqtt/issues"
|
||||
},
|
||||
"time": "2026-09-18T11:27:21+00:00"
|
||||
},
|
||||
{
|
||||
"name": "utopia-php/openapi",
|
||||
"version": "0.2.4",
|
||||
|
||||
@@ -51,6 +51,18 @@ services:
|
||||
- ./src:/usr/src/code/src
|
||||
ports:
|
||||
- 9502:80
|
||||
appwrite-mqtt:
|
||||
build:
|
||||
context: .
|
||||
target: development
|
||||
args:
|
||||
DEBUG: false
|
||||
TESTING: true
|
||||
VERSION: dev
|
||||
image: appwrite-dev
|
||||
volumes:
|
||||
- ./app:/usr/src/code/app
|
||||
- ./src:/usr/src/code/src
|
||||
appwrite-worker:
|
||||
build:
|
||||
context: .
|
||||
@@ -504,6 +516,12 @@ services:
|
||||
- --providers.docker.constraints=Label(`traefik.constraint-label-stack`,`appwrite`)
|
||||
- --entrypoints.appwrite_web.address=:80
|
||||
- --entrypoints.appwrite_websecure.address=:443
|
||||
# This dev override replaces the base command wholesale, so the MQTT entrypoints
|
||||
# must be repeated here or Traefik won't listen on :8883 (mqtts) and :8084 (wss)
|
||||
# and the appwrite-mqtt broker is unreachable (the port publishes survive via
|
||||
# ports concatenation, but have no entrypoint behind them without this).
|
||||
- --entrypoints.appwrite_mqtt.address=:8883
|
||||
- --entrypoints.appwrite_mqtt_ws.address=:8084
|
||||
- --log.level=DEBUG
|
||||
- --api.insecure=true
|
||||
- --accesslog=true
|
||||
|
||||
@@ -21,9 +21,13 @@ services:
|
||||
- --providers.docker.constraints=Label(`traefik.constraint-label-stack`,`appwrite`)
|
||||
- --entrypoints.appwrite_web.address=:80
|
||||
- --entrypoints.appwrite_websecure.address=:443
|
||||
- --entrypoints.appwrite_mqtt.address=:8883
|
||||
- --entrypoints.appwrite_mqtt_ws.address=:8084
|
||||
ports:
|
||||
- ${_APP_HTTP_PORT:-80}:80
|
||||
- ${_APP_HTTPS_PORT:-443}:443
|
||||
- ${_APP_MQTT_PORT:-8883}:8883
|
||||
- ${_APP_MQTT_WSS_PORT:-8084}:8084
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- appwrite-config:/storage/config:ro
|
||||
@@ -355,6 +359,77 @@ services:
|
||||
- _APP_VECTORSDB
|
||||
- _APP_EMBEDDING
|
||||
|
||||
appwrite-mqtt:
|
||||
entrypoint: mqtt
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-file: "5"
|
||||
max-size: 10m
|
||||
container_name: appwrite-mqtt
|
||||
image: ${_APP_IMAGE:-appwrite/appwrite}:${_APP_VERSION:-latest}
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "php", "-r", "exit(@fsockopen('127.0.0.1', 1883) ? 0 : 1);"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 20s
|
||||
labels:
|
||||
- traefik.enable=true
|
||||
- traefik.constraint-label-stack=appwrite
|
||||
- traefik.docker.network=appwrite
|
||||
# TLS is terminated here (mqtts, :8883) and forwarded as plaintext MQTT to :1883,
|
||||
# mirroring how realtime's wss is terminated at the appwrite_websecure entrypoint.
|
||||
- traefik.tcp.routers.appwrite_mqtt.entrypoints=appwrite_mqtt
|
||||
- traefik.tcp.routers.appwrite_mqtt.rule=HostSNI(`*`)
|
||||
- traefik.tcp.routers.appwrite_mqtt.tls=true
|
||||
- traefik.tcp.routers.appwrite_mqtt.service=appwrite_mqtt
|
||||
- traefik.tcp.services.appwrite_mqtt.loadbalancer.server.port=1883
|
||||
# MQTT-over-WebSocket for browser clients: wss://<host>:8084 on its own TLS entrypoint
|
||||
# (a dedicated port, not a URL path), the WebSocket sibling of the mqtts TCP router. The
|
||||
# port identifies the broker, so a single push host serves both mqtts:// (:8883) and
|
||||
# wss:// (:8084). TLS-only, so the reusable CONNECT credential never crosses the wire in
|
||||
# the clear; TLS is terminated here and forwarded to the broker's WebSocket listener (:8083).
|
||||
- traefik.http.routers.appwrite_mqtt_ws.entrypoints=appwrite_mqtt_ws
|
||||
- traefik.http.routers.appwrite_mqtt_ws.rule=PathPrefix(`/`)
|
||||
- traefik.http.routers.appwrite_mqtt_ws.service=appwrite_mqtt_ws
|
||||
- traefik.http.routers.appwrite_mqtt_ws.tls=true
|
||||
- traefik.http.services.appwrite_mqtt_ws.loadbalancer.server.port=8083
|
||||
networks:
|
||||
- appwrite
|
||||
environment:
|
||||
- _APP_ENV
|
||||
- _APP_WORKER_PER_CORE
|
||||
- _APP_OPTIONS_ABUSE
|
||||
- _APP_OPTIONS_ROUTER_PROTECTION
|
||||
- _APP_REALTIME_TAIL_RATE
|
||||
- _APP_OPENSSL_KEY_V1
|
||||
- _APP_REDIS_HOST
|
||||
- _APP_REDIS_PORT
|
||||
- _APP_REDIS_USER
|
||||
- _APP_REDIS_PASS
|
||||
- _APP_DB_ADAPTER
|
||||
- _APP_DB_HOST
|
||||
- _APP_DB_PORT
|
||||
- _APP_DB_SCHEMA
|
||||
- _APP_DB_USER
|
||||
- _APP_DB_PASS
|
||||
- _APP_DB_ADAPTER_VECTORSDB
|
||||
- _APP_DB_HOST_VECTORSDB
|
||||
- _APP_DB_PORT_VECTORSDB
|
||||
- _APP_DB_SCHEMA_VECTORSDB
|
||||
- _APP_DB_USER_VECTORSDB
|
||||
- _APP_DB_PASS_VECTORSDB
|
||||
- _APP_LOGGING_CONFIG
|
||||
- _APP_LOGGING_FORMAT
|
||||
- _APP_LOGGING_CONFIG_REALTIME
|
||||
- _APP_DATABASE_SHARED_TABLES
|
||||
- _APP_MQTT_REPLAY_DEPTH
|
||||
- _APP_MQTT_WS_PORT
|
||||
- _APP_MQTT_MAX_PACKET_SIZE
|
||||
- _APP_POOL_ADAPTER=swoole
|
||||
|
||||
appwrite-worker:
|
||||
entrypoint: worker
|
||||
restart: unless-stopped
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
Create a new Appwrite push provider.
|
||||
@@ -0,0 +1 @@
|
||||
Update an Appwrite push provider by its unique ID.
|
||||
@@ -51,5 +51,8 @@
|
||||
<file>./tests/e2e/Services/Functions/FunctionsCustomServerTest.php</file>
|
||||
<file>./tests/e2e/Services/Functions/FunctionsCustomClientTest.php</file>
|
||||
</testsuite>
|
||||
<testsuite name="mqtt">
|
||||
<directory>./tests/e2e/Services/Mqtt</directory>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
</phpunit>
|
||||
@@ -0,0 +1,90 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Messaging\Adapter;
|
||||
|
||||
use Appwrite\Messaging\Adapter as MessagingAdapter;
|
||||
use Appwrite\PubSub\Adapter as PubSub;
|
||||
use Utopia\Telemetry\Adapter as Telemetry;
|
||||
use Utopia\Telemetry\Counter;
|
||||
use Utopia\Telemetry\Histogram;
|
||||
|
||||
/**
|
||||
* The MQTT producer: publishes messages onto the 'mqtt' pub/sub channel for the broker workers
|
||||
* to fan out, and owns the broker telemetry instruments. Connection state, keep-alive and the
|
||||
* subscription index now live inside the broker (utopia-php/mqtt Server), so they are gone here.
|
||||
*/
|
||||
class Mqtt extends MessagingAdapter
|
||||
{
|
||||
private const CHANNEL = 'mqtt';
|
||||
|
||||
public readonly Counter $messagesPublished;
|
||||
public readonly Counter $messagesDelivered;
|
||||
public readonly Counter $messagesDropped;
|
||||
public readonly Counter $messagesAcked;
|
||||
public readonly Counter $pubacksReceived;
|
||||
public readonly Counter $reauth;
|
||||
public readonly Histogram $authDuration;
|
||||
public readonly Histogram $connectionDuration;
|
||||
public readonly Histogram $messageSize;
|
||||
|
||||
public function __construct(Telemetry $telemetry, private readonly PubSub $pubsub)
|
||||
{
|
||||
$this->messagesPublished = $telemetry->createCounter('mqtt.messages.published');
|
||||
$this->messagesDelivered = $telemetry->createCounter('mqtt.messages.delivered');
|
||||
$this->messagesDropped = $telemetry->createCounter('mqtt.messages.dropped');
|
||||
$this->messagesAcked = $telemetry->createCounter('mqtt.messages.acked');
|
||||
$this->pubacksReceived = $telemetry->createCounter('mqtt.puback.received');
|
||||
$this->reauth = $telemetry->createCounter('mqtt.reauth');
|
||||
$this->authDuration = $telemetry->createHistogram('mqtt.auth.duration', 's');
|
||||
$this->connectionDuration = $telemetry->createHistogram('mqtt.connection.duration', 's');
|
||||
$this->messageSize = $telemetry->createHistogram('mqtt.message.size', 'By');
|
||||
}
|
||||
|
||||
/**
|
||||
* Publish a message onto the 'mqtt' pub/sub channel for fan-out. Every broker worker listens
|
||||
* on that channel and delivers to its own local subscribers. The binary MQTT payload and its
|
||||
* QoS ride in $options ($options['payload'], $options['qos'], $options['sequence']); $channels
|
||||
* holds the topics. The payload is base64-encoded to survive the JSON envelope.
|
||||
*
|
||||
* @param array<mixed> $payload unused for MQTT (payload is binary, see $options)
|
||||
* @param array<int, string> $events ignored
|
||||
* @param array<int, string> $channels topics to publish to
|
||||
* @param array<int, string> $roles ignored
|
||||
* @param array{payload?: string, qos?: int, sequence?: int} $options
|
||||
*/
|
||||
public function send(string $projectId, array $payload, array $events, array $channels, array $roles, array $options = []): void
|
||||
{
|
||||
$message = $options['payload'] ?? '';
|
||||
$qos = $options['qos'] ?? 0;
|
||||
$sequence = (int) ($options['sequence'] ?? 0);
|
||||
|
||||
$this->messageSize->record(\strlen($message));
|
||||
|
||||
foreach ($channels as $topic) {
|
||||
$this->messagesPublished->add(1, ['qos' => $qos]);
|
||||
$this->pubsub->publish(self::CHANNEL, (string) json_encode([
|
||||
'project' => $projectId,
|
||||
'topic' => $topic,
|
||||
'qos' => $qos,
|
||||
'sequence' => $sequence,
|
||||
'payload' => base64_encode($message),
|
||||
]));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Part of the messaging-adapter contract but unused for MQTT: subscriptions are owned by the
|
||||
* broker (utopia-php/mqtt Server records them from the SUBACK the Handler returns).
|
||||
*
|
||||
* @param array<int, string> $roles
|
||||
* @param array<int, string> $channels
|
||||
* @param array<int, mixed> $queryGroup
|
||||
*/
|
||||
public function subscribe(string $projectId, mixed $identifier, string $subscriptionId, array $roles, array $channels, array $queryGroup = []): void
|
||||
{
|
||||
}
|
||||
|
||||
public function unsubscribe(mixed $identifier): void
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Messaging\Adapter\Push;
|
||||
|
||||
use Appwrite\Messaging\Adapter\Mqtt;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Messaging\Adapter\Push as PushAdapter;
|
||||
use Utopia\Messaging\Messages\Push as PushMessage;
|
||||
use Utopia\Messaging\Priority;
|
||||
use Utopia\Messaging\Response;
|
||||
use Utopia\Mqtt\Packet;
|
||||
use Utopia\Telemetry\Adapter as Telemetry;
|
||||
|
||||
class Appwrite extends PushAdapter
|
||||
{
|
||||
protected const NAME = 'Appwrite';
|
||||
|
||||
/**
|
||||
* @param Mqtt $broker the broker adapter used for internal pub/sub fan-out
|
||||
* @param Database $dbForProject the project database holding the topics and ledger
|
||||
* @param string $projectId the project every topic and publish is scoped to
|
||||
* @param string $messageId source campaign message id (client-side dedup key)
|
||||
* @param ?string $messageInternalId source message internal id (correlation)
|
||||
* @param int $qos delivery QoS for the fan-out envelope
|
||||
*/
|
||||
public function __construct(
|
||||
private readonly Mqtt $broker,
|
||||
private readonly Database $dbForProject,
|
||||
private readonly string $projectId,
|
||||
private readonly string $messageId,
|
||||
private readonly ?string $messageInternalId,
|
||||
private readonly int $qos = Packet::QOS_1,
|
||||
?Telemetry $telemetry = null,
|
||||
) {
|
||||
parent::__construct($telemetry);
|
||||
}
|
||||
|
||||
public function getName(): string
|
||||
{
|
||||
return static::NAME;
|
||||
}
|
||||
|
||||
public function getMaxMessagesPerRequest(): int
|
||||
{
|
||||
return 5000;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
protected function process(PushMessage $message): array
|
||||
{
|
||||
$payload = $this->buildPayload($message);
|
||||
$response = new Response($this->getType());
|
||||
|
||||
foreach ($message->getTo() as $topic) {
|
||||
try {
|
||||
$sequence = $this->persist($topic, $payload);
|
||||
$this->broker->send(
|
||||
$this->projectId,
|
||||
[],
|
||||
[],
|
||||
[$topic],
|
||||
[],
|
||||
['payload' => $payload, 'qos' => $this->qos, 'sequence' => $sequence],
|
||||
);
|
||||
|
||||
$response->incrementDeliveredTo();
|
||||
$response->addResult($topic);
|
||||
} catch (\Throwable $error) {
|
||||
$response->addResult($topic, $error->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
return $response->toArray();
|
||||
}
|
||||
|
||||
/**
|
||||
* Append the notification to the ledger and return its per-topic sequence. Idempotent
|
||||
* on (messageId, topic): a retry after a transient publish failure reuses the existing
|
||||
* row and sequence instead of incrementing the topic counter or inserting a duplicate.
|
||||
*
|
||||
* The increment and insert run in one transaction, opened by locking the topic row
|
||||
* (getDocument FOR UPDATE). A concurrent attempt for the same message blocks on that
|
||||
* lock, then re-reads the ledger row this transaction committed rather than racing a
|
||||
* second increment — so the topic sequence can't advance without a matching row, and
|
||||
* two attempts can't open a gap. The unique index on (messageId, topic) remains as a
|
||||
* correctness backstop.
|
||||
*/
|
||||
private function persist(string $topic, string $payload): int
|
||||
{
|
||||
$authorization = $this->dbForProject->getAuthorization();
|
||||
|
||||
$existing = $this->findLedger($authorization, $topic);
|
||||
if (!$existing->isEmpty()) {
|
||||
return (int) $existing->getAttribute('sequence');
|
||||
}
|
||||
|
||||
return (int) $authorization->skip(
|
||||
fn () => $this->dbForProject->withTransaction(function () use ($topic, $payload, $authorization): int {
|
||||
// Lock the topic row so a concurrent attempt for the same message waits here.
|
||||
$this->dbForProject->getDocument('topics', $topic, forUpdate: true);
|
||||
|
||||
// Re-check under the lock: a racing attempt may have persisted it already.
|
||||
$existing = $this->findLedger($authorization, $topic);
|
||||
if (!$existing->isEmpty()) {
|
||||
return (int) $existing->getAttribute('sequence');
|
||||
}
|
||||
|
||||
$sequence = (int) $this->dbForProject
|
||||
->increaseDocumentAttribute('topics', $topic, 'sequence', 1)
|
||||
->getAttribute('sequence');
|
||||
|
||||
$this->dbForProject->createDocument('appwritePushLedger', new Document([
|
||||
'$id' => ID::unique(),
|
||||
'topic' => $topic,
|
||||
'data' => $payload,
|
||||
'messageId' => $this->messageId,
|
||||
'messageInternalId' => $this->messageInternalId,
|
||||
'sequence' => $sequence,
|
||||
]));
|
||||
|
||||
return $sequence;
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
/** The existing ledger row for this campaign message on a topic, or an empty document. */
|
||||
private function findLedger(Authorization $authorization, string $topic): Document
|
||||
{
|
||||
return $authorization->skip(
|
||||
fn () => $this->dbForProject->findOne('appwritePushLedger', [
|
||||
Query::equal('messageId', [$this->messageId]),
|
||||
Query::equal('topic', [$topic]),
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the notification envelope delivered to devices. Mirrors the shape
|
||||
* FCM and APNS expose so SDK consumers render a consistent payload.
|
||||
*/
|
||||
private function buildPayload(PushMessage $message): string
|
||||
{
|
||||
$envelope = [];
|
||||
|
||||
if ($message->getTitle() !== null) {
|
||||
$envelope['notification']['title'] = $message->getTitle();
|
||||
}
|
||||
if ($message->getBody() !== null) {
|
||||
$envelope['notification']['body'] = $message->getBody();
|
||||
}
|
||||
if ($message->getImage() !== null) {
|
||||
$envelope['notification']['image'] = $message->getImage();
|
||||
}
|
||||
if ($message->getIcon() !== null) {
|
||||
$envelope['notification']['icon'] = $message->getIcon();
|
||||
}
|
||||
if ($message->getColor() !== null) {
|
||||
$envelope['notification']['color'] = $message->getColor();
|
||||
}
|
||||
if ($message->getSound() !== null) {
|
||||
$envelope['notification']['sound'] = $message->getSound();
|
||||
}
|
||||
if ($message->getTag() !== null) {
|
||||
$envelope['notification']['tag'] = $message->getTag();
|
||||
}
|
||||
if ($message->getBadge() !== null) {
|
||||
$envelope['notification']['badge'] = $message->getBadge();
|
||||
}
|
||||
if ($message->getAction() !== null) {
|
||||
$envelope['notification']['action'] = $message->getAction();
|
||||
}
|
||||
if ($message->getContentAvailable() !== null) {
|
||||
$envelope['notification']['contentAvailable'] = $message->getContentAvailable();
|
||||
}
|
||||
if ($message->getCritical() !== null) {
|
||||
$envelope['notification']['critical'] = $message->getCritical();
|
||||
}
|
||||
if ($message->getData() !== null) {
|
||||
$envelope['data'] = $message->getData();
|
||||
}
|
||||
if ($message->getPriority() instanceof Priority) {
|
||||
$envelope['priority'] = match ($message->getPriority()) {
|
||||
Priority::HIGH => 'high',
|
||||
Priority::NORMAL => 'normal',
|
||||
};
|
||||
}
|
||||
|
||||
$json = json_encode($envelope, JSON_UNESCAPED_SLASHES);
|
||||
if ($json === false) {
|
||||
throw new \RuntimeException('Failed to encode push payload: ' . json_last_error_msg());
|
||||
}
|
||||
|
||||
return $json;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Mqtt;
|
||||
|
||||
use Appwrite\Messaging\Adapter\Mqtt;
|
||||
use Appwrite\Utopia\Database\Documents\User;
|
||||
use Utopia\Abuse\Abuse;
|
||||
use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Helpers\Role;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\DI\Container;
|
||||
use Utopia\Mqtt\Connection;
|
||||
use Utopia\Mqtt\Handler as MqttHandler;
|
||||
use Utopia\Mqtt\Packet;
|
||||
use Utopia\Mqtt\Packet\Auth;
|
||||
use Utopia\Mqtt\Packet\Connack;
|
||||
use Utopia\Mqtt\Packet\Connect;
|
||||
use Utopia\Mqtt\Packet\Disconnect;
|
||||
use Utopia\Mqtt\Packet\Puback;
|
||||
use Utopia\Mqtt\Packet\Publish;
|
||||
use Utopia\Mqtt\Packet\Suback;
|
||||
use Utopia\Mqtt\Packet\Subscribe;
|
||||
use Utopia\Mqtt\Packet\Unsuback;
|
||||
use Utopia\Mqtt\Packet\Unsubscribe;
|
||||
use Utopia\Mqtt\Server;
|
||||
use Utopia\Span\Span;
|
||||
use Utopia\System\System;
|
||||
|
||||
class Handler implements MqttHandler
|
||||
{
|
||||
private const REPLAY_TTL = 3600;
|
||||
|
||||
public function __construct(
|
||||
private readonly Container $container,
|
||||
private readonly Mqtt $mqtt,
|
||||
private readonly \Closure $getCache,
|
||||
private readonly \Closure $planForUser,
|
||||
) {
|
||||
}
|
||||
|
||||
public function onConnect(Connect $connect, Connection $connection): Connack|Auth
|
||||
{
|
||||
$authMethod = $connect->authMethod ?? '';
|
||||
|
||||
$connection->prefix = $connect->userProperties()['projectId'] ?? '';
|
||||
Span::add('project.id', $connection->prefix);
|
||||
Span::add('mqtt.clean_start', $connect->cleanStart);
|
||||
Span::add('mqtt.auth_method', $authMethod);
|
||||
|
||||
$authenticator = $this->container->get('authenticator');
|
||||
|
||||
$start = microtime(true);
|
||||
$identity = $authenticator($connection->prefix, $authMethod, $connect->authData ?? '');
|
||||
$this->mqtt->authDuration->record(microtime(true) - $start);
|
||||
|
||||
if ($identity === []) {
|
||||
Span::add('mqtt.result', 'rejected');
|
||||
return Connack::refuse(Connack::NOT_AUTHORIZED);
|
||||
}
|
||||
|
||||
$connection->identity = $identity;
|
||||
Span::add('user.id', $identity['userId'] ?? '');
|
||||
|
||||
// Rate-limit CONNECT per user (keyed on userId until the infra surfaces client IP).
|
||||
if (System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') {
|
||||
$getRedis = $this->container->get('getRedis');
|
||||
$timeLimit = new TimeLimitRedis('mqtt:connect:{userId}', 128, 60, $getRedis());
|
||||
$timeLimit->setParam('{userId}', $identity['userId'] ?? '');
|
||||
|
||||
if ((new Abuse($timeLimit))->check()) {
|
||||
Span::add('mqtt.result', 'abuse');
|
||||
return Connack::refuse(Connack::QUOTA_EXCEEDED);
|
||||
}
|
||||
}
|
||||
|
||||
// The per-device session anchor: the client id, or an account-level fallback so reconnects
|
||||
// resume the same session (the offline-replay cursor is keyed on it).
|
||||
$clientId = $connect->clientId;
|
||||
if ($clientId === '') {
|
||||
$clientId = 'custom_' . $connection->prefix . '_' . ($identity['userId'] ?? '');
|
||||
}
|
||||
$connection->setClientId($clientId);
|
||||
Span::add('mqtt.client_id', $connection->getClientId());
|
||||
|
||||
return Connack::accept();
|
||||
}
|
||||
|
||||
public function onAuthenticate(Auth $auth, Connection $connection): Connack|Auth|Disconnect
|
||||
{
|
||||
$projectId = $auth->userProperties()['projectId'] ?? '';
|
||||
|
||||
$identity = [];
|
||||
if ($projectId !== '' && $projectId === $connection->prefix) {
|
||||
$authenticator = $this->container->get('authenticator');
|
||||
$start = microtime(true);
|
||||
$identity = $authenticator($projectId, $auth->method, $auth->data);
|
||||
$this->mqtt->authDuration->record(microtime(true) - $start);
|
||||
}
|
||||
|
||||
// Reauth only refreshes the credential for the identity resolved at CONNECT — it must not
|
||||
// switch user or project, which would leak the previous user's subscriptions.
|
||||
if ($identity === [] || ($identity['userId'] ?? '') !== ($connection->identity['userId'] ?? '')) {
|
||||
$this->mqtt->reauth->add(1, ['result' => 'rejected']);
|
||||
Span::add('mqtt.result', 'rejected');
|
||||
return Disconnect::refuse(Disconnect::NOT_AUTHORIZED);
|
||||
}
|
||||
|
||||
$connection->identity = $identity;
|
||||
$this->mqtt->reauth->add(1, ['result' => 'success']);
|
||||
Span::add('mqtt.result', 'reauthenticated');
|
||||
|
||||
return Auth::success($auth->method);
|
||||
}
|
||||
|
||||
public function onSubscribe(Subscribe $subscribe, Connection $connection): Suback
|
||||
{
|
||||
$authorizer = $this->container->get('authorizer');
|
||||
$identity = $connection->identity;
|
||||
|
||||
$projectDB = $this->getProjectDB($connection->prefix);
|
||||
|
||||
$allowed = [];
|
||||
foreach ($subscribe->filters() as $filter) {
|
||||
if (!isset($allowed[$filter->topic]) && ($authorizer === null || $authorizer($identity, $filter->topic))) {
|
||||
$allowed[$filter->topic] = true;
|
||||
}
|
||||
}
|
||||
|
||||
$authorization = new Authorization();
|
||||
$projectDB->setAuthorization($authorization);
|
||||
|
||||
/** @var User $user */
|
||||
$user = $authorization->skip(fn () => $projectDB->getDocument('users', $identity['userId'] ?? ''));
|
||||
$roles = $user->getRoles($authorization);
|
||||
|
||||
$topicsById = [];
|
||||
if ($allowed !== []) {
|
||||
$documents = $authorization->skip(fn () => $projectDB->find('topics', [
|
||||
Query::equal('$id', array_keys($allowed)),
|
||||
Query::limit(\count($allowed)),
|
||||
]));
|
||||
foreach ($documents as $document) {
|
||||
$topicsById[$document->getId()] = $document;
|
||||
}
|
||||
}
|
||||
|
||||
$suback = new Suback();
|
||||
$grantedTopics = [];
|
||||
|
||||
foreach ($subscribe->filters() as $filter) {
|
||||
Span::add('mqtt.topic', $filter->topic);
|
||||
|
||||
$document = $topicsById[$filter->topic] ?? null;
|
||||
if (!isset($allowed[$filter->topic]) || $document === null) {
|
||||
$suback->deny();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!$this->authorizedForTopic($document->getAttribute('subscribe', []) ?? [], $roles)) {
|
||||
$suback->deny();
|
||||
continue;
|
||||
}
|
||||
|
||||
$topicQos = $document->getAttribute('qos');
|
||||
$grantedQos = min($filter->qos, $topicQos === null ? Packet::QOS_1 : (int) $topicQos);
|
||||
|
||||
$suback->grant($grantedQos);
|
||||
$grantedTopics[$filter->topic] = [$document, $grantedQos];
|
||||
}
|
||||
|
||||
if ($grantedTopics !== []) {
|
||||
$this->replayBacklog($connection, $projectDB, $grantedTopics);
|
||||
}
|
||||
|
||||
return $suback;
|
||||
}
|
||||
|
||||
public function onUnsubscribe(Unsubscribe $unsubscribe, Connection $connection): Unsuback
|
||||
{
|
||||
// The broker has already dropped the filters from its subscription index.
|
||||
$unsuback = new Unsuback();
|
||||
foreach ($unsubscribe->filters() as $filter) {
|
||||
$unsuback->success();
|
||||
}
|
||||
|
||||
return $unsuback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Appwrite is server -> client push: clients never PUBLISH, so an inbound one is ignored.
|
||||
* Server-initiated fan-out goes through deliver() instead.
|
||||
*
|
||||
* @param iterable<array{0: Connection, 1: int}> $subscribers
|
||||
*/
|
||||
public function onPublish(Publish $publish, Connection $connection, iterable $subscribers): void
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Fan a message out to a topic's local subscribers, each at the QoS granted to that
|
||||
* subscription (clamped by the message QoS). Returns the number delivered.
|
||||
*/
|
||||
public function deliver(Server $server, string $projectId, string $topic, string $message, int $qos, int $sequence): int
|
||||
{
|
||||
$delivered = 0;
|
||||
|
||||
foreach ($server->subscribers($projectId, $topic) as [$connection, $grantedQos]) {
|
||||
$deliveryQos = min($qos, $grantedQos);
|
||||
$connection->publish($topic, $message, qos: $deliveryQos, sequence: $sequence);
|
||||
$this->mqtt->messagesDelivered->add(1, ['qos' => $deliveryQos]);
|
||||
$delivered++;
|
||||
}
|
||||
|
||||
return $delivered;
|
||||
}
|
||||
|
||||
public function onPuback(Puback $puback, Connection $connection): void
|
||||
{
|
||||
$delivery = $connection->acknowledge($puback->packetId);
|
||||
if ($delivery === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Span::add('mqtt.topic', $delivery['topic']);
|
||||
Span::add('mqtt.sequence', $delivery['sequence']);
|
||||
$this->mqtt->messagesAcked->add(1);
|
||||
$this->mqtt->pubacksReceived->add(1);
|
||||
|
||||
$cache = ($this->getCache)();
|
||||
$key = $this->cursorKey($connection);
|
||||
$topic = $delivery['topic'];
|
||||
$cursor = (int) $delivery['cursor'];
|
||||
|
||||
$current = $cache->loadMany($key, self::REPLAY_TTL, [$topic]);
|
||||
if ($cursor > (int) ($current[$topic]['sequence'] ?? -1)) {
|
||||
$cache->saveMany($key, [$topic => ['sequence' => $cursor]], 0);
|
||||
}
|
||||
}
|
||||
|
||||
public function onDisconnect(?Disconnect $disconnect, Connection $connection): void
|
||||
{
|
||||
// The broker already removed the connection, its subscriptions and keep-alive slot, and
|
||||
// records the active-connections gauge itself. Record the connection lifetime for
|
||||
// accepted sessions.
|
||||
if (($connection->identity['userId'] ?? '') !== '') {
|
||||
$this->mqtt->connectionDuration->record(microtime(true) - $connection->openedAt);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the subscriber's roles satisfy a topic's subscribe roles. No configured roles (or an
|
||||
* explicit `any`) means the topic is open to everyone.
|
||||
*
|
||||
* @param array<int, string> $topicRoles
|
||||
* @param array<int, string> $userRoles
|
||||
*/
|
||||
private function authorizedForTopic(array $topicRoles, array $userRoles): bool
|
||||
{
|
||||
if ($topicRoles === [] || \in_array(Role::any()->toString(), $topicRoles, true)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return \array_intersect($topicRoles, $userRoles) !== [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Replay the offline backlog for freshly granted QoS 1 topics: resume each topic's cursor,
|
||||
* then re-deliver stored messages via the connection (DUP flagged), tagging each with its
|
||||
* durable sequence so a later PUBACK maps back.
|
||||
*
|
||||
* @param array<string, array{0: Document, 1: int}> $grantedTopics filter => [document, grantedQos]
|
||||
*/
|
||||
private function replayBacklog(Connection $connection, Database $projectDB, array $grantedTopics): void
|
||||
{
|
||||
$cache = ($this->getCache)();
|
||||
$key = $this->cursorKey($connection);
|
||||
$maxDepth = max(1, ($this->planForUser)(new Document(['$id' => $connection->prefix]), $connection->identity['userId'] ?? ''));
|
||||
|
||||
$expiry = 0;
|
||||
foreach ($grantedTopics as [$document]) {
|
||||
$expiry = max($expiry, (int) $document->getAttribute('expiry', 0));
|
||||
}
|
||||
if ($expiry <= 0) {
|
||||
$expiry = self::REPLAY_TTL;
|
||||
}
|
||||
|
||||
if ($connection->cleanStart) {
|
||||
$cache->purge($key);
|
||||
}
|
||||
|
||||
$cursors = $cache->loadMany($key, $expiry);
|
||||
$persist = [];
|
||||
|
||||
foreach ($grantedTopics as $filter => [$document, $grantedQos]) {
|
||||
$tail = (int) $document->getAttribute('sequence', 0);
|
||||
$from = (int) ($cursors[$filter]['sequence'] ?? $tail);
|
||||
$persist[$filter] = ['sequence' => $from];
|
||||
|
||||
if ($grantedQos < 1 || $tail <= $from) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$connection->resume($filter, $from);
|
||||
|
||||
$start = max($from + 1, $tail - $maxDepth + 1);
|
||||
$messages = $projectDB->getAuthorization()->skip(fn () => $projectDB->find('appwritePushLedger', [
|
||||
Query::equal('topic', [$filter]),
|
||||
Query::greaterThanEqual('sequence', $start),
|
||||
Query::orderAsc('sequence'),
|
||||
Query::limit($maxDepth),
|
||||
]));
|
||||
|
||||
foreach ($messages as $message) {
|
||||
$stored = $message->getAttribute('data');
|
||||
$data = \is_string($stored) ? $stored : (string) json_encode($stored);
|
||||
$connection->publish($filter, $data, qos: 1, dup: true, sequence: (int) $message->getAttribute('sequence'));
|
||||
}
|
||||
}
|
||||
|
||||
if ($persist !== []) {
|
||||
$cache->saveMany($key, $persist, $expiry);
|
||||
}
|
||||
}
|
||||
|
||||
private function cursorKey(Connection $connection): string
|
||||
{
|
||||
return 'appwrite:push:cursor:' . $connection->prefix
|
||||
. ':' . ($connection->identity['userId'] ?? '')
|
||||
. ':' . $connection->getClientId();
|
||||
}
|
||||
|
||||
private function getProjectDB(string $projectId): Database
|
||||
{
|
||||
$getConsoleDB = $this->container->get('getConsoleDB');
|
||||
$getProjectDB = $this->container->get('getProjectDB');
|
||||
|
||||
$consoleDB = $getConsoleDB();
|
||||
$authorization = new Authorization();
|
||||
$consoleDB->setAuthorization($authorization);
|
||||
$project = $authorization->skip(fn () => $consoleDB->getDocument('projects', $projectId));
|
||||
|
||||
return $getProjectDB($project);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\Messaging\Http\Providers\Appwrite;
|
||||
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\SDK\AuthType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Database\Validator\CustomId;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Exception\Duplicate as DuplicateException;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\Validator\Boolean;
|
||||
use Utopia\Validator\Nullable;
|
||||
use Utopia\Validator\Range;
|
||||
use Utopia\Validator\Text;
|
||||
|
||||
class Create extends Action
|
||||
{
|
||||
use HTTP;
|
||||
|
||||
public static function getName()
|
||||
{
|
||||
return 'createAppwriteProvider';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->setHttpMethod(Action::HTTP_REQUEST_METHOD_POST)
|
||||
->setHttpPath('/v1/messaging/providers/appwrite')
|
||||
->desc('Create Appwrite provider')
|
||||
->groups(['api', 'messaging'])
|
||||
->label('audits.event', 'provider.create')
|
||||
->label('audits.resource', 'provider/{response.$id}')
|
||||
->label('event', 'providers.[providerId].create')
|
||||
->label('scope', 'providers.write')
|
||||
->label('resourceType', RESOURCE_TYPE_PROVIDERS)
|
||||
->label('sdk', new Method(
|
||||
namespace: 'messaging',
|
||||
group: 'providers',
|
||||
name: 'createAppwriteProvider',
|
||||
description: '/docs/references/messaging/create-appwrite-provider.md',
|
||||
auth: [AuthType::ADMIN, AuthType::KEY],
|
||||
responses: [
|
||||
new SDKResponse(
|
||||
code: Response::STATUS_CODE_CREATED,
|
||||
model: Response::MODEL_PROVIDER,
|
||||
)
|
||||
]
|
||||
))
|
||||
->param('providerId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Provider ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.', false, ['dbForProject'])
|
||||
->param('name', '', new Text(128), 'Provider name.')
|
||||
->param('enabled', null, new Nullable(new Boolean()), 'Set as enabled.', true)
|
||||
->param('qos', null, new Nullable(new Range(0, 1)), 'Default QoS for topics on this provider (0 or 1). Null lets the subscriber choose.', true)
|
||||
->param('expiry', null, new Nullable(new Range(0, 604800)), 'Default message retention in seconds for offline delivery. Max 7 days (604800).', true)
|
||||
->inject('queueForEvents')
|
||||
->inject('dbForProject')
|
||||
->inject('response')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
public function action(string $providerId, string $name, ?bool $enabled, ?int $qos, ?int $expiry, Event $queueForEvents, Database $dbForProject, Response $response)
|
||||
{
|
||||
$providerId = $providerId == 'unique()' ? ID::unique() : $providerId;
|
||||
|
||||
$provider = new Document([
|
||||
'$id' => $providerId,
|
||||
'name' => $name,
|
||||
'provider' => MESSAGE_PROVIDER_APPWRITE,
|
||||
'type' => MESSAGE_TYPE_PUSH,
|
||||
'enabled' => $enabled ?? true,
|
||||
'credentials' => [],
|
||||
'options' => [
|
||||
'qos' => $qos,
|
||||
'expiry' => $expiry,
|
||||
],
|
||||
]);
|
||||
|
||||
try {
|
||||
$provider = $dbForProject->createDocument('providers', $provider);
|
||||
} catch (DuplicateException) {
|
||||
throw new Exception(Exception::PROVIDER_ALREADY_EXISTS);
|
||||
}
|
||||
|
||||
$queueForEvents
|
||||
->setParam('providerId', $provider->getId());
|
||||
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_CREATED)
|
||||
->dynamic($provider, Response::MODEL_PROVIDER);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\Messaging\Http\Providers\Appwrite;
|
||||
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\SDK\AuthType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Request;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Validator\UID;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\Validator\Boolean;
|
||||
use Utopia\Validator\Nullable;
|
||||
use Utopia\Validator\Range;
|
||||
use Utopia\Validator\Text;
|
||||
|
||||
class Update extends Action
|
||||
{
|
||||
use HTTP;
|
||||
|
||||
public static function getName()
|
||||
{
|
||||
return 'updateAppwriteProvider';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->setHttpMethod(Action::HTTP_REQUEST_METHOD_PATCH)
|
||||
->setHttpPath('/v1/messaging/providers/appwrite/:providerId')
|
||||
->desc('Update Appwrite provider')
|
||||
->groups(['api', 'messaging'])
|
||||
->label('audits.event', 'provider.update')
|
||||
->label('audits.resource', 'provider/{response.$id}')
|
||||
->label('event', 'providers.[providerId].update')
|
||||
->label('scope', 'providers.write')
|
||||
->label('resourceType', RESOURCE_TYPE_PROVIDERS)
|
||||
->label('sdk', new Method(
|
||||
namespace: 'messaging',
|
||||
group: 'providers',
|
||||
name: 'updateAppwriteProvider',
|
||||
description: '/docs/references/messaging/update-appwrite-provider.md',
|
||||
auth: [AuthType::ADMIN, AuthType::KEY],
|
||||
responses: [
|
||||
new SDKResponse(
|
||||
code: Response::STATUS_CODE_OK,
|
||||
model: Response::MODEL_PROVIDER,
|
||||
)
|
||||
]
|
||||
))
|
||||
->param('providerId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Provider ID.', false, ['dbForProject'])
|
||||
->param('name', '', new Text(128), 'Provider name.', true)
|
||||
->param('enabled', null, new Nullable(new Boolean()), 'Set as enabled.', true)
|
||||
->param('qos', null, new Nullable(new Range(0, 1)), 'Default QoS for topics on this provider (0 or 1). Null lets the subscriber choose.', true)
|
||||
->param('expiry', null, new Nullable(new Range(0, 604800)), 'Default message retention in seconds for offline delivery. Max 7 days (604800).', true)
|
||||
->inject('request')
|
||||
->inject('queueForEvents')
|
||||
->inject('dbForProject')
|
||||
->inject('response')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
public function action(string $providerId, string $name, ?bool $enabled, ?int $qos, ?int $expiry, Request $request, Event $queueForEvents, Database $dbForProject, Response $response)
|
||||
{
|
||||
$provider = $dbForProject->getDocument('providers', $providerId);
|
||||
|
||||
if ($provider->isEmpty()) {
|
||||
throw new Exception(Exception::PROVIDER_NOT_FOUND);
|
||||
}
|
||||
|
||||
if ($provider->getAttribute('provider') !== MESSAGE_PROVIDER_APPWRITE) {
|
||||
throw new Exception(Exception::PROVIDER_INCORRECT_TYPE);
|
||||
}
|
||||
|
||||
if (!empty($name)) {
|
||||
$provider->setAttribute('name', $name);
|
||||
}
|
||||
|
||||
if (!\is_null($enabled)) {
|
||||
$provider->setAttribute('enabled', $enabled);
|
||||
}
|
||||
|
||||
// qos and expiry are nullable defaults: an explicit null resets them (subscriber
|
||||
// chooses the QoS, no expiry). An omitted param and an explicit null both arrive as
|
||||
// null, so key on the request body's presence rather than the value.
|
||||
$params = $request->getParams();
|
||||
if (\array_key_exists('qos', $params) || \array_key_exists('expiry', $params)) {
|
||||
$options = $provider->getAttribute('options', []);
|
||||
if (\array_key_exists('qos', $params)) {
|
||||
$options['qos'] = $qos;
|
||||
}
|
||||
if (\array_key_exists('expiry', $params)) {
|
||||
$options['expiry'] = $expiry;
|
||||
}
|
||||
$provider->setAttribute('options', $options);
|
||||
}
|
||||
|
||||
$provider = $dbForProject->updateDocument('providers', $provider->getId(), $provider);
|
||||
|
||||
$queueForEvents
|
||||
->setParam('providerId', $provider->getId());
|
||||
|
||||
$response
|
||||
->dynamic($provider, Response::MODEL_PROVIDER);
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,8 @@ use Utopia\Database\Helpers\ID;
|
||||
use Utopia\Database\Validator\Roles;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\Validator\Nullable;
|
||||
use Utopia\Validator\Range;
|
||||
use Utopia\Validator\Text;
|
||||
|
||||
class Create extends Action
|
||||
@@ -56,13 +58,15 @@ class Create extends Action
|
||||
->param('topicId', '', fn (Database $dbForProject) => new CustomId(false, $dbForProject->getAdapter()->getMaxUIDLength()), 'Topic ID. Choose a custom Topic ID or a new Topic ID.', false, ['dbForProject'])
|
||||
->param('name', '', new Text(128), 'Topic Name.')
|
||||
->param('subscribe', [Role::users()], new Roles(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of role strings with subscribe permission. By default all users are granted with any subscribe permission. [learn more about roles](https://appwrite.io/docs/permissions#permission-roles). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each 64 characters long.', true)
|
||||
->param('qos', null, new Nullable(new Range(0, 1)), 'MQTT delivery quality of service for this topic. 0 is fire-and-forget (delivered only to clients connected at publish time). 1 persists each message and replays it when a client reconnects without having acknowledged it. Null lets the subscriber choose.', true)
|
||||
->param('expiry', null, new Nullable(new Range(0, 604800)), 'Message retention in seconds for offline delivery. Max 7 days (604800).', true)
|
||||
->inject('queueForEvents')
|
||||
->inject('dbForProject')
|
||||
->inject('response')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
public function action(string $topicId, string $name, array $subscribe, Event $queueForEvents, Database $dbForProject, Response $response)
|
||||
public function action(string $topicId, string $name, array $subscribe, ?int $qos, ?int $expiry, Event $queueForEvents, Database $dbForProject, Response $response)
|
||||
{
|
||||
$topicId = $topicId == 'unique()' ? ID::unique() : $topicId;
|
||||
|
||||
@@ -70,6 +74,8 @@ class Create extends Action
|
||||
'$id' => $topicId,
|
||||
'name' => $name,
|
||||
'subscribe' => $subscribe,
|
||||
'qos' => $qos,
|
||||
'expiry' => $expiry,
|
||||
]);
|
||||
|
||||
try {
|
||||
|
||||
@@ -7,6 +7,7 @@ use Appwrite\Extend\Exception;
|
||||
use Appwrite\SDK\AuthType;
|
||||
use Appwrite\SDK\Method;
|
||||
use Appwrite\SDK\Response as SDKResponse;
|
||||
use Appwrite\Utopia\Request;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Validator\Roles;
|
||||
@@ -14,6 +15,7 @@ use Utopia\Database\Validator\UID;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\Validator\Nullable;
|
||||
use Utopia\Validator\Range;
|
||||
use Utopia\Validator\Text;
|
||||
|
||||
class Update extends Action
|
||||
@@ -53,13 +55,16 @@ class Update extends Action
|
||||
->param('topicId', '', fn (Database $dbForProject) => new UID($dbForProject->getAdapter()->getMaxUIDLength()), 'Topic ID.', false, ['dbForProject'])
|
||||
->param('name', null, new Nullable(new Text(128)), 'Topic Name.', true)
|
||||
->param('subscribe', null, new Nullable(new Roles(APP_LIMIT_ARRAY_PARAMS_SIZE)), 'An array of role strings with subscribe permission. By default all users are granted with any subscribe permission. [learn more about roles](https://appwrite.io/docs/permissions#permission-roles). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each 64 characters long.', true)
|
||||
->param('qos', null, new Nullable(new Range(0, 1)), 'MQTT delivery quality of service for this topic. 0 is fire-and-forget (delivered only to clients connected at publish time). 1 persists each message and replays it when a client reconnects without having acknowledged it. Null lets the subscriber choose.', true)
|
||||
->param('expiry', null, new Nullable(new Range(0, 604800)), 'Message retention in seconds for offline delivery. Max 7 days (604800).', true)
|
||||
->inject('request')
|
||||
->inject('queueForEvents')
|
||||
->inject('dbForProject')
|
||||
->inject('response')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
public function action(string $topicId, ?string $name, ?array $subscribe, Event $queueForEvents, Database $dbForProject, Response $response)
|
||||
public function action(string $topicId, ?string $name, ?array $subscribe, ?int $qos, ?int $expiry, Request $request, Event $queueForEvents, Database $dbForProject, Response $response)
|
||||
{
|
||||
$topic = $dbForProject->getDocument('topics', $topicId);
|
||||
|
||||
@@ -75,6 +80,20 @@ class Update extends Action
|
||||
$topic->setAttribute('subscribe', $subscribe);
|
||||
}
|
||||
|
||||
// qos and expiry are nullable settings: an explicit null resets them to the
|
||||
// documented default (subscriber-chosen QoS, no expiry). Since an omitted param
|
||||
// and an explicit null both arrive as null, honour the key's presence in the
|
||||
// request body rather than the value, so a reset isn't silently ignored.
|
||||
$params = $request->getParams();
|
||||
|
||||
if (\array_key_exists('qos', $params)) {
|
||||
$topic->setAttribute('qos', $qos);
|
||||
}
|
||||
|
||||
if (\array_key_exists('expiry', $params)) {
|
||||
$topic->setAttribute('expiry', $expiry);
|
||||
}
|
||||
|
||||
$topic = $dbForProject->updateDocument('topics', $topicId, $topic);
|
||||
|
||||
$queueForEvents
|
||||
|
||||
@@ -14,6 +14,8 @@ use Appwrite\Platform\Modules\Messaging\Http\Messages\Targets\XList as ListTarge
|
||||
use Appwrite\Platform\Modules\Messaging\Http\Messages\XList as ListMessages;
|
||||
use Appwrite\Platform\Modules\Messaging\Http\Providers\Apns\Create as CreateApnsProvider;
|
||||
use Appwrite\Platform\Modules\Messaging\Http\Providers\Apns\Update as UpdateApnsProvider;
|
||||
use Appwrite\Platform\Modules\Messaging\Http\Providers\Appwrite\Create as CreateAppwriteProvider;
|
||||
use Appwrite\Platform\Modules\Messaging\Http\Providers\Appwrite\Update as UpdateAppwriteProvider;
|
||||
use Appwrite\Platform\Modules\Messaging\Http\Providers\Delete as DeleteProvider;
|
||||
use Appwrite\Platform\Modules\Messaging\Http\Providers\Fcm\Create as CreateFcmProvider;
|
||||
use Appwrite\Platform\Modules\Messaging\Http\Providers\Fcm\Update as UpdateFcmProvider;
|
||||
@@ -81,6 +83,8 @@ class Http extends Service
|
||||
$this->addAction(UpdateFcmProvider::getName(), new UpdateFcmProvider());
|
||||
$this->addAction(CreateApnsProvider::getName(), new CreateApnsProvider());
|
||||
$this->addAction(UpdateApnsProvider::getName(), new UpdateApnsProvider());
|
||||
$this->addAction(CreateAppwriteProvider::getName(), new CreateAppwriteProvider());
|
||||
$this->addAction(UpdateAppwriteProvider::getName(), new UpdateAppwriteProvider());
|
||||
$this->addAction(ListProviders::getName(), new ListProviders());
|
||||
$this->addAction(GetProvider::getName(), new GetProvider());
|
||||
$this->addAction(DeleteProvider::getName(), new DeleteProvider());
|
||||
|
||||
@@ -344,6 +344,7 @@ class Deletes extends Action
|
||||
$this->deleteExpiredChallenges($project, $getProjectDB);
|
||||
$this->deleteExpiredTransactions($project, $getProjectDB);
|
||||
$this->deleteExpiredPresences($project, $getProjectDB, $publisherForUsage);
|
||||
$this->deleteExpiredPushLedger($project, $getProjectDB);
|
||||
$this->deleteOldDeployments($publisherForDeletes, $project, $getProjectDB);
|
||||
$this->updateProcessingMigrations($project, $getProjectDB);
|
||||
break;
|
||||
@@ -1918,6 +1919,26 @@ class Deletes extends Action
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The push ledger (appwritePushLedger) is the append-only record of QoS 1 push
|
||||
* messages the MQTT broker keeps so it can replay any a client missed while offline.
|
||||
* Replay only ever reaches back one week, so entries older than that are dead weight
|
||||
* and are pruned here, mirroring how expired presences are cleaned up.
|
||||
*/
|
||||
private function deleteExpiredPushLedger(Document $project, callable $getProjectDB): void
|
||||
{
|
||||
Console::info('Delete expired push ledger messages');
|
||||
|
||||
$dbForProject = $getProjectDB($project);
|
||||
$expired = DateTime::addSeconds(new \DateTime(), -1 * 60 * 60 * 24 * 7);
|
||||
|
||||
$dbForProject->deleteDocuments('appwritePushLedger', [
|
||||
Query::lessThan('$createdAt', $expired),
|
||||
], onError: function (Throwable $th) {
|
||||
// Swallow errors (e.g. projects without the push ledger collection).
|
||||
});
|
||||
}
|
||||
|
||||
private function deleteExpiredPresences(Document $project, callable $getProjectDB, UsagePublisher $publisherForUsage): void
|
||||
{
|
||||
Console::info('Delete expired presences');
|
||||
|
||||
@@ -4,8 +4,11 @@ namespace Appwrite\Platform\Workers;
|
||||
|
||||
use Appwrite\Event\Message\Usage;
|
||||
use Appwrite\Event\Publisher\Usage as UsagePublisher;
|
||||
use Appwrite\Messaging\Adapter\Mqtt;
|
||||
use Appwrite\Messaging\Adapter\Push\Appwrite as AppwritePush;
|
||||
use Appwrite\Messaging\Status as MessageStatus;
|
||||
use Appwrite\OpenSSL\OpenSSL;
|
||||
use Appwrite\PubSub\Adapter\Pool as PubSubPool;
|
||||
use Appwrite\Usage\Context as UsageContext;
|
||||
use Utopia\Compression\Algorithms\GZIP;
|
||||
use Utopia\Compression\Algorithms\Zstd;
|
||||
@@ -44,7 +47,9 @@ use Utopia\Messaging\Messages\Email\Attachment;
|
||||
use Utopia\Messaging\Messages\Push;
|
||||
use Utopia\Messaging\Messages\SMS;
|
||||
use Utopia\Messaging\Priority;
|
||||
use Utopia\Mqtt\Packet;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Pools\Group;
|
||||
use Utopia\Psr7\Stream;
|
||||
use Utopia\Queue\Message;
|
||||
use Utopia\Span\Span;
|
||||
@@ -62,6 +67,8 @@ class Messaging extends Action
|
||||
|
||||
private Telemetry $telemetry;
|
||||
|
||||
private Group $pools;
|
||||
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'messaging';
|
||||
@@ -80,6 +87,7 @@ class Messaging extends Action
|
||||
->inject('deviceForFiles')
|
||||
->inject('publisherForUsage')
|
||||
->inject('telemetry')
|
||||
->inject('pools')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
@@ -90,6 +98,7 @@ class Messaging extends Action
|
||||
* @param Device $deviceForFiles
|
||||
* @param UsagePublisher $publisherForUsage
|
||||
* @param Telemetry $telemetry
|
||||
* @param Group $pools
|
||||
* @return void
|
||||
* @throws \Exception
|
||||
*/
|
||||
@@ -99,9 +108,11 @@ class Messaging extends Action
|
||||
Database $dbForProject,
|
||||
Device $deviceForFiles,
|
||||
UsagePublisher $publisherForUsage,
|
||||
Telemetry $telemetry
|
||||
Telemetry $telemetry,
|
||||
Group $pools
|
||||
): void {
|
||||
$this->telemetry = $telemetry;
|
||||
$this->pools = $pools;
|
||||
$payload = $message->getPayload();
|
||||
|
||||
if (empty($payload)) {
|
||||
@@ -248,7 +259,7 @@ class Messaging extends Action
|
||||
|
||||
$adapter = match ($resolvedProviderType) {
|
||||
MESSAGE_TYPE_SMS => $this->getSmsAdapter($provider),
|
||||
MESSAGE_TYPE_PUSH => $this->getPushAdapter($provider),
|
||||
MESSAGE_TYPE_PUSH => $this->getPushAdapter($provider, $dbForProject, $project, $message),
|
||||
MESSAGE_TYPE_EMAIL => $this->getEmailAdapter($provider),
|
||||
default => throw new \Exception('Provider with the requested ID is of the incorrect type')
|
||||
};
|
||||
@@ -902,7 +913,7 @@ class Messaging extends Action
|
||||
return $adapter;
|
||||
}
|
||||
|
||||
protected function getPushAdapter(Document $provider): ?PushAdapter
|
||||
protected function getPushAdapter(Document $provider, Database $dbForProject, Document $project, Document $message): ?PushAdapter
|
||||
{
|
||||
$credentials = $provider->getAttribute('credentials');
|
||||
$options = $provider->getAttribute('options');
|
||||
@@ -917,6 +928,14 @@ class Messaging extends Action
|
||||
$options['sandbox'] ?? false
|
||||
),
|
||||
'fcm' => new FCM(\json_encode($credentials['serviceAccountJSON'])),
|
||||
'appwrite' => new AppwritePush(
|
||||
new Mqtt($this->telemetry, new PubSubPool($this->pools->get('pubsub'))),
|
||||
$dbForProject,
|
||||
$project->getId(),
|
||||
$message->getId(),
|
||||
$message->getSequence(),
|
||||
$options['qos'] ?? Packet::QOS_1,
|
||||
),
|
||||
default => null
|
||||
};
|
||||
|
||||
|
||||
@@ -58,6 +58,20 @@ class Topic extends Model
|
||||
'default' => ['users'],
|
||||
'example' => 'users',
|
||||
'array' => true,
|
||||
])
|
||||
->addRule('qos', [
|
||||
'type' => self::TYPE_INTEGER,
|
||||
'description' => 'MQTT QoS for delivery on this topic. Null lets the subscriber choose their level.',
|
||||
'default' => null,
|
||||
'example' => 1,
|
||||
'required' => false,
|
||||
])
|
||||
->addRule('expiry', [
|
||||
'type' => self::TYPE_INTEGER,
|
||||
'description' => 'Message retention in seconds for offline delivery.',
|
||||
'default' => null,
|
||||
'example' => 604800,
|
||||
'required' => false,
|
||||
]);
|
||||
}
|
||||
|
||||
|
||||
@@ -861,6 +861,214 @@ trait MessagingBase
|
||||
}
|
||||
}
|
||||
|
||||
public function testCreateAppwriteProvider(): void
|
||||
{
|
||||
// Test for SUCCESS: the built-in Appwrite (MQTT) push provider needs no credentials.
|
||||
$provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/appwrite', [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
], [
|
||||
'providerId' => ID::unique(),
|
||||
'name' => 'Appwrite1',
|
||||
'enabled' => true,
|
||||
'qos' => 1,
|
||||
'expiry' => 3600,
|
||||
]);
|
||||
|
||||
$this->assertEquals(201, $provider['headers']['status-code']);
|
||||
$this->assertEquals('Appwrite1', $provider['body']['name']);
|
||||
$this->assertEquals('appwrite', $provider['body']['provider']);
|
||||
$this->assertEquals('push', $provider['body']['type']);
|
||||
$this->assertTrue($provider['body']['enabled']);
|
||||
$this->assertEquals(1, $provider['body']['options']['qos']);
|
||||
$this->assertEquals(3600, $provider['body']['options']['expiry']);
|
||||
}
|
||||
|
||||
public function testUpdateAppwriteProvider(): void
|
||||
{
|
||||
$provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/appwrite', [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
], [
|
||||
'providerId' => ID::unique(),
|
||||
'name' => 'Appwrite-before',
|
||||
'enabled' => true,
|
||||
]);
|
||||
$this->assertEquals(201, $provider['headers']['status-code']);
|
||||
|
||||
// Test for SUCCESS: name and enabled are updated.
|
||||
$response = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/appwrite/' . $provider['body']['$id'], [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
], [
|
||||
'name' => 'Appwrite-after',
|
||||
'enabled' => false,
|
||||
'qos' => 0,
|
||||
'expiry' => 7200,
|
||||
]);
|
||||
|
||||
$this->assertEquals(200, $response['headers']['status-code']);
|
||||
$this->assertEquals('Appwrite-after', $response['body']['name']);
|
||||
$this->assertFalse($response['body']['enabled']);
|
||||
$this->assertEquals(0, $response['body']['options']['qos']);
|
||||
$this->assertEquals(7200, $response['body']['options']['expiry']);
|
||||
|
||||
// Test for FAILURE: an unknown provider id is not found.
|
||||
$missing = $this->client->call(Client::METHOD_PATCH, '/messaging/providers/appwrite/' . ID::unique(), [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
], [
|
||||
'name' => 'Nope',
|
||||
]);
|
||||
|
||||
$this->assertEquals(404, $missing['headers']['status-code']);
|
||||
}
|
||||
|
||||
public function testCreateTopicWithMqttSettings(): void
|
||||
{
|
||||
// Test for SUCCESS: a topic carries its MQTT qos and expiry settings.
|
||||
$topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
], [
|
||||
'topicId' => ID::unique(),
|
||||
'name' => 'mqtt-settings-topic',
|
||||
'qos' => 1,
|
||||
'expiry' => 3600,
|
||||
]);
|
||||
|
||||
$this->assertEquals(201, $topic['headers']['status-code']);
|
||||
$this->assertEquals(1, $topic['body']['qos']);
|
||||
$this->assertEquals(3600, $topic['body']['expiry']);
|
||||
|
||||
// Test for SUCCESS: the settings update.
|
||||
$response = $this->client->call(Client::METHOD_PATCH, '/messaging/topics/' . $topic['body']['$id'], [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
], [
|
||||
'qos' => 0,
|
||||
'expiry' => 7200,
|
||||
]);
|
||||
|
||||
$this->assertEquals(200, $response['headers']['status-code']);
|
||||
$this->assertEquals(0, $response['body']['qos']);
|
||||
$this->assertEquals(7200, $response['body']['expiry']);
|
||||
}
|
||||
|
||||
/**
|
||||
* The built-in Appwrite (MQTT) push provider is server-published only: a message
|
||||
* campaign is the single publish path, never an outside MQTT PUBLISH. This drives
|
||||
* that path end to end through the public API — provider, topic, subscriber, then
|
||||
* a push message the worker fans out into the broker — and asserts delivery.
|
||||
*
|
||||
* Unlike the FCM/APNS push tests, this needs no external credentials or DSN, so it
|
||||
* always runs: the broker is Appwrite's own.
|
||||
*/
|
||||
public function testSendPushViaAppwriteProvider(): void
|
||||
{
|
||||
// The built-in push provider: no credentials, enabled by default here.
|
||||
$provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/appwrite', [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
], [
|
||||
'providerId' => ID::unique(),
|
||||
'name' => 'Appwrite-send',
|
||||
'enabled' => true,
|
||||
]);
|
||||
$this->assertEquals(201, $provider['headers']['status-code']);
|
||||
|
||||
// The topic doubles as the MQTT topic devices subscribe to; its id is the
|
||||
// delivery topic the broker fans out on.
|
||||
$topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
], [
|
||||
'topicId' => ID::unique(),
|
||||
'name' => 'appwrite-push-topic',
|
||||
'qos' => 1,
|
||||
]);
|
||||
$this->assertEquals(201, $topic['headers']['status-code']);
|
||||
$topicId = $topic['body']['$id'];
|
||||
|
||||
$user = $this->client->call(Client::METHOD_POST, '/users', [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
], [
|
||||
'userId' => ID::unique(),
|
||||
'email' => 'appwrite-push-' . ID::unique() . '@mail.org',
|
||||
'password' => 'password',
|
||||
'name' => 'Appwrite Push User',
|
||||
]);
|
||||
$this->assertEquals(201, $user['headers']['status-code']);
|
||||
|
||||
// The push target's identifier is the MQTT topic the device listens on, which
|
||||
// for the built-in provider is the topic id itself.
|
||||
$target = $this->client->call(Client::METHOD_POST, '/users/' . $user['body']['$id'] . '/targets', [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
], [
|
||||
'targetId' => ID::unique(),
|
||||
'providerType' => 'push',
|
||||
'providerId' => $provider['body']['$id'],
|
||||
'identifier' => $topicId,
|
||||
]);
|
||||
$this->assertEquals(201, $target['headers']['status-code']);
|
||||
|
||||
$subscriber = $this->client->call(Client::METHOD_POST, '/messaging/topics/' . $topicId . '/subscribers', \array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
], $this->getHeaders()), [
|
||||
'subscriberId' => ID::unique(),
|
||||
'targetId' => $target['body']['$id'],
|
||||
]);
|
||||
$this->assertEquals(201, $subscriber['headers']['status-code']);
|
||||
|
||||
// Publish through the campaign path, targeting the topic.
|
||||
$push = $this->client->call(Client::METHOD_POST, '/messaging/messages/push', [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
], [
|
||||
'messageId' => ID::unique(),
|
||||
'topics' => [$topicId],
|
||||
'title' => 'Match update',
|
||||
'body' => 'India needs 12 off 6',
|
||||
'data' => ['matchId' => '42'],
|
||||
]);
|
||||
$this->assertEquals(201, $push['headers']['status-code']);
|
||||
$pushMessageId = $push['body']['$id'];
|
||||
|
||||
// Test for SUCCESS: the worker fans the message into the broker and marks it sent.
|
||||
$this->assertEventually(function () use ($pushMessageId) {
|
||||
$response = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $pushMessageId, [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
]);
|
||||
$this->assertContains($response['body']['status'], [MessageStatus::SENT, MessageStatus::FAILED]);
|
||||
}, 30000, 500);
|
||||
|
||||
$message = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $pushMessageId, [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
]);
|
||||
$this->assertEquals(200, $message['headers']['status-code']);
|
||||
$this->assertEquals(MessageStatus::SENT, $message['body']['status']);
|
||||
$this->assertEquals(1, $message['body']['deliveredTotal']);
|
||||
$this->assertCount(0, $message['body']['deliveryErrors']);
|
||||
}
|
||||
|
||||
public function testUpdateProviders(): void
|
||||
{
|
||||
$providers = $this->setupCreatedProviders();
|
||||
|
||||
@@ -0,0 +1,715 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\E2E\Services\Mqtt;
|
||||
|
||||
use Appwrite\Messaging\Status as MessageStatus;
|
||||
use PHPUnit\Framework\Attributes\Group;
|
||||
use Swoole\Coroutine\Http\Client as WebSocketClient;
|
||||
use Swoole\WebSocket\Frame;
|
||||
use Tests\E2E\Client;
|
||||
use Tests\E2E\Scopes\ProjectCustom;
|
||||
use Tests\E2E\Scopes\Scope;
|
||||
use Tests\E2E\Scopes\SideServer;
|
||||
use Utopia\Database\Helpers\ID;
|
||||
use Utopia\Mqtt\Packet;
|
||||
use Utopia\Mqtt\Packet\Specs\V5;
|
||||
use Utopia\Mqtt\Properties;
|
||||
use Utopia\Mqtt\Property;
|
||||
|
||||
use function Swoole\Coroutine\run;
|
||||
|
||||
/**
|
||||
* End-to-end tests for the MQTT push broker (src/Appwrite/Mqtt). Publishing is a server
|
||||
* privilege driven entirely through the Messaging HTTP campaign path (there is no
|
||||
* outside MQTT publish); the subscriber side is a self-contained MqttSubscriber over
|
||||
* TCP. Exercises enhanced-auth CONNECT against the project/user graph, campaign
|
||||
* fan-out to a live subscriber, and offline QoS 1 session replay from the ledger.
|
||||
*
|
||||
* Blocked/unauthenticated accounts are refused at CONNECT (CONNACK reason 0x87).
|
||||
*
|
||||
* Grouped `mqtt` because it needs the broker container (appwrite-mqtt:1883): the
|
||||
* standard e2e stacks exclude this group, and a dedicated lane that runs the broker
|
||||
* selects it with --group mqtt.
|
||||
*/
|
||||
#[Group('mqtt')]
|
||||
final class MqttServerTest extends Scope
|
||||
{
|
||||
use ProjectCustom;
|
||||
use SideServer;
|
||||
|
||||
private const BROKER_HOST = 'appwrite-mqtt';
|
||||
private const BROKER_PORT = 1883;
|
||||
private const BROKER_WS_PORT = 8083;
|
||||
|
||||
/**
|
||||
* Create a user and mint a session-less JWT for it. The broker's JWT path skips
|
||||
* the session check when the payload carries no sessionId, so the user resolves
|
||||
* as long as it exists in the project.
|
||||
*
|
||||
* @return array{userId: string, jwt: string}
|
||||
*/
|
||||
private function createUser(): array
|
||||
{
|
||||
$userId = ID::unique();
|
||||
|
||||
$user = $this->client->call(Client::METHOD_POST, '/users', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
], $this->getHeaders()), [
|
||||
'userId' => $userId,
|
||||
'email' => 'mqtt-' . $userId . '@appwrite.io',
|
||||
'password' => 'password',
|
||||
]);
|
||||
$this->assertEquals(201, $user['headers']['status-code']);
|
||||
|
||||
$jwt = $this->client->call(Client::METHOD_POST, '/users/' . $userId . '/jwts', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
], $this->getHeaders()), []);
|
||||
$this->assertEquals(201, $jwt['headers']['status-code']);
|
||||
$this->assertNotEmpty($jwt['body']['jwt']);
|
||||
|
||||
return ['userId' => $userId, 'jwt' => $jwt['body']['jwt']];
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a server-side session for a user and return its credential — the base64({id,
|
||||
* secret}) store blob the broker's appwrite-session auth decodes (the same value a client
|
||||
* holds as its session cookie).
|
||||
*/
|
||||
private function createSession(string $userId): string
|
||||
{
|
||||
$session = $this->client->call(Client::METHOD_POST, '/users/' . $userId . '/sessions', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
], $this->getHeaders()));
|
||||
$this->assertEquals(201, $session['headers']['status-code']);
|
||||
$this->assertNotEmpty($session['body']['secret']);
|
||||
|
||||
return $session['body']['secret'];
|
||||
}
|
||||
|
||||
public function testUnauthorizedConnectRejected(): void
|
||||
{
|
||||
$projectId = $this->getProject()['$id'];
|
||||
|
||||
// Test for FAILURE: a bogus credential is refused at CONNECT with reason 0x87
|
||||
// (not authorized).
|
||||
$subscriber = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0x87, $subscriber->connect($projectId, 'not.a.valid.jwt', 'e2e-reject', cleanStart: true));
|
||||
$subscriber->disconnect();
|
||||
}
|
||||
|
||||
public function testBlockedUserConnectRejected(): void
|
||||
{
|
||||
$projectId = $this->getProject()['$id'];
|
||||
['userId' => $userId, 'jwt' => $jwt] = $this->createUser();
|
||||
|
||||
// Block the account.
|
||||
$status = $this->client->call(Client::METHOD_PATCH, '/users/' . $userId . '/status', array_merge([
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $projectId,
|
||||
], $this->getHeaders()), ['status' => false]);
|
||||
$this->assertEquals(200, $status['headers']['status-code']);
|
||||
|
||||
// Test for FAILURE: a blocked account is refused at CONNECT with reason 0x87.
|
||||
$subscriber = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0x87, $subscriber->connect($projectId, $jwt, 'e2e-blocked', cleanStart: true));
|
||||
$subscriber->disconnect();
|
||||
}
|
||||
|
||||
public function testSessionAuthConnect(): void
|
||||
{
|
||||
$projectId = $this->getProject()['$id'];
|
||||
['userId' => $userId] = $this->createUser();
|
||||
$credential = $this->createSession($userId);
|
||||
|
||||
// Test for SUCCESS: a valid session credential authenticates via appwrite-session,
|
||||
// the session-based analogue of the JWT auth method.
|
||||
$subscriber = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $subscriber->connect($projectId, $credential, 'e2e-session-' . $userId, cleanStart: true, authMethod: 'appwrite-session'));
|
||||
$subscriber->disconnect();
|
||||
}
|
||||
|
||||
public function testSessionAuthWithInvalidSecretRejected(): void
|
||||
{
|
||||
$projectId = $this->getProject()['$id'];
|
||||
['userId' => $userId] = $this->createUser();
|
||||
|
||||
// Test for FAILURE: a well-formed store blob with a wrong secret is refused at CONNECT
|
||||
// with reason 0x87 (the session does not verify).
|
||||
$credential = base64_encode((string) json_encode(['id' => $userId, 'secret' => 'not-a-real-secret']));
|
||||
$subscriber = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0x87, $subscriber->connect($projectId, $credential, 'e2e-badsession-' . $userId, cleanStart: true, authMethod: 'appwrite-session'));
|
||||
$subscriber->disconnect();
|
||||
}
|
||||
|
||||
public function testSubscribeToUnknownTopicRejected(): void
|
||||
{
|
||||
$projectId = $this->getProject()['$id'];
|
||||
['userId' => $userId, 'jwt' => $jwt] = $this->createUser();
|
||||
|
||||
$subscriber = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $subscriber->connect($projectId, $jwt, 'e2e-unknown-' . $userId, cleanStart: true));
|
||||
|
||||
try {
|
||||
// Test for FAILURE: a filter that maps to no project topic is refused in the
|
||||
// SUBACK (reason 0x80), not silently granted.
|
||||
$codes = $subscriber->subscribe(['does-not-exist-' . $userId]);
|
||||
$this->assertSame([0x80], $codes);
|
||||
} finally {
|
||||
$subscriber->disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
public function testSubscribeForbiddenByTopicRoles(): void
|
||||
{
|
||||
$projectId = $this->getProject()['$id'];
|
||||
['userId' => $ownerId, 'jwt' => $ownerJwt] = $this->createUser();
|
||||
['userId' => $otherId, 'jwt' => $otherJwt] = $this->createUser();
|
||||
|
||||
$server = [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $projectId,
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
];
|
||||
|
||||
// A topic only the owner may subscribe to.
|
||||
$topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', $server, [
|
||||
'topicId' => ID::unique(),
|
||||
'name' => 'mqtt-restricted',
|
||||
'subscribe' => ['user:' . $ownerId],
|
||||
]);
|
||||
$this->assertEquals(201, $topic['headers']['status-code']);
|
||||
$topicId = $topic['body']['$id'];
|
||||
|
||||
// Test for FAILURE: a user outside the topic's subscribe roles is refused (0x80).
|
||||
$other = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $other->connect($projectId, $otherJwt, 'e2e-acl-other-' . $otherId, cleanStart: true));
|
||||
try {
|
||||
$this->assertSame([0x80], $other->subscribe([$topicId]));
|
||||
} finally {
|
||||
$other->disconnect();
|
||||
}
|
||||
|
||||
// Test for SUCCESS: the owner is inside the roles and is granted.
|
||||
$owner = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $owner->connect($projectId, $ownerJwt, 'e2e-acl-owner-' . $ownerId, cleanStart: true));
|
||||
try {
|
||||
$this->assertSame([1], $owner->subscribe([$topicId]));
|
||||
} finally {
|
||||
$owner->disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
public function testKeepAliveReapsSilentClient(): void
|
||||
{
|
||||
$projectId = $this->getProject()['$id'];
|
||||
['userId' => $userId, 'jwt' => $jwt] = $this->createUser();
|
||||
|
||||
// Connect with a 1s keep-alive (reap deadline 1.5s) and then stay silent. The reaper
|
||||
// must close the socket. Its tick period bounds detection, so allow generous slack.
|
||||
$subscriber = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $subscriber->connect($projectId, $jwt, 'e2e-keepalive-' . $userId, cleanStart: true, keepAlive: 1));
|
||||
|
||||
try {
|
||||
$this->assertTrue($subscriber->awaitClose(40.0), 'broker did not reap a silent client past its keep-alive');
|
||||
} finally {
|
||||
$subscriber->disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
public function testKeepAlivePingKeepsClientAlive(): void
|
||||
{
|
||||
$projectId = $this->getProject()['$id'];
|
||||
['userId' => $userId, 'jwt' => $jwt] = $this->createUser();
|
||||
|
||||
// 1s keep-alive (reap deadline 1.5s). Ping steadily across a full reaper interval: each
|
||||
// PINGREQ must draw a PINGRESP and push the deadline forward, so the client is never reaped.
|
||||
$subscriber = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $subscriber->connect($projectId, $jwt, 'e2e-ping-' . $userId, cleanStart: true, keepAlive: 1));
|
||||
|
||||
try {
|
||||
$deadline = microtime(true) + 25; // span at least one reaper tick (broker interval is 20s)
|
||||
while (microtime(true) < $deadline) {
|
||||
$this->assertTrue($subscriber->ping(2.0), 'broker did not answer PINGRESP');
|
||||
usleep(800_000); // < the 1.5s deadline, so activity stays ahead of the reaper
|
||||
}
|
||||
|
||||
// Still connected: an immediate close-await times out rather than seeing an EOF.
|
||||
$this->assertFalse($subscriber->awaitClose(0.5), 'an actively pinging client was reaped');
|
||||
} finally {
|
||||
$subscriber->disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
public function testGrantedQosIsCappedByTopicConfig(): void
|
||||
{
|
||||
$projectId = $this->getProject()['$id'];
|
||||
['userId' => $userId, 'jwt' => $jwt] = $this->createUser();
|
||||
|
||||
$server = [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $projectId,
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
];
|
||||
$topicQos1 = $this->setupPushTopic($server, $userId, 'mqtt-qos1', qos: 1);
|
||||
$topicQos0 = $this->setupPushTopic($server, $userId, 'mqtt-qos0', qos: 0);
|
||||
|
||||
$subscriber = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $subscriber->connect($projectId, $jwt, 'e2e-qos-' . $userId, cleanStart: true));
|
||||
|
||||
try {
|
||||
// The client requests QoS 1 for both: the qos-1 topic grants 1, the qos-0 topic
|
||||
// caps the grant to 0 (the topic's configured QoS wins from the DB).
|
||||
$this->assertSame([1], $subscriber->subscribe([$topicQos1], 1));
|
||||
$this->assertSame([0], $subscriber->subscribe([$topicQos0], 1));
|
||||
} finally {
|
||||
$subscriber->disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The messaging graph for the built-in Appwrite push provider: a provider, a topic
|
||||
* (whose id is the MQTT delivery channel), a user with a push target identified by
|
||||
* that topic id, and a subscription. Returns the topic id.
|
||||
*
|
||||
* @param array<string, string> $server server-key headers
|
||||
*/
|
||||
private function setupPushTopic(array $server, string $userId, string $name, int $qos = 1): string
|
||||
{
|
||||
$provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/appwrite', $server, [
|
||||
'providerId' => ID::unique(),
|
||||
'name' => $name,
|
||||
'enabled' => true,
|
||||
]);
|
||||
$this->assertEquals(201, $provider['headers']['status-code']);
|
||||
|
||||
$topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', $server, [
|
||||
'topicId' => ID::unique(),
|
||||
'name' => $name,
|
||||
'qos' => $qos,
|
||||
]);
|
||||
$this->assertEquals(201, $topic['headers']['status-code']);
|
||||
$topicId = $topic['body']['$id'];
|
||||
|
||||
$target = $this->client->call(Client::METHOD_POST, '/users/' . $userId . '/targets', $server, [
|
||||
'targetId' => ID::unique(),
|
||||
'providerType' => 'push',
|
||||
'providerId' => $provider['body']['$id'],
|
||||
'identifier' => $topicId,
|
||||
]);
|
||||
$this->assertEquals(201, $target['headers']['status-code']);
|
||||
|
||||
$subscriber = $this->client->call(Client::METHOD_POST, '/messaging/topics/' . $topicId . '/subscribers', $server, [
|
||||
'subscriberId' => ID::unique(),
|
||||
'targetId' => $target['body']['$id'],
|
||||
]);
|
||||
$this->assertEquals(201, $subscriber['headers']['status-code']);
|
||||
|
||||
return $topicId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Publish a push campaign to a topic and block until the worker marks it terminal
|
||||
* (SENT persists it to the ledger and fans it into the broker).
|
||||
*
|
||||
* @param array<string, string> $server
|
||||
* @param array<string, mixed> $data
|
||||
*/
|
||||
private function publishCampaign(array $server, string $topicId, string $title, string $body, array $data = []): void
|
||||
{
|
||||
$push = $this->client->call(Client::METHOD_POST, '/messaging/messages/push', $server, [
|
||||
'messageId' => ID::unique(),
|
||||
'topics' => [$topicId],
|
||||
'title' => $title,
|
||||
'body' => $body,
|
||||
'data' => $data,
|
||||
]);
|
||||
$this->assertEquals(201, $push['headers']['status-code']);
|
||||
$messageId = $push['body']['$id'];
|
||||
|
||||
$this->assertEventually(function () use ($server, $messageId) {
|
||||
$message = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $messageId, $server);
|
||||
$this->assertContains($message['body']['status'], [MessageStatus::SENT, MessageStatus::FAILED]);
|
||||
}, 30000, 500);
|
||||
}
|
||||
|
||||
/**
|
||||
* Full flow: a Messaging push campaign fans out through the worker and the built-in
|
||||
* Appwrite provider into the broker, and a live MQTT subscriber on the topic-id
|
||||
* channel receives it. The subscriber is our own MqttSubscriber (no messaging
|
||||
* adapter): connect + subscribe, publish the campaign, then read the live delivery.
|
||||
*/
|
||||
public function testCampaignFansOutToMqttSubscriber(): void
|
||||
{
|
||||
$projectId = $this->getProject()['$id'];
|
||||
['userId' => $userId, 'jwt' => $jwt] = $this->createUser();
|
||||
|
||||
$server = [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $projectId,
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
];
|
||||
$topicId = $this->setupPushTopic($server, $userId, 'appwrite-mqtt-flow');
|
||||
|
||||
// Subscribe first (the fan-out only reaches connected subscribers), then publish,
|
||||
// then read the delivery that lands on the still-open socket.
|
||||
$subscriber = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $subscriber->connect($projectId, $jwt, 'e2e-flow-' . $userId, cleanStart: true));
|
||||
$subscriber->subscribe([$topicId]);
|
||||
|
||||
try {
|
||||
$this->publishCampaign($server, $topicId, 'Match update', 'India needs 12 off 6', ['matchId' => '42']);
|
||||
$received = $subscriber->consume(limit: 1, timeout: 20.0);
|
||||
} finally {
|
||||
$subscriber->disconnect();
|
||||
}
|
||||
|
||||
// Test for SUCCESS: the campaign reached the live subscriber through the broker.
|
||||
$this->assertCount(1, $received, 'subscriber did not receive the campaign');
|
||||
$this->assertSame($topicId, $received[0]['topic']);
|
||||
|
||||
$payload = \json_decode($received[0]['payload'], true);
|
||||
$this->assertEquals('Match update', $payload['notification']['title']);
|
||||
$this->assertEquals('India needs 12 off 6', $payload['notification']['body']);
|
||||
$this->assertEquals(['matchId' => '42'], $payload['data']);
|
||||
}
|
||||
|
||||
/**
|
||||
* QoS 0 is fire-and-forget and must stay segregated from QoS 1: a subscription to a
|
||||
* QoS-0 topic is delivered at QoS 0, so the PUBLISH carries no packet id and the client
|
||||
* sends no PUBACK (the broker tracks nothing and advances no replay cursor). The client
|
||||
* requests QoS 1 but the topic caps the grant to 0.
|
||||
*/
|
||||
public function testQos0DeliveryHasNoPuback(): void
|
||||
{
|
||||
$projectId = $this->getProject()['$id'];
|
||||
['userId' => $userId, 'jwt' => $jwt] = $this->createUser();
|
||||
|
||||
$server = [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $projectId,
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
];
|
||||
$topicId = $this->setupPushTopic($server, $userId, 'appwrite-mqtt-qos0', qos: 0);
|
||||
|
||||
$subscriber = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $subscriber->connect($projectId, $jwt, 'e2e-qos0-' . $userId, cleanStart: true));
|
||||
$this->assertSame([0], $subscriber->subscribe([$topicId], Packet::QOS_1), 'the QoS-0 topic must cap the grant to 0');
|
||||
|
||||
try {
|
||||
$this->publishCampaign($server, $topicId, 'No ack', 'delivered at qos 0', ['k' => 'v']);
|
||||
$received = $subscriber->consume(limit: 1, timeout: 20.0);
|
||||
} finally {
|
||||
$subscriber->disconnect();
|
||||
}
|
||||
|
||||
// Test for SUCCESS: delivered once, at QoS 0 (no packet id, so the client never PUBACKs).
|
||||
$this->assertCount(1, $received, 'qos 0 subscriber did not receive the campaign');
|
||||
$this->assertSame($topicId, $received[0]['topic']);
|
||||
$this->assertSame(0, $received[0]['qos'], 'delivery to a QoS-0 subscription must be QoS 0');
|
||||
$this->assertFalse($received[0]['dup'], 'a QoS-0 delivery is never marked DUP');
|
||||
}
|
||||
|
||||
/**
|
||||
* MQTT over WebSocket: browser clients reach the broker through its WebSocket listener
|
||||
* (appwrite-mqtt:8083, exposed as wss://<host>:8084 on its own TLS port) instead of raw TCP. The same
|
||||
* enhanced-auth CONNECT and SUBSCRIBE exchange must work with each MQTT packet carried in
|
||||
* a WebSocket binary frame, exercising the broker's WebSocket transport, packet reassembly
|
||||
* and framed send in both directions.
|
||||
*/
|
||||
public function testWebSocketConnectAndSubscribe(): void
|
||||
{
|
||||
$projectId = $this->getProject()['$id'];
|
||||
['userId' => $userId, 'jwt' => $jwt] = $this->createUser();
|
||||
|
||||
$server = [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $projectId,
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
];
|
||||
$topicId = $this->setupPushTopic($server, $userId, 'appwrite-mqtt-ws');
|
||||
|
||||
$connackReason = null;
|
||||
$codes = [];
|
||||
|
||||
run(function () use ($projectId, $jwt, $userId, $topicId, &$connackReason, &$codes) {
|
||||
$client = new WebSocketClient(self::BROKER_HOST, self::BROKER_WS_PORT);
|
||||
$client->set(['timeout' => 10]);
|
||||
$this->assertTrue($client->upgrade('/'), 'websocket upgrade failed');
|
||||
|
||||
$properties = (new Properties())
|
||||
->add(new Property(Property::AUTHENTICATION_METHOD, 'appwrite-jwt'))
|
||||
->add(new Property(Property::AUTHENTICATION_DATA, $jwt))
|
||||
->add(new Property(Property::USER, ['projectId' => $projectId]));
|
||||
|
||||
$buffer = '';
|
||||
|
||||
$client->push(V5::connect('e2e-ws-' . $userId, 60, true, $properties), WEBSOCKET_OPCODE_BINARY);
|
||||
$connack = $this->wsReceive($client, $buffer);
|
||||
$connackReason = \ord($connack->body[1] ?? "\x80");
|
||||
|
||||
$client->push(V5::subscribe(1, [$topicId], 1), WEBSOCKET_OPCODE_BINARY);
|
||||
$suback = $this->wsReceive($client, $buffer);
|
||||
// SUBACK body: [packetId:2][properties][one reason code per filter].
|
||||
$offset = Properties::skip($suback->body, 2);
|
||||
for ($i = $offset; $i < \strlen($suback->body); $i++) {
|
||||
$codes[] = \ord($suback->body[$i]);
|
||||
}
|
||||
|
||||
$client->close();
|
||||
});
|
||||
|
||||
// Test for SUCCESS: the WebSocket carrier authenticated the CONNECT and granted the
|
||||
// subscription at the topic's QoS, proving browser clients can consume the broker.
|
||||
$this->assertSame(0, $connackReason, 'CONNACK over WebSocket was not success');
|
||||
$this->assertSame([1], $codes, 'SUBACK over WebSocket did not grant QoS 1');
|
||||
}
|
||||
|
||||
/** Read WebSocket frames until one whole MQTT packet reassembles out of their payloads. */
|
||||
private function wsReceive(WebSocketClient $client, string &$buffer): Packet
|
||||
{
|
||||
while (true) {
|
||||
[$packets, $buffer] = Packet::frames($buffer);
|
||||
if ($packets !== []) {
|
||||
return Packet::parse($packets[0]);
|
||||
}
|
||||
|
||||
$frame = $client->recv(10);
|
||||
$this->assertInstanceOf(Frame::class, $frame, 'websocket receive failed');
|
||||
$buffer .= (string) $frame->data;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Offline QoS 1 session replay: a persistent-session subscriber (cleanStart = false,
|
||||
* stable client id) subscribes once to register its cursor, disconnects, and while it
|
||||
* is offline two campaigns are published. On reconnect the broker replays the missed
|
||||
* messages from the ledger, in order, and PUBACK advances the cursor.
|
||||
*/
|
||||
public function testSessionReplaysOfflineMessages(): void
|
||||
{
|
||||
$projectId = $this->getProject()['$id'];
|
||||
['userId' => $userId, 'jwt' => $jwt] = $this->createUser();
|
||||
|
||||
$server = [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $projectId,
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
];
|
||||
$topicId = $this->setupPushTopic($server, $userId, 'appwrite-mqtt-replay');
|
||||
|
||||
$clientId = 'e2e-replay-' . $userId;
|
||||
|
||||
// 1) Persistent session seeds its cursor at the current tail; a new topic does not
|
||||
// replay, so it receives nothing.
|
||||
$seed = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $seed->connect($projectId, $jwt, $clientId, cleanStart: false));
|
||||
$seed->subscribe([$topicId]);
|
||||
$this->assertCount(0, $seed->consume(limit: 1, timeout: 2.0));
|
||||
$seed->disconnect();
|
||||
|
||||
// 2) Two campaigns are published while that subscriber is offline.
|
||||
$bodies = ['first offline message', 'second offline message'];
|
||||
foreach ($bodies as $index => $body) {
|
||||
$this->publishCampaign($server, $topicId, 'Update ' . $index, $body, ['n' => (string) $index]);
|
||||
}
|
||||
|
||||
// 3) The same persistent session reconnects and the broker replays both, in order.
|
||||
$resume = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $resume->connect($projectId, $jwt, $clientId, cleanStart: false));
|
||||
$resume->subscribe([$topicId]);
|
||||
$received = $resume->consume(limit: 2, timeout: 10.0);
|
||||
$resume->disconnect();
|
||||
|
||||
// Test for SUCCESS: both offline messages replayed, in sequence order.
|
||||
$this->assertCount(2, $received, 'offline messages were not replayed');
|
||||
foreach ($received as $message) {
|
||||
$this->assertSame($topicId, $message['topic']);
|
||||
$this->assertTrue($message['dup'], 'replayed messages carry the DUP flag');
|
||||
}
|
||||
$replayedBodies = \array_map(
|
||||
fn (array $message): string => \json_decode($message['payload'], true)['notification']['body'],
|
||||
$received,
|
||||
);
|
||||
$this->assertSame($bodies, $replayedBodies);
|
||||
|
||||
// 4) The PUBACKs advanced the cursor past the earlier message, so a third connect
|
||||
// never re-delivers it. (Acks run in per-packet coroutines, so under QoS 1 the tail
|
||||
// may be re-delivered; the cursor never rewinds below the first ack.)
|
||||
\usleep(1000000);
|
||||
$again = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $again->connect($projectId, $jwt, $clientId, cleanStart: false));
|
||||
$again->subscribe([$topicId]);
|
||||
$leftover = $again->consume(limit: 2, timeout: 3.0);
|
||||
$again->disconnect();
|
||||
|
||||
$leftoverBodies = \array_map(
|
||||
fn (array $message): string => \json_decode($message['payload'], true)['notification']['body'],
|
||||
$leftover,
|
||||
);
|
||||
$this->assertNotContains($bodies[0], $leftoverBodies, 'an already-acked message was re-delivered');
|
||||
}
|
||||
|
||||
/**
|
||||
* Non-contiguous acks must not drop the gap. A persistent session replays three
|
||||
* messages and acks the first and third but not the middle; the cursor advances only
|
||||
* to the highest contiguous ack, so on reconnect the broker re-delivers the unacked
|
||||
* middle message rather than skipping past it.
|
||||
*/
|
||||
public function testNonContiguousAckDoesNotDropMessages(): void
|
||||
{
|
||||
$projectId = $this->getProject()['$id'];
|
||||
['userId' => $userId, 'jwt' => $jwt] = $this->createUser();
|
||||
|
||||
$server = [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $projectId,
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
];
|
||||
$topicId = $this->setupPushTopic($server, $userId, 'appwrite-mqtt-gap');
|
||||
$clientId = 'e2e-gap-' . $userId;
|
||||
|
||||
// Seed the persistent session, then publish three messages while offline.
|
||||
$seed = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $seed->connect($projectId, $jwt, $clientId, cleanStart: false));
|
||||
$seed->subscribe([$topicId]);
|
||||
$this->assertCount(0, $seed->consume(limit: 1, timeout: 2.0));
|
||||
$seed->disconnect();
|
||||
|
||||
foreach (['first', 'second', 'third'] as $index => $body) {
|
||||
$this->publishCampaign($server, $topicId, 'Update ' . $index, $body, ['n' => (string) $index]);
|
||||
}
|
||||
|
||||
// Replay all three, but ack only the first and third — leave the middle unacked.
|
||||
$resume = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $resume->connect($projectId, $jwt, $clientId, cleanStart: false));
|
||||
$resume->subscribe([$topicId]);
|
||||
$received = $resume->consume(limit: 3, timeout: 10.0, shouldAck: fn (int $i): bool => $i !== 1);
|
||||
$resume->disconnect();
|
||||
$this->assertCount(3, $received);
|
||||
|
||||
// On reconnect the unacked middle message is re-delivered, not skipped.
|
||||
\usleep(1000000);
|
||||
$again = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $again->connect($projectId, $jwt, $clientId, cleanStart: false));
|
||||
$again->subscribe([$topicId]);
|
||||
$redelivered = $again->consume(limit: 3, timeout: 5.0);
|
||||
$again->disconnect();
|
||||
|
||||
$redeliveredBodies = \array_map(
|
||||
fn (array $message): string => \json_decode($message['payload'], true)['notification']['body'],
|
||||
$redelivered,
|
||||
);
|
||||
$this->assertContains('second', $redeliveredBodies, 'the unacked middle message was dropped');
|
||||
}
|
||||
|
||||
/**
|
||||
* Replay is bounded by the broker's max depth: when more messages accumulate offline
|
||||
* than the cap, only the most recent $maxDepth are replayed (the older ones are
|
||||
* dropped and the cursor jumps to the tail), and a later reconnect replays nothing.
|
||||
*/
|
||||
public function testReplayIsBoundedByMaxDepth(): void
|
||||
{
|
||||
// Mirrors the handler's cap (Subscribe.php $maxDepth).
|
||||
$maxDepth = 5;
|
||||
$overflow = $maxDepth + 2;
|
||||
|
||||
$projectId = $this->getProject()['$id'];
|
||||
['userId' => $userId, 'jwt' => $jwt] = $this->createUser();
|
||||
|
||||
$server = [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $projectId,
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
];
|
||||
$topicId = $this->setupPushTopic($server, $userId, 'appwrite-mqtt-depth');
|
||||
$clientId = 'e2e-depth-' . $userId;
|
||||
|
||||
// Seed the persistent session at the current tail, then go offline.
|
||||
$seed = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $seed->connect($projectId, $jwt, $clientId, cleanStart: false));
|
||||
$seed->subscribe([$topicId]);
|
||||
$this->assertCount(0, $seed->consume(limit: 1, timeout: 2.0));
|
||||
$seed->disconnect();
|
||||
|
||||
// Publish more than the cap while offline.
|
||||
$bodies = [];
|
||||
for ($i = 0; $i < $overflow; $i++) {
|
||||
$body = 'offline message ' . $i;
|
||||
$bodies[] = $body;
|
||||
$this->publishCampaign($server, $topicId, 'Update ' . $i, $body, ['n' => (string) $i]);
|
||||
}
|
||||
|
||||
// Reconnect: only the last $maxDepth replay, in order.
|
||||
$resume = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $resume->connect($projectId, $jwt, $clientId, cleanStart: false));
|
||||
$resume->subscribe([$topicId]);
|
||||
$received = $resume->consume(limit: $overflow, timeout: 10.0);
|
||||
$resume->disconnect();
|
||||
|
||||
$this->assertCount($maxDepth, $received, 'replay was not capped at max depth');
|
||||
$replayedBodies = \array_map(
|
||||
fn (array $message): string => \json_decode($message['payload'], true)['notification']['body'],
|
||||
$received,
|
||||
);
|
||||
$this->assertSame(\array_slice($bodies, -$maxDepth), $replayedBodies);
|
||||
|
||||
// The dropped (capped-out) messages are gone for good: a later reconnect never
|
||||
// re-delivers them. (Acks are processed in per-packet coroutines, so under QoS 1 a
|
||||
// tail message may be re-delivered; the guarantee we assert is that the cursor never
|
||||
// rewinds below the replayed window, so the dropped set is never seen again.)
|
||||
\usleep(1000000);
|
||||
$again = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $again->connect($projectId, $jwt, $clientId, cleanStart: false));
|
||||
$again->subscribe([$topicId]);
|
||||
$leftover = $again->consume(limit: $overflow, timeout: 3.0);
|
||||
$again->disconnect();
|
||||
|
||||
$leftoverBodies = \array_map(
|
||||
fn (array $message): string => \json_decode($message['payload'], true)['notification']['body'],
|
||||
$leftover,
|
||||
);
|
||||
foreach (\array_slice($bodies, 0, $overflow - $maxDepth) as $droppedBody) {
|
||||
$this->assertNotContains($droppedBody, $leftoverBodies, 'a capped-out message was re-delivered');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A clean-start reconnect discards the persisted session: the cursor is purged and the
|
||||
* client re-seeds at the current tail, so an offline backlog is NOT replayed.
|
||||
*/
|
||||
public function testCleanStartDiscardsSession(): void
|
||||
{
|
||||
$projectId = $this->getProject()['$id'];
|
||||
['userId' => $userId, 'jwt' => $jwt] = $this->createUser();
|
||||
|
||||
$server = [
|
||||
'content-type' => 'application/json',
|
||||
'x-appwrite-project' => $projectId,
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
];
|
||||
$topicId = $this->setupPushTopic($server, $userId, 'appwrite-mqtt-clean');
|
||||
$clientId = 'e2e-clean-' . $userId;
|
||||
|
||||
// Establish a persistent session, then go offline.
|
||||
$seed = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $seed->connect($projectId, $jwt, $clientId, cleanStart: false));
|
||||
$seed->subscribe([$topicId]);
|
||||
$this->assertCount(0, $seed->consume(limit: 1, timeout: 2.0));
|
||||
$seed->disconnect();
|
||||
|
||||
// A message accumulates while offline.
|
||||
$this->publishCampaign($server, $topicId, 'Update', 'while offline', ['n' => '0']);
|
||||
|
||||
// Reconnect with cleanStart = true: the session is discarded, so nothing replays.
|
||||
$fresh = new MqttSubscriber(self::BROKER_HOST, self::BROKER_PORT);
|
||||
$this->assertSame(0, $fresh->connect($projectId, $jwt, $clientId, cleanStart: true));
|
||||
$fresh->subscribe([$topicId]);
|
||||
$this->assertCount(0, $fresh->consume(limit: 1, timeout: 3.0), 'clean-start session should not replay a backlog');
|
||||
$fresh->disconnect();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\E2E\Services\Mqtt;
|
||||
|
||||
use Utopia\Mqtt\Packet;
|
||||
use Utopia\Mqtt\Packet\Specs\V5;
|
||||
use Utopia\Mqtt\Properties;
|
||||
use Utopia\Mqtt\Property;
|
||||
|
||||
/**
|
||||
* A minimal MQTT 5.0 subscriber for the broker e2e tests, driven straight over a
|
||||
* socket with the utopia-php/mqtt codec: enhanced-auth CONNECT, SUBSCRIBE, and QoS 1
|
||||
* consume (PUBACK per message). Purpose-built so tests control the client id and
|
||||
* cleanStart flag — i.e. persistent sessions, which the messaging push adapter does
|
||||
* not expose — so offline session replay can be exercised end to end.
|
||||
*/
|
||||
final class MqttSubscriber
|
||||
{
|
||||
/** @var resource|null */
|
||||
private $socket = null;
|
||||
|
||||
private int $packetId = 0;
|
||||
|
||||
/** @var Packet[] PUBLISH packets read before the SUBACK (replay), drained by consume(). */
|
||||
private array $pending = [];
|
||||
|
||||
public function __construct(
|
||||
private readonly string $host,
|
||||
private readonly int $port,
|
||||
private readonly float $connectTimeout = 10.0,
|
||||
) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Enhanced-auth CONNECT. Returns the CONNACK reason code (0 = success); the broker
|
||||
* rejects with 0x87 (not authorized). Throws only on a transport failure.
|
||||
*/
|
||||
public function connect(string $projectId, string $credential, string $clientId, bool $cleanStart, string $authMethod = 'appwrite-jwt', int $keepAlive = 60): int
|
||||
{
|
||||
$errno = 0;
|
||||
$errstr = '';
|
||||
$socket = @stream_socket_client("tcp://{$this->host}:{$this->port}", $errno, $errstr, $this->connectTimeout);
|
||||
if ($socket === false) {
|
||||
throw new \RuntimeException("mqtt connect failed: {$errstr} ({$errno})");
|
||||
}
|
||||
$this->socket = $socket;
|
||||
|
||||
$properties = new Properties();
|
||||
$properties->add(new Property(Property::AUTHENTICATION_METHOD, $authMethod));
|
||||
$properties->add(new Property(Property::AUTHENTICATION_DATA, $credential));
|
||||
$properties->add(new Property(Property::USER, ['projectId' => $projectId]));
|
||||
|
||||
$this->write(V5::connect($clientId, $keepAlive, $cleanStart, $properties));
|
||||
|
||||
$packet = $this->readPacket(microtime(true) + $this->connectTimeout);
|
||||
if ($packet === null || $packet->type !== Packet::CONNACK) {
|
||||
throw new \RuntimeException('mqtt: no CONNACK received');
|
||||
}
|
||||
|
||||
// CONNACK body: [acknowledge flags][reason code][properties].
|
||||
return \ord($packet->body[1] ?? "\x80");
|
||||
}
|
||||
|
||||
/**
|
||||
* SUBSCRIBE to the given topic filters at the requested max QoS; returns the SUBACK
|
||||
* reason code per filter (a granted QoS 0x00/0x01, or a failure such as 0x87), index
|
||||
* aligned with $topics.
|
||||
*
|
||||
* @param string[] $topics
|
||||
* @return array<int, int>
|
||||
*/
|
||||
public function subscribe(array $topics, int $qos = Packet::QOS_1, float $timeout = 5.0): array
|
||||
{
|
||||
$this->write(V5::subscribe($this->nextPacketId(), $topics, $qos));
|
||||
|
||||
$deadline = microtime(true) + $timeout;
|
||||
while (true) {
|
||||
$packet = $this->readPacket($deadline);
|
||||
if ($packet === null) {
|
||||
throw new \RuntimeException('mqtt: no SUBACK received');
|
||||
}
|
||||
if ($packet->type === Packet::SUBACK) {
|
||||
// SUBACK body: [packetId:2][properties][one reason code per filter].
|
||||
$offset = Properties::skip($packet->body, 2);
|
||||
|
||||
$codes = [];
|
||||
for ($i = $offset; $i < \strlen($packet->body); $i++) {
|
||||
$codes[] = \ord($packet->body[$i]);
|
||||
}
|
||||
|
||||
return $codes;
|
||||
}
|
||||
if ($packet->type === Packet::PUBLISH) {
|
||||
$this->pending[] = $packet; // replay delivered before the SUBACK; consume() drains it
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Consume up to $limit QoS 1 PUBLISH messages within $timeout seconds, PUBACKing each.
|
||||
* $shouldAck, given the 0-based index of a received message, may return false to skip
|
||||
* its PUBACK — used to simulate non-contiguous acknowledgement.
|
||||
*
|
||||
* @param (callable(int): bool)|null $shouldAck
|
||||
* @return array<int, array{topic: string, payload: string, dup: bool, qos: int}>
|
||||
*/
|
||||
public function consume(int $limit, float $timeout, ?callable $shouldAck = null): array
|
||||
{
|
||||
$deadline = microtime(true) + $timeout;
|
||||
$received = [];
|
||||
|
||||
while (\count($received) < $limit) {
|
||||
if ($this->pending !== []) {
|
||||
$packet = \array_shift($this->pending);
|
||||
} else {
|
||||
$packet = $this->readPacket($deadline);
|
||||
if ($packet === null) {
|
||||
break; // timeout or disconnect
|
||||
}
|
||||
}
|
||||
if ($packet->type !== Packet::PUBLISH) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$body = $packet->body;
|
||||
[$topic, $offset] = Packet::readString($body, 0);
|
||||
|
||||
$packetIdBytes = '';
|
||||
if ($packet->qos() > 0) {
|
||||
$packetIdBytes = substr($body, $offset, 2);
|
||||
$offset += 2;
|
||||
}
|
||||
$offset = Properties::skip($body, $offset);
|
||||
$payload = substr($body, $offset);
|
||||
|
||||
$received[] = ['topic' => $topic, 'payload' => $payload, 'dup' => $packet->dup(), 'qos' => $packet->qos()];
|
||||
|
||||
$ack = $shouldAck === null || $shouldAck(\count($received) - 1);
|
||||
if ($ack && $packet->qos() === 1 && $packetIdBytes !== '') {
|
||||
$this->write(V5::puback($packetIdBytes));
|
||||
}
|
||||
}
|
||||
|
||||
return $received;
|
||||
}
|
||||
|
||||
/** Send a PINGREQ and return true once the broker answers PINGRESP within $timeout. */
|
||||
public function ping(float $timeout = 2.0): bool
|
||||
{
|
||||
$this->write(Packet::pingreq());
|
||||
|
||||
$deadline = microtime(true) + $timeout;
|
||||
while (true) {
|
||||
$packet = $this->readPacket($deadline);
|
||||
if ($packet === null) {
|
||||
return false;
|
||||
}
|
||||
if ($packet->type === Packet::PINGRESP) {
|
||||
return true;
|
||||
}
|
||||
// Ignore any interleaved PUBLISH; keep waiting for the PINGRESP.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Block until the broker closes the connection, or $timeout seconds elapse. Returns true
|
||||
* if the broker closed it (keep-alive reap), false on timeout while still connected.
|
||||
*/
|
||||
public function awaitClose(float $timeout): bool
|
||||
{
|
||||
if (!\is_resource($this->socket)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$deadline = microtime(true) + $timeout;
|
||||
while (true) {
|
||||
$remaining = $deadline - microtime(true);
|
||||
if ($remaining <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$read = [$this->socket];
|
||||
$write = null;
|
||||
$except = null;
|
||||
$seconds = (int) $remaining;
|
||||
$ready = @stream_select($read, $write, $except, $seconds, (int) (($remaining - $seconds) * 1_000_000));
|
||||
if ($ready === false) {
|
||||
return false;
|
||||
}
|
||||
if ($ready === 0) {
|
||||
continue; // idle but still open
|
||||
}
|
||||
|
||||
// Readable: a broker-closed socket reads '' at EOF; anything else keeps it open.
|
||||
$chunk = @fread($this->socket, 1024);
|
||||
if (($chunk === '' || $chunk === false) && feof($this->socket)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function disconnect(): void
|
||||
{
|
||||
if (!\is_resource($this->socket)) {
|
||||
return;
|
||||
}
|
||||
@fwrite($this->socket, V5::disconnect(0)); // 0x00 = normal disconnection
|
||||
@fclose($this->socket);
|
||||
$this->socket = null;
|
||||
}
|
||||
|
||||
private function write(string $packet): void
|
||||
{
|
||||
if (!\is_resource($this->socket)) {
|
||||
throw new \RuntimeException('mqtt: socket is not connected');
|
||||
}
|
||||
@fwrite($this->socket, $packet);
|
||||
}
|
||||
|
||||
/** Read one whole MQTT packet (fixed header + remaining length + body), or null on timeout/EOF. */
|
||||
private function readPacket(float $deadline): ?Packet
|
||||
{
|
||||
$header = $this->readBytes(1, $deadline);
|
||||
if ($header === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Remaining Length is a 1-4 byte variable-length integer.
|
||||
$lengthBytes = '';
|
||||
$multiplier = 1;
|
||||
$length = 0;
|
||||
do {
|
||||
$next = $this->readBytes(1, $deadline);
|
||||
if ($next === null) {
|
||||
return null;
|
||||
}
|
||||
$lengthBytes .= $next;
|
||||
$byte = \ord($next);
|
||||
$length += ($byte & 0x7F) * $multiplier;
|
||||
$multiplier *= 128;
|
||||
} while (($byte & 0x80) !== 0 && \strlen($lengthBytes) < 4);
|
||||
|
||||
$body = $length > 0 ? $this->readBytes($length, $deadline) : '';
|
||||
if ($body === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Packet::parse($header . $lengthBytes . $body);
|
||||
}
|
||||
|
||||
/** Read exactly $n bytes before $deadline, or null on timeout/EOF. */
|
||||
private function readBytes(int $n, float $deadline): ?string
|
||||
{
|
||||
$buffer = '';
|
||||
while (\strlen($buffer) < $n) {
|
||||
$remaining = $deadline - microtime(true);
|
||||
if ($remaining <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$read = [$this->socket];
|
||||
$write = null;
|
||||
$except = null;
|
||||
$seconds = (int) $remaining;
|
||||
$ready = @stream_select($read, $write, $except, $seconds, (int) (($remaining - $seconds) * 1_000_000));
|
||||
if ($ready === false) {
|
||||
return null;
|
||||
}
|
||||
if ($ready === 0) {
|
||||
continue; // nothing readable yet; re-check the deadline
|
||||
}
|
||||
|
||||
$chunk = @fread($this->socket, $n - \strlen($buffer));
|
||||
if ($chunk === false || ($chunk === '' && feof($this->socket))) {
|
||||
return null;
|
||||
}
|
||||
$buffer .= $chunk;
|
||||
}
|
||||
|
||||
return $buffer;
|
||||
}
|
||||
|
||||
private function nextPacketId(): int
|
||||
{
|
||||
$this->packetId = ($this->packetId % 65535) + 1;
|
||||
|
||||
return $this->packetId;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Unit\Appwrite\Messaging\Adapter\Push;
|
||||
|
||||
use Appwrite\Messaging\Adapter\Mqtt;
|
||||
use Appwrite\Messaging\Adapter\Push\Appwrite as AppwritePush;
|
||||
use Appwrite\PubSub\Adapter as PubSub;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
use Utopia\Cache\Adapter\None as NoCache;
|
||||
use Utopia\Cache\Cache;
|
||||
use Utopia\Database\Adapter\Memory;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Helpers\Permission;
|
||||
use Utopia\Database\Helpers\Role;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Messaging\Messages\Email;
|
||||
use Utopia\Messaging\Messages\Push;
|
||||
use Utopia\Messaging\Priority;
|
||||
use Utopia\Telemetry\Adapter\None as NoTelemetry;
|
||||
|
||||
/**
|
||||
* A stand-in broker that records each fan-out instead of touching Redis. The real
|
||||
* Mqtt::send() publishes onto the pub/sub pool (a live pool, exercised by e2e); here
|
||||
* we only care that the adapter hands it the right project, topic, payload and QoS.
|
||||
*/
|
||||
final class FakeBroker extends Mqtt
|
||||
{
|
||||
/** @var array<int, array{projectId: string, channels: array<int, string>, options: array<string, mixed>}> */
|
||||
public array $published = [];
|
||||
|
||||
/** @var array<int, string> topics to reject, to simulate a fan-out failure */
|
||||
public array $failTopics = [];
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
// send() is overridden below, so the pub/sub is never used; a null-object satisfies it.
|
||||
parent::__construct(new NoTelemetry(), new class () implements PubSub {
|
||||
public function ping($message = null): bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
public function subscribe($channels, $callback)
|
||||
{
|
||||
}
|
||||
|
||||
public function publish($channel, $message)
|
||||
{
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public function send(string $projectId, array $payload, array $events, array $channels, array $roles, array $options = []): void
|
||||
{
|
||||
foreach ($channels as $topic) {
|
||||
if (\in_array($topic, $this->failTopics, true)) {
|
||||
throw new \RuntimeException("broker refused {$topic}");
|
||||
}
|
||||
}
|
||||
|
||||
$this->published[] = [
|
||||
'projectId' => $projectId,
|
||||
'channels' => $channels,
|
||||
'options' => $options,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unit tests for the built-in Appwrite push provider: the append-to-ledger plus
|
||||
* internal fan-out that process() performs per topic. A real in-memory Database
|
||||
* backs the topics counter and the messages_appwrite ledger, and a FakeBroker
|
||||
* captures the fan-out, so no Redis or transport is involved.
|
||||
*
|
||||
* The broker CONNECT/publish/subscribe transport is covered by the mqtt-grouped
|
||||
* e2e (MessagingMqttServerTest) instead.
|
||||
*/
|
||||
final class AppwriteTest extends TestCase
|
||||
{
|
||||
private const PROJECT_ID = 'project-a';
|
||||
|
||||
private Database $database;
|
||||
|
||||
protected function setUp(): void
|
||||
{
|
||||
$authorization = new Authorization();
|
||||
$authorization->addRole(Role::any()->toString());
|
||||
|
||||
$this->database = new Database(new Memory(), new Cache(new NoCache()));
|
||||
$this->database
|
||||
->setAuthorization($authorization)
|
||||
->setDatabase('mqttTests')
|
||||
->setNamespace('mqtt_' . \uniqid());
|
||||
$this->database->create();
|
||||
|
||||
$any = [
|
||||
Permission::create(Role::any()),
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
];
|
||||
|
||||
// The topic counter: only the monotonic `sequence` the adapter increments.
|
||||
$this->database->createCollection('topics', [], [], $any, false);
|
||||
$this->database->createAttribute('topics', 'sequence', Database::VAR_INTEGER, 0, false, 0);
|
||||
|
||||
// The append-only ledger. `data` is a plain string here (the adapter passes an
|
||||
// already-encoded JSON envelope); the production collection's json filter is a
|
||||
// storage detail, not adapter behaviour.
|
||||
$this->database->createCollection('appwritePushLedger', [], [], $any, false);
|
||||
$this->database->createAttribute('appwritePushLedger', 'topic', Database::VAR_STRING, 255, true);
|
||||
$this->database->createAttribute('appwritePushLedger', 'data', Database::VAR_STRING, 65535, true);
|
||||
$this->database->createAttribute('appwritePushLedger', 'messageId', Database::VAR_STRING, 255, false);
|
||||
$this->database->createAttribute('appwritePushLedger', 'messageInternalId', Database::VAR_STRING, 255, false);
|
||||
$this->database->createAttribute('appwritePushLedger', 'sequence', Database::VAR_INTEGER, 0, true);
|
||||
}
|
||||
|
||||
/** Seed a topic row with a starting sequence (the current tail). */
|
||||
private function seedTopic(string $topicId, int $sequence = 0): void
|
||||
{
|
||||
$this->database->createDocument('topics', new Document([
|
||||
'$id' => $topicId,
|
||||
'sequence' => $sequence,
|
||||
]));
|
||||
}
|
||||
|
||||
private function adapter(FakeBroker $broker, int $qos = 1, string $messageId = 'msg-1', ?string $messageInternalId = 'internal-1'): AppwritePush
|
||||
{
|
||||
return new AppwritePush(
|
||||
$broker,
|
||||
$this->database,
|
||||
self::PROJECT_ID,
|
||||
$messageId,
|
||||
$messageInternalId,
|
||||
$qos,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, Document>
|
||||
*/
|
||||
private function ledger(): array
|
||||
{
|
||||
return $this->database->getAuthorization()->skip(fn () => $this->database->find('appwritePushLedger'));
|
||||
}
|
||||
|
||||
/** The topic counter row, read past authorization. */
|
||||
private function topic(string $topicId): Document
|
||||
{
|
||||
return $this->database->getAuthorization()->skip(fn () => $this->database->getDocument('topics', $topicId));
|
||||
}
|
||||
|
||||
public function testMetadata(): void
|
||||
{
|
||||
$adapter = $this->adapter(new FakeBroker());
|
||||
|
||||
$this->assertSame('Appwrite', $adapter->getName());
|
||||
$this->assertSame('push', $adapter->getType());
|
||||
$this->assertSame(Push::class, $adapter->getMessageType());
|
||||
$this->assertSame(5000, $adapter->getMaxMessagesPerRequest());
|
||||
}
|
||||
|
||||
public function testPublishWritesLedgerAndFansOut(): void
|
||||
{
|
||||
$this->seedTopic('topic-1', 0);
|
||||
$broker = new FakeBroker();
|
||||
|
||||
$response = $this->adapter($broker, qos: 1)->send(new Push(
|
||||
to: ['topic-1'],
|
||||
title: 'Hi',
|
||||
body: 'Hello',
|
||||
));
|
||||
|
||||
// Response: one topic, delivered, no failures.
|
||||
$this->assertSame('push', $response['type']);
|
||||
$this->assertSame(1, $response['deliveredTo']);
|
||||
$this->assertCount(1, $response['results']);
|
||||
$this->assertSame('success', $response['results'][0]['status']);
|
||||
$this->assertSame('topic-1', $response['results'][0]['recipient']);
|
||||
|
||||
// The topic counter advanced 0 -> 1.
|
||||
$topic = $this->topic('topic-1');
|
||||
$this->assertSame(1, $topic->getAttribute('sequence'));
|
||||
|
||||
// One ledger row, carrying the copied sequence and the source message ids.
|
||||
$ledger = $this->ledger();
|
||||
$this->assertCount(1, $ledger);
|
||||
$row = $ledger[0];
|
||||
$this->assertSame('topic-1', $row->getAttribute('topic'));
|
||||
$this->assertSame(1, $row->getAttribute('sequence'));
|
||||
$this->assertSame('msg-1', $row->getAttribute('messageId'));
|
||||
$this->assertSame('internal-1', $row->getAttribute('messageInternalId'));
|
||||
|
||||
// The fan-out reached the broker with the same project, topic, payload and QoS.
|
||||
$this->assertCount(1, $broker->published);
|
||||
$publish = $broker->published[0];
|
||||
$this->assertSame(self::PROJECT_ID, $publish['projectId']);
|
||||
$this->assertSame(['topic-1'], $publish['channels']);
|
||||
$this->assertSame(1, $publish['options']['qos']);
|
||||
// The ledger stores exactly what was published.
|
||||
$this->assertSame($row->getAttribute('data'), $publish['options']['payload']);
|
||||
}
|
||||
|
||||
public function testPayloadEnvelopeShape(): void
|
||||
{
|
||||
$this->seedTopic('topic-1', 0);
|
||||
$broker = new FakeBroker();
|
||||
|
||||
$this->adapter($broker)->send(new Push(
|
||||
to: ['topic-1'],
|
||||
title: 'Match tonight',
|
||||
body: 'India vs Australia',
|
||||
data: ['matchId' => '42'],
|
||||
priority: Priority::HIGH,
|
||||
));
|
||||
|
||||
$payload = \json_decode($broker->published[0]['options']['payload'], true);
|
||||
|
||||
$this->assertSame('Match tonight', $payload['notification']['title']);
|
||||
$this->assertSame('India vs Australia', $payload['notification']['body']);
|
||||
$this->assertSame(['matchId' => '42'], $payload['data']);
|
||||
$this->assertSame('high', $payload['priority']);
|
||||
}
|
||||
|
||||
public function testSequenceAdvancesPerPublish(): void
|
||||
{
|
||||
$this->seedTopic('topic-1', 0);
|
||||
$broker = new FakeBroker();
|
||||
|
||||
// Two distinct campaigns (distinct message ids) to the same topic.
|
||||
$this->adapter($broker, messageId: 'msg-1')->send(new Push(to: ['topic-1'], title: 'first'));
|
||||
$this->adapter($broker, messageId: 'msg-2')->send(new Push(to: ['topic-1'], title: 'second'));
|
||||
|
||||
$sequences = \array_map(
|
||||
static fn (Document $row): int => $row->getAttribute('sequence'),
|
||||
$this->ledger(),
|
||||
);
|
||||
\sort($sequences);
|
||||
$this->assertSame([1, 2], $sequences);
|
||||
|
||||
$topic = $this->topic('topic-1');
|
||||
$this->assertSame(2, $topic->getAttribute('sequence'));
|
||||
}
|
||||
|
||||
public function testRepublishIsIdempotent(): void
|
||||
{
|
||||
// Re-sending the same campaign (a retry after a transient failure) must not
|
||||
// advance the sequence or add a second ledger row for the (message, topic) pair.
|
||||
$this->seedTopic('topic-1', 0);
|
||||
$broker = new FakeBroker();
|
||||
|
||||
$this->adapter($broker, messageId: 'msg-1')->send(new Push(to: ['topic-1'], title: 'once'));
|
||||
$this->adapter($broker, messageId: 'msg-1')->send(new Push(to: ['topic-1'], title: 'once'));
|
||||
|
||||
$this->assertCount(1, $this->ledger());
|
||||
$this->assertSame(1, $this->topic('topic-1')->getAttribute('sequence'));
|
||||
}
|
||||
|
||||
public function testEachTopicGetsItsOwnSequence(): void
|
||||
{
|
||||
// Two topics at different tails: each advances independently.
|
||||
$this->seedTopic('topic-1', 5);
|
||||
$this->seedTopic('topic-2', 0);
|
||||
$broker = new FakeBroker();
|
||||
|
||||
$response = $this->adapter($broker)->send(new Push(
|
||||
to: ['topic-1', 'topic-2'],
|
||||
title: 'broadcast',
|
||||
));
|
||||
|
||||
$this->assertSame(2, $response['deliveredTo']);
|
||||
$this->assertCount(2, $broker->published);
|
||||
|
||||
$sequenceByTopic = [];
|
||||
foreach ($this->ledger() as $row) {
|
||||
$sequenceByTopic[$row->getAttribute('topic')] = $row->getAttribute('sequence');
|
||||
}
|
||||
$this->assertSame(6, $sequenceByTopic['topic-1']);
|
||||
$this->assertSame(1, $sequenceByTopic['topic-2']);
|
||||
}
|
||||
|
||||
public function testUnknownTopicFailsWithoutSinkingOthers(): void
|
||||
{
|
||||
// Only the first topic exists; the second has no counter row to increment.
|
||||
$this->seedTopic('topic-1', 0);
|
||||
$broker = new FakeBroker();
|
||||
|
||||
$response = $this->adapter($broker)->send(new Push(
|
||||
to: ['topic-1', 'missing-topic'],
|
||||
title: 'Hi',
|
||||
));
|
||||
|
||||
// The good topic is delivered; the missing one is a per-recipient failure.
|
||||
$this->assertSame(1, $response['deliveredTo']);
|
||||
$this->assertCount(2, $response['results']);
|
||||
|
||||
$byRecipient = [];
|
||||
foreach ($response['results'] as $result) {
|
||||
$byRecipient[$result['recipient']] = $result;
|
||||
}
|
||||
$this->assertSame('success', $byRecipient['topic-1']['status']);
|
||||
$this->assertSame('failure', $byRecipient['missing-topic']['status']);
|
||||
$this->assertNotSame('', $byRecipient['missing-topic']['error']);
|
||||
|
||||
// The failure left no ledger row and never reached the broker.
|
||||
$ledger = $this->ledger();
|
||||
$this->assertCount(1, $ledger);
|
||||
$this->assertSame('topic-1', $ledger[0]->getAttribute('topic'));
|
||||
$this->assertCount(1, $broker->published);
|
||||
$this->assertSame(['topic-1'], $broker->published[0]['channels']);
|
||||
}
|
||||
|
||||
public function testFanOutFailureIsRecordedPerTopic(): void
|
||||
{
|
||||
// The ledger write succeeds but the broker rejects the fan-out: the topic is a
|
||||
// failure, yet the durable row is already written for later replay.
|
||||
$this->seedTopic('topic-1', 0);
|
||||
$broker = new FakeBroker();
|
||||
$broker->failTopics = ['topic-1'];
|
||||
|
||||
$response = $this->adapter($broker)->send(new Push(
|
||||
to: ['topic-1'],
|
||||
title: 'Hi',
|
||||
));
|
||||
|
||||
$this->assertSame(0, $response['deliveredTo']);
|
||||
$this->assertSame('failure', $response['results'][0]['status']);
|
||||
$this->assertCount(1, $this->ledger());
|
||||
$this->assertCount(0, $broker->published);
|
||||
}
|
||||
|
||||
public function testInvalidMessageTypeRejected(): void
|
||||
{
|
||||
$this->expectException(\Exception::class);
|
||||
$this->expectExceptionMessage('Invalid message type.');
|
||||
|
||||
// A non-push message is rejected by the base adapter before process() runs.
|
||||
$this->adapter(new FakeBroker())->send(new Email(
|
||||
to: ['nobody@appwrite.io'],
|
||||
subject: 'nope',
|
||||
content: 'nope',
|
||||
fromName: 'Appwrite',
|
||||
fromEmail: 'noreply@appwrite.io',
|
||||
));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user