From db0a971384978b92e59f6cd4c96bac6e4a9af56d Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Wed, 15 Apr 2026 11:43:53 +0530 Subject: [PATCH 01/20] feat(sites): add screenshot success telemetry to usage stats --- app/init/constants.php | 6 ++ .../Modules/Functions/Workers/Screenshots.php | 55 ++++++++++++++++++- .../Platform/Modules/Sites/Http/Usage/Get.php | 22 ++++++-- .../Modules/Sites/Http/Usage/XList.php | 23 ++++++-- .../Utopia/Response/Model/UsageSite.php | 32 +++++++++++ .../Utopia/Response/Model/UsageSites.php | 32 +++++++++++ tests/e2e/General/UsageTest.php | 14 ++++- 7 files changed, 169 insertions(+), 15 deletions(-) diff --git a/app/init/constants.php b/app/init/constants.php index f2127cd666..57763df911 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -351,6 +351,8 @@ const METRIC_BUILDS_STORAGE = 'builds.storage'; const METRIC_BUILDS_COMPUTE = 'builds.compute'; const METRIC_BUILDS_COMPUTE_SUCCESS = 'builds.compute.success'; const METRIC_BUILDS_COMPUTE_FAILED = 'builds.compute.failed'; +const METRIC_SCREENSHOTS_SUCCESS = 'screenshots.success'; +const METRIC_SCREENSHOTS_FAILED = 'screenshots.failed'; const METRIC_BUILDS_MB_SECONDS = 'builds.mbSeconds'; const METRIC_EXECUTIONS = 'executions'; const METRIC_EXECUTIONS_COMPUTE = 'executions.compute'; @@ -363,6 +365,8 @@ const METRIC_RESOURCE_TYPE_ID_BUILDS_FAILED = '{resourceType}.{resourceInternal const METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE = '{resourceType}.{resourceInternalId}.builds.compute'; const METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_SUCCESS = '{resourceType}.{resourceInternalId}.builds.compute.success'; const METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_FAILED = '{resourceType}.{resourceInternalId}.builds.compute.failed'; +const METRIC_RESOURCE_TYPE_ID_SCREENSHOTS_SUCCESS = '{resourceType}.{resourceInternalId}.screenshots.success'; +const METRIC_RESOURCE_TYPE_ID_SCREENSHOTS_FAILED = '{resourceType}.{resourceInternalId}.screenshots.failed'; const METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS = '{resourceType}.{resourceInternalId}.builds.mbSeconds'; const METRIC_RESOURCE_TYPE_ID_BUILDS = '{resourceType}.{resourceInternalId}.builds'; const METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE = '{resourceType}.{resourceInternalId}.builds.storage'; @@ -376,6 +380,8 @@ const METRIC_RESOURCE_TYPE_BUILDS_FAILED = '{resourceType}.builds.failed'; const METRIC_RESOURCE_TYPE_BUILDS_COMPUTE = '{resourceType}.builds.compute'; const METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_SUCCESS = '{resourceType}.builds.compute.success'; const METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_FAILED = '{resourceType}.builds.compute.failed'; +const METRIC_RESOURCE_TYPE_SCREENSHOTS_SUCCESS = '{resourceType}.screenshots.success'; +const METRIC_RESOURCE_TYPE_SCREENSHOTS_FAILED = '{resourceType}.screenshots.failed'; const METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS = '{resourceType}.builds.mbSeconds'; const METRIC_RESOURCE_TYPE_BUILDS = '{resourceType}.builds'; const METRIC_RESOURCE_TYPE_BUILDS_STORAGE = '{resourceType}.builds.storage'; diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php b/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php index 065fe477eb..fa7e394710 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php @@ -3,9 +3,12 @@ namespace Appwrite\Platform\Modules\Functions\Workers; use Ahc\Jwt\JWT; +use Appwrite\Event\Message\Usage as UsageMessage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; use Appwrite\Permission; use Appwrite\Role; +use Appwrite\Usage\Context; use Exception; use Utopia\Compression\Compression; use Utopia\Config\Config; @@ -43,6 +46,7 @@ class Screenshots extends Action ->inject('dbForProject') ->inject('project') ->inject('deviceForFiles') + ->inject('publisherForUsage') ->callback($this->action(...)); } @@ -52,7 +56,8 @@ class Screenshots extends Action Database $dbForPlatform, Database $dbForProject, Document $project, - Device $deviceForFiles + Device $deviceForFiles, + UsagePublisher $publisherForUsage ): void { Console::log('Screenshot action started'); @@ -257,6 +262,13 @@ class Screenshots extends Action 'deploymentScreenshotDark' => $deployment->getAttribute('screenshotDark', ''), 'deploymentScreenshotLight' => $deployment->getAttribute('screenshotLight', ''), ])); + + $this->publishUsage( + project: $project, + site: $site, + publisherForUsage: $publisherForUsage, + metric: METRIC_SCREENSHOTS_SUCCESS + ); } catch (\Throwable $th) { Console::warning("Screenshot failed to generate:"); Console::warning($th->getMessage()); @@ -265,10 +277,51 @@ class Screenshots extends Action $date = \date('H:i:s'); $this->appendToLogs($dbForProject, $deployment->getId(), $queueForRealtime, "[$date] [appwrite] Screenshot capturing failed. Deployment will continue. \n"); + $this->publishUsage( + project: $project, + site: $site, + publisherForUsage: $publisherForUsage, + metric: METRIC_SCREENSHOTS_FAILED + ); + throw $th; } } + protected function publishUsage( + Document $project, + Document $site, + UsagePublisher $publisherForUsage, + string $metric + ): void { + [$resourceMetric, $resourceIdMetric] = match ($metric) { + METRIC_SCREENSHOTS_SUCCESS => [ + METRIC_RESOURCE_TYPE_SCREENSHOTS_SUCCESS, + METRIC_RESOURCE_TYPE_ID_SCREENSHOTS_SUCCESS, + ], + METRIC_SCREENSHOTS_FAILED => [ + METRIC_RESOURCE_TYPE_SCREENSHOTS_FAILED, + METRIC_RESOURCE_TYPE_ID_SCREENSHOTS_FAILED, + ], + default => throw new \InvalidArgumentException('Unknown screenshot metric: ' . $metric), + }; + + $usage = (new Context()) + ->addMetric($metric, 1) + ->addMetric(str_replace('{resourceType}', RESOURCE_TYPE_SITES, $resourceMetric), 1) + ->addMetric(str_replace( + ['{resourceType}', '{resourceInternalId}'], + [RESOURCE_TYPE_SITES, $site->getSequence()], + $resourceIdMetric + ), 1); + + $publisherForUsage->enqueue(new UsageMessage( + project: $project, + metrics: $usage->getMetrics(), + reduce: $usage->getReduce() + )); + } + protected function appendToLogs(Database $dbForProject, string $deploymentId, Realtime $queueForRealtime, string $logs) { $deployment = $dbForProject->getDocument('deployments', $deploymentId); diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Usage/Get.php b/src/Appwrite/Platform/Modules/Sites/Http/Usage/Get.php index a6768462d1..a1af4bd8fd 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Usage/Get.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Usage/Get.php @@ -87,6 +87,8 @@ class Get extends Base str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $site->getSequence()], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_MB_SECONDS), str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $site->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_SUCCESS), str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $site->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_FAILED), + str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $site->getSequence()], METRIC_RESOURCE_TYPE_ID_SCREENSHOTS_SUCCESS), + str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $site->getSequence()], METRIC_RESOURCE_TYPE_ID_SCREENSHOTS_FAILED), str_replace(['{siteInternalId}'], [$site->getSequence()], METRIC_SITES_ID_REQUESTS), str_replace(['{siteInternalId}'], [$site->getSequence()], METRIC_SITES_ID_INBOUND), str_replace(['{siteInternalId}'], [$site->getSequence()], METRIC_SITES_ID_OUTBOUND), @@ -139,6 +141,9 @@ class Get extends Base $buildsTimeTotal = $usage[$metrics[4]]['total'] ?? 0; $buildsTotal = $usage[$metrics[2]]['total'] ?? 0; + $screenshotsSuccessTotal = $usage[$metrics[11]]['total'] ?? 0; + $screenshotsFailedTotal = $usage[$metrics[12]]['total'] ?? 0; + $screenshotsTotal = $screenshotsSuccessTotal + $screenshotsFailedTotal; $response->dynamic(new Document([ 'range' => $range, 'deploymentsTotal' => $usage[$metrics[0]]['total'], @@ -153,9 +158,12 @@ class Get extends Base 'executionsMbSecondsTotal' => $usage[$metrics[8]]['total'], 'buildsSuccessTotal' => $usage[$metrics[9]]['total'], 'buildsFailedTotal' => $usage[$metrics[10]]['total'], - 'requestsTotal' => $usage[$metrics[11]]['total'], - 'inboundTotal' => $usage[$metrics[12]]['total'], - 'outboundTotal' => $usage[$metrics[13]]['total'], + 'screenshotsSuccessTotal' => $screenshotsSuccessTotal, + 'screenshotsFailedTotal' => $screenshotsFailedTotal, + 'screenshotsSuccessRate' => $screenshotsTotal === 0 ? 0 : $screenshotsSuccessTotal / $screenshotsTotal, + 'requestsTotal' => $usage[$metrics[13]]['total'], + 'inboundTotal' => $usage[$metrics[14]]['total'], + 'outboundTotal' => $usage[$metrics[15]]['total'], 'deployments' => $usage[$metrics[0]]['data'], 'deploymentsStorage' => $usage[$metrics[1]]['data'], 'builds' => $usage[$metrics[2]]['data'], @@ -167,9 +175,11 @@ class Get extends Base 'executionsMbSeconds' => $usage[$metrics[8]]['data'], 'buildsSuccess' => $usage[$metrics[9]]['data'], 'buildsFailed' => $usage[$metrics[10]]['data'], - 'requests' => $usage[$metrics[11]]['data'], - 'inbound' => $usage[$metrics[12]]['data'], - 'outbound' => $usage[$metrics[13]]['data'], + 'screenshotsSuccess' => $usage[$metrics[11]]['data'], + 'screenshotsFailed' => $usage[$metrics[12]]['data'], + 'requests' => $usage[$metrics[13]]['data'], + 'inbound' => $usage[$metrics[14]]['data'], + 'outbound' => $usage[$metrics[15]]['data'], ]), Response::MODEL_USAGE_SITE); } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Usage/XList.php b/src/Appwrite/Platform/Modules/Sites/Http/Usage/XList.php index a90cb0cab9..3b3c759fd5 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Usage/XList.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Usage/XList.php @@ -74,6 +74,8 @@ class XList extends Base str_replace("{resourceType}", RESOURCE_TYPE_SITES, METRIC_RESOURCE_TYPE_EXECUTIONS_MB_SECONDS), str_replace("{resourceType}", RESOURCE_TYPE_SITES, METRIC_RESOURCE_TYPE_BUILDS_SUCCESS), str_replace("{resourceType}", RESOURCE_TYPE_SITES, METRIC_RESOURCE_TYPE_BUILDS_FAILED), + str_replace("{resourceType}", RESOURCE_TYPE_SITES, METRIC_RESOURCE_TYPE_SCREENSHOTS_SUCCESS), + str_replace("{resourceType}", RESOURCE_TYPE_SITES, METRIC_RESOURCE_TYPE_SCREENSHOTS_FAILED), METRIC_SITES_REQUESTS, METRIC_SITES_INBOUND, METRIC_SITES_OUTBOUND, @@ -122,6 +124,10 @@ class XList extends Base ]; } } + $screenshotsSuccessTotal = $usage[$metrics[12]]['total'] ?? 0; + $screenshotsFailedTotal = $usage[$metrics[13]]['total'] ?? 0; + $screenshotsTotal = $screenshotsSuccessTotal + $screenshotsFailedTotal; + $response->dynamic(new Document([ 'range' => $range, 'sitesTotal' => $usage[$metrics[0]]['total'], @@ -136,9 +142,12 @@ class XList extends Base 'executionsMbSecondsTotal' => $usage[$metrics[9]]['total'], 'buildsSuccessTotal' => $usage[$metrics[10]]['total'], 'buildsFailedTotal' => $usage[$metrics[11]]['total'], - 'requestsTotal' => $usage[$metrics[12]]['total'], - 'inboundTotal' => $usage[$metrics[13]]['total'], - 'outboundTotal' => $usage[$metrics[14]]['total'], + 'screenshotsSuccessTotal' => $screenshotsSuccessTotal, + 'screenshotsFailedTotal' => $screenshotsFailedTotal, + 'screenshotsSuccessRate' => $screenshotsTotal === 0 ? 0 : $screenshotsSuccessTotal / $screenshotsTotal, + 'requestsTotal' => $usage[$metrics[14]]['total'], + 'inboundTotal' => $usage[$metrics[15]]['total'], + 'outboundTotal' => $usage[$metrics[16]]['total'], 'sites' => $usage[$metrics[0]]['data'], 'deployments' => $usage[$metrics[1]]['data'], 'deploymentsStorage' => $usage[$metrics[2]]['data'], @@ -151,9 +160,11 @@ class XList extends Base 'executionsMbSeconds' => $usage[$metrics[9]]['data'], 'buildsSuccess' => $usage[$metrics[10]]['data'], 'buildsFailed' => $usage[$metrics[11]]['data'], - 'requests' => $usage[$metrics[12]]['data'], - 'inbound' => $usage[$metrics[13]]['data'], - 'outbound' => $usage[$metrics[14]]['data'], + 'screenshotsSuccess' => $usage[$metrics[12]]['data'], + 'screenshotsFailed' => $usage[$metrics[13]]['data'], + 'requests' => $usage[$metrics[14]]['data'], + 'inbound' => $usage[$metrics[15]]['data'], + 'outbound' => $usage[$metrics[16]]['data'], ]), Response::MODEL_USAGE_SITES); } } diff --git a/src/Appwrite/Utopia/Response/Model/UsageSite.php b/src/Appwrite/Utopia/Response/Model/UsageSite.php index c45dc831cd..27828426d3 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageSite.php +++ b/src/Appwrite/Utopia/Response/Model/UsageSite.php @@ -49,6 +49,38 @@ class UsageSite extends UsageFunction 'example' => [], 'array' => true ]) + ->addRule('screenshotsSuccessTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of successful site screenshots.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('screenshotsFailedTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of failed site screenshots.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('screenshotsSuccessRate', [ + 'type' => self::TYPE_FLOAT, + 'description' => 'Success rate of site screenshots from 0 to 1.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('screenshotsSuccess', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of successful site screenshots per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('screenshotsFailed', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of failed site screenshots per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } diff --git a/src/Appwrite/Utopia/Response/Model/UsageSites.php b/src/Appwrite/Utopia/Response/Model/UsageSites.php index fea87c7718..241e08e6ba 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageSites.php +++ b/src/Appwrite/Utopia/Response/Model/UsageSites.php @@ -122,6 +122,38 @@ class UsageSites extends Model 'example' => [], 'array' => true ]) + ->addRule('screenshotsSuccessTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of successful site screenshots.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('screenshotsFailedTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of failed site screenshots.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('screenshotsSuccessRate', [ + 'type' => self::TYPE_FLOAT, + 'description' => 'Success rate of site screenshots from 0 to 1.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('screenshotsSuccess', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of successful site screenshots per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('screenshotsFailed', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of failed site screenshots per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ->addRule('deployments', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of sites deployment per period.', diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index f6eb963967..0d2c0a7ed7 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1675,11 +1675,14 @@ class UsageTest extends Scope ); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(30, count($response['body'])); + $this->assertEquals(35, count($response['body'])); $this->assertEquals('30d', $response['body']['range']); $this->assertIsArray($response['body']['deployments']); $this->assertEquals($deploymentsSuccess, $response['body']['buildsSuccessTotal']); $this->assertEquals($deploymentsFailed, $response['body']['buildsFailedTotal']); + $this->assertGreaterThanOrEqual(1, $response['body']['screenshotsSuccessTotal']); + $this->assertEquals(0, $response['body']['screenshotsFailedTotal']); + $this->assertEquals(1.0, $response['body']['screenshotsSuccessRate']); $this->assertIsArray($response['body']['deploymentsStorage']); $this->assertIsNumeric($response['body']['deploymentsStorageTotal']); $this->assertIsNumeric($response['body']['buildsMbSecondsTotal']); @@ -1692,6 +1695,8 @@ class UsageTest extends Scope $this->assertIsArray($response['body']['executionsMbSeconds']); $this->assertIsArray($response['body']['buildsSuccess']); $this->assertIsArray($response['body']['buildsFailed']); + $this->assertIsArray($response['body']['screenshotsSuccess']); + $this->assertIsArray($response['body']['screenshotsFailed']); $this->assertIsArray($response['body']['requests']); $this->assertIsArray($response['body']['inbound']); $this->assertIsArray($response['body']['outbound']); @@ -1709,7 +1714,7 @@ class UsageTest extends Scope ); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(31, count($response['body'])); + $this->assertEquals(36, count($response['body'])); $this->assertEquals($response['body']['range'], '30d'); $this->assertIsArray($response['body']['sites']); $this->assertIsArray($response['body']['deployments']); @@ -1722,6 +1727,11 @@ class UsageTest extends Scope $this->assertIsArray($response['body']['executionsMbSeconds']); $this->assertIsArray($response['body']['buildsSuccess']); $this->assertIsArray($response['body']['buildsFailed']); + $this->assertGreaterThanOrEqual(1, $response['body']['screenshotsSuccessTotal']); + $this->assertEquals(0, $response['body']['screenshotsFailedTotal']); + $this->assertEquals(1.0, $response['body']['screenshotsSuccessRate']); + $this->assertIsArray($response['body']['screenshotsSuccess']); + $this->assertIsArray($response['body']['screenshotsFailed']); $this->assertIsArray($response['body']['requests']); $this->assertIsArray($response['body']['inbound']); $this->assertIsArray($response['body']['outbound']); From 37540aa4f5448009525de189d1e4bda98440bb65 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Wed, 15 Apr 2026 11:52:39 +0530 Subject: [PATCH 02/20] addressed comments --- .../Modules/Functions/Workers/Screenshots.php | 30 +++-- .../Utopia/Response/Model/UsageSite.php | 64 +++++----- .../Utopia/Response/Model/UsageSites.php | 116 +++++++++--------- 3 files changed, 107 insertions(+), 103 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php b/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php index fa7e394710..dbd0e217c9 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php @@ -62,6 +62,7 @@ class Screenshots extends Action Console::log('Screenshot action started'); $payload = $message->getPayload() ?? []; + $screenshotCompleted = false; if (empty($payload)) { throw new \Exception('Missing payload'); @@ -262,13 +263,7 @@ class Screenshots extends Action 'deploymentScreenshotDark' => $deployment->getAttribute('screenshotDark', ''), 'deploymentScreenshotLight' => $deployment->getAttribute('screenshotLight', ''), ])); - - $this->publishUsage( - project: $project, - site: $site, - publisherForUsage: $publisherForUsage, - metric: METRIC_SCREENSHOTS_SUCCESS - ); + $screenshotCompleted = true; } catch (\Throwable $th) { Console::warning("Screenshot failed to generate:"); Console::warning($th->getMessage()); @@ -277,15 +272,24 @@ class Screenshots extends Action $date = \date('H:i:s'); $this->appendToLogs($dbForProject, $deployment->getId(), $queueForRealtime, "[$date] [appwrite] Screenshot capturing failed. Deployment will continue. \n"); - $this->publishUsage( - project: $project, - site: $site, - publisherForUsage: $publisherForUsage, - metric: METRIC_SCREENSHOTS_FAILED - ); + if (!$screenshotCompleted) { + $this->publishUsage( + project: $project, + site: $site, + publisherForUsage: $publisherForUsage, + metric: METRIC_SCREENSHOTS_FAILED + ); + } throw $th; } + + $this->publishUsage( + project: $project, + site: $site, + publisherForUsage: $publisherForUsage, + metric: METRIC_SCREENSHOTS_SUCCESS + ); } protected function publishUsage( diff --git a/src/Appwrite/Utopia/Response/Model/UsageSite.php b/src/Appwrite/Utopia/Response/Model/UsageSite.php index 27828426d3..af41a34639 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageSite.php +++ b/src/Appwrite/Utopia/Response/Model/UsageSite.php @@ -10,6 +10,38 @@ class UsageSite extends UsageFunction { parent::__construct(); $this + ->addRule('screenshotsSuccessTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of successful site screenshots.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('screenshotsFailedTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of failed site screenshots.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('screenshotsSuccessRate', [ + 'type' => self::TYPE_FLOAT, + 'description' => 'Success rate of site screenshots from 0 to 1.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('screenshotsSuccess', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of successful site screenshots per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('screenshotsFailed', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of failed site screenshots per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ->addRule('requestsTotal', [ 'type' => self::TYPE_INTEGER, 'description' => 'Total aggregated number of requests.', @@ -49,38 +81,6 @@ class UsageSite extends UsageFunction 'example' => [], 'array' => true ]) - ->addRule('screenshotsSuccessTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total aggregated number of successful site screenshots.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('screenshotsFailedTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total aggregated number of failed site screenshots.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('screenshotsSuccessRate', [ - 'type' => self::TYPE_FLOAT, - 'description' => 'Success rate of site screenshots from 0 to 1.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('screenshotsSuccess', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of successful site screenshots per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - ->addRule('screenshotsFailed', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of failed site screenshots per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) ; } diff --git a/src/Appwrite/Utopia/Response/Model/UsageSites.php b/src/Appwrite/Utopia/Response/Model/UsageSites.php index 241e08e6ba..94ee6e645f 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageSites.php +++ b/src/Appwrite/Utopia/Response/Model/UsageSites.php @@ -83,6 +83,36 @@ class UsageSites extends Model 'default' => 0, 'example' => 0, ]) + ->addRule('buildsSuccessTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of successful site builds.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('buildsFailedTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of failed site builds.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('screenshotsSuccessTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of successful site screenshots.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('screenshotsFailedTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of failed site screenshots.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('screenshotsSuccessRate', [ + 'type' => self::TYPE_FLOAT, + 'description' => 'Success rate of site screenshots from 0 to 1.', + 'default' => 0, + 'example' => 0, + ]) ->addRule('requestsTotal', [ 'type' => self::TYPE_INTEGER, 'description' => 'Total aggregated number of requests.', @@ -122,38 +152,6 @@ class UsageSites extends Model 'example' => [], 'array' => true ]) - ->addRule('screenshotsSuccessTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total aggregated number of successful site screenshots.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('screenshotsFailedTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total aggregated number of failed site screenshots.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('screenshotsSuccessRate', [ - 'type' => self::TYPE_FLOAT, - 'description' => 'Success rate of site screenshots from 0 to 1.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('screenshotsSuccess', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of successful site screenshots per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - ->addRule('screenshotsFailed', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of failed site screenshots per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) ->addRule('deployments', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of sites deployment per period.', @@ -168,18 +166,6 @@ class UsageSites extends Model 'example' => [], 'array' => true ]) - ->addRule('buildsSuccessTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total aggregated number of successful site builds.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('buildsFailedTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total aggregated number of failed site builds.', - 'default' => 0, - 'example' => 0, - ]) ->addRule('builds', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of sites build per period.', @@ -208,6 +194,34 @@ class UsageSites extends Model 'example' => [], 'array' => true ]) + ->addRule('buildsSuccess', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of successful site builds per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('buildsFailed', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of failed site builds per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('screenshotsSuccess', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of successful site screenshots per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('screenshotsFailed', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of failed site screenshots per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ->addRule('executions', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of sites execution per period.', @@ -229,20 +243,6 @@ class UsageSites extends Model 'example' => [], 'array' => true ]) - ->addRule('buildsSuccess', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of successful site builds per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - ->addRule('buildsFailed', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of failed site builds per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) ; } From 23233d0dcb2b9550e56d1cc2f6dc9cf47888aef5 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Wed, 15 Apr 2026 11:59:02 +0530 Subject: [PATCH 03/20] add try catch --- .../Modules/Functions/Workers/Screenshots.php | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php b/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php index dbd0e217c9..d1f2b0d0bf 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php @@ -273,12 +273,16 @@ class Screenshots extends Action $this->appendToLogs($dbForProject, $deployment->getId(), $queueForRealtime, "[$date] [appwrite] Screenshot capturing failed. Deployment will continue. \n"); if (!$screenshotCompleted) { - $this->publishUsage( - project: $project, - site: $site, - publisherForUsage: $publisherForUsage, - metric: METRIC_SCREENSHOTS_FAILED - ); + try { + $this->publishUsage( + project: $project, + site: $site, + publisherForUsage: $publisherForUsage, + metric: METRIC_SCREENSHOTS_FAILED + ); + } catch (\Throwable) { + // Usage publish is best-effort; preserve the original screenshot exception. + } } throw $th; From 730c62bda60cbb32da196c01b01c7ac4d6c98c22 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Wed, 15 Apr 2026 18:35:09 +0530 Subject: [PATCH 04/20] remove usage stats --- app/init/constants.php | 6 -- .../Modules/Functions/Workers/Screenshots.php | 74 ++++------------ .../Platform/Modules/Sites/Http/Usage/Get.php | 22 ++--- .../Modules/Sites/Http/Usage/XList.php | 23 ++--- .../Utopia/Response/Model/UsageSite.php | 32 ------- .../Utopia/Response/Model/UsageSites.php | 84 ++++++------------- tests/e2e/General/UsageTest.php | 14 +--- 7 files changed, 57 insertions(+), 198 deletions(-) diff --git a/app/init/constants.php b/app/init/constants.php index 57763df911..f2127cd666 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -351,8 +351,6 @@ const METRIC_BUILDS_STORAGE = 'builds.storage'; const METRIC_BUILDS_COMPUTE = 'builds.compute'; const METRIC_BUILDS_COMPUTE_SUCCESS = 'builds.compute.success'; const METRIC_BUILDS_COMPUTE_FAILED = 'builds.compute.failed'; -const METRIC_SCREENSHOTS_SUCCESS = 'screenshots.success'; -const METRIC_SCREENSHOTS_FAILED = 'screenshots.failed'; const METRIC_BUILDS_MB_SECONDS = 'builds.mbSeconds'; const METRIC_EXECUTIONS = 'executions'; const METRIC_EXECUTIONS_COMPUTE = 'executions.compute'; @@ -365,8 +363,6 @@ const METRIC_RESOURCE_TYPE_ID_BUILDS_FAILED = '{resourceType}.{resourceInternal const METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE = '{resourceType}.{resourceInternalId}.builds.compute'; const METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_SUCCESS = '{resourceType}.{resourceInternalId}.builds.compute.success'; const METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_FAILED = '{resourceType}.{resourceInternalId}.builds.compute.failed'; -const METRIC_RESOURCE_TYPE_ID_SCREENSHOTS_SUCCESS = '{resourceType}.{resourceInternalId}.screenshots.success'; -const METRIC_RESOURCE_TYPE_ID_SCREENSHOTS_FAILED = '{resourceType}.{resourceInternalId}.screenshots.failed'; const METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS = '{resourceType}.{resourceInternalId}.builds.mbSeconds'; const METRIC_RESOURCE_TYPE_ID_BUILDS = '{resourceType}.{resourceInternalId}.builds'; const METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE = '{resourceType}.{resourceInternalId}.builds.storage'; @@ -380,8 +376,6 @@ const METRIC_RESOURCE_TYPE_BUILDS_FAILED = '{resourceType}.builds.failed'; const METRIC_RESOURCE_TYPE_BUILDS_COMPUTE = '{resourceType}.builds.compute'; const METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_SUCCESS = '{resourceType}.builds.compute.success'; const METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_FAILED = '{resourceType}.builds.compute.failed'; -const METRIC_RESOURCE_TYPE_SCREENSHOTS_SUCCESS = '{resourceType}.screenshots.success'; -const METRIC_RESOURCE_TYPE_SCREENSHOTS_FAILED = '{resourceType}.screenshots.failed'; const METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS = '{resourceType}.builds.mbSeconds'; const METRIC_RESOURCE_TYPE_BUILDS = '{resourceType}.builds'; const METRIC_RESOURCE_TYPE_BUILDS_STORAGE = '{resourceType}.builds.storage'; diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php b/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php index bac72b993f..0cb42822b3 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php @@ -4,12 +4,9 @@ namespace Appwrite\Platform\Modules\Functions\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Message\Screenshot; -use Appwrite\Event\Message\Usage as UsageMessage; -use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; use Appwrite\Permission; use Appwrite\Role; -use Appwrite\Usage\Context; use Exception; use Utopia\Compression\Compression; use Utopia\Config\Config; @@ -23,6 +20,7 @@ use Utopia\Platform\Action; use Utopia\Queue\Message; use Utopia\Storage\Device; use Utopia\System\System; +use Utopia\Telemetry\Adapter as Telemetry; use function Swoole\Coroutine\batch; @@ -47,7 +45,7 @@ class Screenshots extends Action ->inject('dbForProject') ->inject('project') ->inject('deviceForFiles') - ->inject('publisherForUsage') + ->inject('telemetry') ->callback($this->action(...)); } @@ -58,12 +56,11 @@ class Screenshots extends Action Database $dbForProject, Document $project, Device $deviceForFiles, - UsagePublisher $publisherForUsage + Telemetry $telemetry ): void { Console::log('Screenshot action started'); $payload = $message->getPayload() ?? []; - $screenshotCompleted = false; if (empty($payload)) { throw new \Exception('Missing payload'); @@ -266,7 +263,6 @@ class Screenshots extends Action 'deploymentScreenshotDark' => $deployment->getAttribute('screenshotDark', ''), 'deploymentScreenshotLight' => $deployment->getAttribute('screenshotLight', ''), ])); - $screenshotCompleted = true; } catch (\Throwable $th) { Console::warning("Screenshot failed to generate:"); Console::warning($th->getMessage()); @@ -275,62 +271,26 @@ class Screenshots extends Action $date = \date('H:i:s'); $this->appendToLogs($dbForProject, $deployment->getId(), $queueForRealtime, "[$date] [appwrite] Screenshot capturing failed. Deployment will continue. \n"); - if (!$screenshotCompleted) { - try { - $this->publishUsage( - project: $project, - site: $site, - publisherForUsage: $publisherForUsage, - metric: METRIC_SCREENSHOTS_FAILED - ); - } catch (\Throwable) { - // Usage publish is best-effort; preserve the original screenshot exception. - } - } + $this->recordTelemetry($telemetry, 'failure'); throw $th; } - $this->publishUsage( - project: $project, - site: $site, - publisherForUsage: $publisherForUsage, - metric: METRIC_SCREENSHOTS_SUCCESS - ); + $this->recordTelemetry($telemetry, 'success'); } - protected function publishUsage( - Document $project, - Document $site, - UsagePublisher $publisherForUsage, - string $metric - ): void { - [$resourceMetric, $resourceIdMetric] = match ($metric) { - METRIC_SCREENSHOTS_SUCCESS => [ - METRIC_RESOURCE_TYPE_SCREENSHOTS_SUCCESS, - METRIC_RESOURCE_TYPE_ID_SCREENSHOTS_SUCCESS, - ], - METRIC_SCREENSHOTS_FAILED => [ - METRIC_RESOURCE_TYPE_SCREENSHOTS_FAILED, - METRIC_RESOURCE_TYPE_ID_SCREENSHOTS_FAILED, - ], - default => throw new \InvalidArgumentException('Unknown screenshot metric: ' . $metric), - }; - - $usage = (new Context()) - ->addMetric($metric, 1) - ->addMetric(str_replace('{resourceType}', RESOURCE_TYPE_SITES, $resourceMetric), 1) - ->addMetric(str_replace( - ['{resourceType}', '{resourceInternalId}'], - [RESOURCE_TYPE_SITES, $site->getSequence()], - $resourceIdMetric - ), 1); - - $publisherForUsage->enqueue(new UsageMessage( - project: $project, - metrics: $usage->getMetrics(), - reduce: $usage->getReduce() - )); + protected function recordTelemetry(Telemetry $telemetry, string $result): void + { + try { + $telemetry + ->createCounter('worker.screenshots.capture') + ->add(1, [ + 'resourceType' => RESOURCE_TYPE_SITES, + 'result' => $result, + ]); + } catch (\Throwable) { + // Telemetry should never affect screenshot processing. + } } protected function appendToLogs(Database $dbForProject, string $deploymentId, Realtime $queueForRealtime, string $logs) diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Usage/Get.php b/src/Appwrite/Platform/Modules/Sites/Http/Usage/Get.php index a1af4bd8fd..a6768462d1 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Usage/Get.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Usage/Get.php @@ -87,8 +87,6 @@ class Get extends Base str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $site->getSequence()], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_MB_SECONDS), str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $site->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_SUCCESS), str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $site->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_FAILED), - str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $site->getSequence()], METRIC_RESOURCE_TYPE_ID_SCREENSHOTS_SUCCESS), - str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_SITES, $site->getSequence()], METRIC_RESOURCE_TYPE_ID_SCREENSHOTS_FAILED), str_replace(['{siteInternalId}'], [$site->getSequence()], METRIC_SITES_ID_REQUESTS), str_replace(['{siteInternalId}'], [$site->getSequence()], METRIC_SITES_ID_INBOUND), str_replace(['{siteInternalId}'], [$site->getSequence()], METRIC_SITES_ID_OUTBOUND), @@ -141,9 +139,6 @@ class Get extends Base $buildsTimeTotal = $usage[$metrics[4]]['total'] ?? 0; $buildsTotal = $usage[$metrics[2]]['total'] ?? 0; - $screenshotsSuccessTotal = $usage[$metrics[11]]['total'] ?? 0; - $screenshotsFailedTotal = $usage[$metrics[12]]['total'] ?? 0; - $screenshotsTotal = $screenshotsSuccessTotal + $screenshotsFailedTotal; $response->dynamic(new Document([ 'range' => $range, 'deploymentsTotal' => $usage[$metrics[0]]['total'], @@ -158,12 +153,9 @@ class Get extends Base 'executionsMbSecondsTotal' => $usage[$metrics[8]]['total'], 'buildsSuccessTotal' => $usage[$metrics[9]]['total'], 'buildsFailedTotal' => $usage[$metrics[10]]['total'], - 'screenshotsSuccessTotal' => $screenshotsSuccessTotal, - 'screenshotsFailedTotal' => $screenshotsFailedTotal, - 'screenshotsSuccessRate' => $screenshotsTotal === 0 ? 0 : $screenshotsSuccessTotal / $screenshotsTotal, - 'requestsTotal' => $usage[$metrics[13]]['total'], - 'inboundTotal' => $usage[$metrics[14]]['total'], - 'outboundTotal' => $usage[$metrics[15]]['total'], + 'requestsTotal' => $usage[$metrics[11]]['total'], + 'inboundTotal' => $usage[$metrics[12]]['total'], + 'outboundTotal' => $usage[$metrics[13]]['total'], 'deployments' => $usage[$metrics[0]]['data'], 'deploymentsStorage' => $usage[$metrics[1]]['data'], 'builds' => $usage[$metrics[2]]['data'], @@ -175,11 +167,9 @@ class Get extends Base 'executionsMbSeconds' => $usage[$metrics[8]]['data'], 'buildsSuccess' => $usage[$metrics[9]]['data'], 'buildsFailed' => $usage[$metrics[10]]['data'], - 'screenshotsSuccess' => $usage[$metrics[11]]['data'], - 'screenshotsFailed' => $usage[$metrics[12]]['data'], - 'requests' => $usage[$metrics[13]]['data'], - 'inbound' => $usage[$metrics[14]]['data'], - 'outbound' => $usage[$metrics[15]]['data'], + 'requests' => $usage[$metrics[11]]['data'], + 'inbound' => $usage[$metrics[12]]['data'], + 'outbound' => $usage[$metrics[13]]['data'], ]), Response::MODEL_USAGE_SITE); } } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Usage/XList.php b/src/Appwrite/Platform/Modules/Sites/Http/Usage/XList.php index 3b3c759fd5..a90cb0cab9 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Usage/XList.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Usage/XList.php @@ -74,8 +74,6 @@ class XList extends Base str_replace("{resourceType}", RESOURCE_TYPE_SITES, METRIC_RESOURCE_TYPE_EXECUTIONS_MB_SECONDS), str_replace("{resourceType}", RESOURCE_TYPE_SITES, METRIC_RESOURCE_TYPE_BUILDS_SUCCESS), str_replace("{resourceType}", RESOURCE_TYPE_SITES, METRIC_RESOURCE_TYPE_BUILDS_FAILED), - str_replace("{resourceType}", RESOURCE_TYPE_SITES, METRIC_RESOURCE_TYPE_SCREENSHOTS_SUCCESS), - str_replace("{resourceType}", RESOURCE_TYPE_SITES, METRIC_RESOURCE_TYPE_SCREENSHOTS_FAILED), METRIC_SITES_REQUESTS, METRIC_SITES_INBOUND, METRIC_SITES_OUTBOUND, @@ -124,10 +122,6 @@ class XList extends Base ]; } } - $screenshotsSuccessTotal = $usage[$metrics[12]]['total'] ?? 0; - $screenshotsFailedTotal = $usage[$metrics[13]]['total'] ?? 0; - $screenshotsTotal = $screenshotsSuccessTotal + $screenshotsFailedTotal; - $response->dynamic(new Document([ 'range' => $range, 'sitesTotal' => $usage[$metrics[0]]['total'], @@ -142,12 +136,9 @@ class XList extends Base 'executionsMbSecondsTotal' => $usage[$metrics[9]]['total'], 'buildsSuccessTotal' => $usage[$metrics[10]]['total'], 'buildsFailedTotal' => $usage[$metrics[11]]['total'], - 'screenshotsSuccessTotal' => $screenshotsSuccessTotal, - 'screenshotsFailedTotal' => $screenshotsFailedTotal, - 'screenshotsSuccessRate' => $screenshotsTotal === 0 ? 0 : $screenshotsSuccessTotal / $screenshotsTotal, - 'requestsTotal' => $usage[$metrics[14]]['total'], - 'inboundTotal' => $usage[$metrics[15]]['total'], - 'outboundTotal' => $usage[$metrics[16]]['total'], + 'requestsTotal' => $usage[$metrics[12]]['total'], + 'inboundTotal' => $usage[$metrics[13]]['total'], + 'outboundTotal' => $usage[$metrics[14]]['total'], 'sites' => $usage[$metrics[0]]['data'], 'deployments' => $usage[$metrics[1]]['data'], 'deploymentsStorage' => $usage[$metrics[2]]['data'], @@ -160,11 +151,9 @@ class XList extends Base 'executionsMbSeconds' => $usage[$metrics[9]]['data'], 'buildsSuccess' => $usage[$metrics[10]]['data'], 'buildsFailed' => $usage[$metrics[11]]['data'], - 'screenshotsSuccess' => $usage[$metrics[12]]['data'], - 'screenshotsFailed' => $usage[$metrics[13]]['data'], - 'requests' => $usage[$metrics[14]]['data'], - 'inbound' => $usage[$metrics[15]]['data'], - 'outbound' => $usage[$metrics[16]]['data'], + 'requests' => $usage[$metrics[12]]['data'], + 'inbound' => $usage[$metrics[13]]['data'], + 'outbound' => $usage[$metrics[14]]['data'], ]), Response::MODEL_USAGE_SITES); } } diff --git a/src/Appwrite/Utopia/Response/Model/UsageSite.php b/src/Appwrite/Utopia/Response/Model/UsageSite.php index af41a34639..c45dc831cd 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageSite.php +++ b/src/Appwrite/Utopia/Response/Model/UsageSite.php @@ -10,38 +10,6 @@ class UsageSite extends UsageFunction { parent::__construct(); $this - ->addRule('screenshotsSuccessTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total aggregated number of successful site screenshots.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('screenshotsFailedTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total aggregated number of failed site screenshots.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('screenshotsSuccessRate', [ - 'type' => self::TYPE_FLOAT, - 'description' => 'Success rate of site screenshots from 0 to 1.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('screenshotsSuccess', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of successful site screenshots per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - ->addRule('screenshotsFailed', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of failed site screenshots per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) ->addRule('requestsTotal', [ 'type' => self::TYPE_INTEGER, 'description' => 'Total aggregated number of requests.', diff --git a/src/Appwrite/Utopia/Response/Model/UsageSites.php b/src/Appwrite/Utopia/Response/Model/UsageSites.php index 94ee6e645f..fea87c7718 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageSites.php +++ b/src/Appwrite/Utopia/Response/Model/UsageSites.php @@ -83,36 +83,6 @@ class UsageSites extends Model 'default' => 0, 'example' => 0, ]) - ->addRule('buildsSuccessTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total aggregated number of successful site builds.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('buildsFailedTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total aggregated number of failed site builds.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('screenshotsSuccessTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total aggregated number of successful site screenshots.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('screenshotsFailedTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total aggregated number of failed site screenshots.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('screenshotsSuccessRate', [ - 'type' => self::TYPE_FLOAT, - 'description' => 'Success rate of site screenshots from 0 to 1.', - 'default' => 0, - 'example' => 0, - ]) ->addRule('requestsTotal', [ 'type' => self::TYPE_INTEGER, 'description' => 'Total aggregated number of requests.', @@ -166,6 +136,18 @@ class UsageSites extends Model 'example' => [], 'array' => true ]) + ->addRule('buildsSuccessTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of successful site builds.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('buildsFailedTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of failed site builds.', + 'default' => 0, + 'example' => 0, + ]) ->addRule('builds', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of sites build per period.', @@ -194,34 +176,6 @@ class UsageSites extends Model 'example' => [], 'array' => true ]) - ->addRule('buildsSuccess', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of successful site builds per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - ->addRule('buildsFailed', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of failed site builds per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - ->addRule('screenshotsSuccess', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of successful site screenshots per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - ->addRule('screenshotsFailed', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of failed site screenshots per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) ->addRule('executions', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of sites execution per period.', @@ -243,6 +197,20 @@ class UsageSites extends Model 'example' => [], 'array' => true ]) + ->addRule('buildsSuccess', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of successful site builds per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('buildsFailed', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of failed site builds per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 0d2c0a7ed7..f6eb963967 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1675,14 +1675,11 @@ class UsageTest extends Scope ); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(35, count($response['body'])); + $this->assertEquals(30, count($response['body'])); $this->assertEquals('30d', $response['body']['range']); $this->assertIsArray($response['body']['deployments']); $this->assertEquals($deploymentsSuccess, $response['body']['buildsSuccessTotal']); $this->assertEquals($deploymentsFailed, $response['body']['buildsFailedTotal']); - $this->assertGreaterThanOrEqual(1, $response['body']['screenshotsSuccessTotal']); - $this->assertEquals(0, $response['body']['screenshotsFailedTotal']); - $this->assertEquals(1.0, $response['body']['screenshotsSuccessRate']); $this->assertIsArray($response['body']['deploymentsStorage']); $this->assertIsNumeric($response['body']['deploymentsStorageTotal']); $this->assertIsNumeric($response['body']['buildsMbSecondsTotal']); @@ -1695,8 +1692,6 @@ class UsageTest extends Scope $this->assertIsArray($response['body']['executionsMbSeconds']); $this->assertIsArray($response['body']['buildsSuccess']); $this->assertIsArray($response['body']['buildsFailed']); - $this->assertIsArray($response['body']['screenshotsSuccess']); - $this->assertIsArray($response['body']['screenshotsFailed']); $this->assertIsArray($response['body']['requests']); $this->assertIsArray($response['body']['inbound']); $this->assertIsArray($response['body']['outbound']); @@ -1714,7 +1709,7 @@ class UsageTest extends Scope ); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(36, count($response['body'])); + $this->assertEquals(31, count($response['body'])); $this->assertEquals($response['body']['range'], '30d'); $this->assertIsArray($response['body']['sites']); $this->assertIsArray($response['body']['deployments']); @@ -1727,11 +1722,6 @@ class UsageTest extends Scope $this->assertIsArray($response['body']['executionsMbSeconds']); $this->assertIsArray($response['body']['buildsSuccess']); $this->assertIsArray($response['body']['buildsFailed']); - $this->assertGreaterThanOrEqual(1, $response['body']['screenshotsSuccessTotal']); - $this->assertEquals(0, $response['body']['screenshotsFailedTotal']); - $this->assertEquals(1.0, $response['body']['screenshotsSuccessRate']); - $this->assertIsArray($response['body']['screenshotsSuccess']); - $this->assertIsArray($response['body']['screenshotsFailed']); $this->assertIsArray($response['body']['requests']); $this->assertIsArray($response['body']['inbound']); $this->assertIsArray($response['body']['outbound']); From 93b9500a95af55447e438c2b04fadfd2e2a172e2 Mon Sep 17 00:00:00 2001 From: harsh mahajan Date: Thu, 16 Apr 2026 15:22:45 +0530 Subject: [PATCH 05/20] align it with cloud pattern --- .../Modules/Functions/Workers/Screenshots.php | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php b/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php index 0cb42822b3..8fed46aa00 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Screenshots.php @@ -21,6 +21,7 @@ use Utopia\Queue\Message; use Utopia\Storage\Device; use Utopia\System\System; use Utopia\Telemetry\Adapter as Telemetry; +use Utopia\Telemetry\Counter; use function Swoole\Coroutine\batch; @@ -67,6 +68,7 @@ class Screenshots extends Action } $screenshotMessage = Screenshot::fromArray($payload); + $counter = $telemetry->createCounter('worker.screenshots.capture'); Console::log('Site screenshot started'); @@ -271,23 +273,21 @@ class Screenshots extends Action $date = \date('H:i:s'); $this->appendToLogs($dbForProject, $deployment->getId(), $queueForRealtime, "[$date] [appwrite] Screenshot capturing failed. Deployment will continue. \n"); - $this->recordTelemetry($telemetry, 'failure'); + $this->recordTelemetry($counter, 'failure'); throw $th; } - $this->recordTelemetry($telemetry, 'success'); + $this->recordTelemetry($counter, 'success'); } - protected function recordTelemetry(Telemetry $telemetry, string $result): void + protected function recordTelemetry(Counter $counter, string $result): void { try { - $telemetry - ->createCounter('worker.screenshots.capture') - ->add(1, [ - 'resourceType' => RESOURCE_TYPE_SITES, - 'result' => $result, - ]); + $counter->add(1, [ + 'resourceType' => RESOURCE_TYPE_SITES, + 'result' => $result, + ]); } catch (\Throwable) { // Telemetry should never affect screenshot processing. } From 13f48797d43e01532956a1058286762d5998371c Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sat, 18 Apr 2026 23:01:41 +0530 Subject: [PATCH 06/20] added project region --- src/Appwrite/Platform/Workers/Migrations.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 118ff7acf9..3656ea712b 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -195,9 +195,16 @@ class Migrations extends Action $migrationOptions = $migration->getAttribute('options'); /** @var Database|null $projectDB */ $projectDB = null; + $useAppwriteApiSource = false; if ($credentials['projectId']) { $this->sourceProject = $this->dbForPlatform->getDocument('projects', $credentials['projectId']); - $projectDB = call_user_func($this->getProjectDB, $this->sourceProject); + $sourceRegion = $this->sourceProject->getAttribute('region', 'default'); + $destinationRegion = $this->project->getAttribute('region', 'default'); + $useAppwriteApiSource = $source === SourceAppwrite::getName() + && $sourceRegion !== $destinationRegion; + if (! $useAppwriteApiSource) { + $projectDB = call_user_func($this->getProjectDB, $this->sourceProject); + } } $getDatabasesDB = fn (Document $database): Database => $this->getDatabasesDBForProject($database); @@ -233,7 +240,7 @@ class Migrations extends Action $credentials['endpoint'], $credentials['apiKey'], $getDatabasesDB, - SourceAppwrite::SOURCE_DATABASE, + $useAppwriteApiSource ? SourceAppwrite::SOURCE_API : SourceAppwrite::SOURCE_DATABASE, $projectDB, $queries ), From 2793bcac3893a0c9d5c9da27aa51d97a52dd1353 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sat, 18 Apr 2026 23:11:04 +0530 Subject: [PATCH 07/20] updated --- src/Appwrite/Platform/Workers/Migrations.php | 21 +++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 3656ea712b..7a16b02642 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -198,12 +198,18 @@ class Migrations extends Action $useAppwriteApiSource = false; if ($credentials['projectId']) { $this->sourceProject = $this->dbForPlatform->getDocument('projects', $credentials['projectId']); - $sourceRegion = $this->sourceProject->getAttribute('region', 'default'); - $destinationRegion = $this->project->getAttribute('region', 'default'); - $useAppwriteApiSource = $source === SourceAppwrite::getName() - && $sourceRegion !== $destinationRegion; - if (! $useAppwriteApiSource) { - $projectDB = call_user_func($this->getProjectDB, $this->sourceProject); + if ($this->sourceProject->isEmpty()) { + if ($source === SourceAppwrite::getName()) { + $useAppwriteApiSource = true; + } + } else { + $sourceRegion = $this->sourceProject->getAttribute('region', 'default'); + $destinationRegion = $this->project->getAttribute('region', 'default'); + $useAppwriteApiSource = $source === SourceAppwrite::getName() + && $sourceRegion !== $destinationRegion; + if (! $useAppwriteApiSource) { + $projectDB = call_user_func($this->getProjectDB, $this->sourceProject); + } } } $getDatabasesDB = fn (Document $database): Database => @@ -585,9 +591,10 @@ class Migrations extends Action protected function getDatabasesDBForProject(Document $database) { - if ($this->sourceProject) { + if (isset($this->sourceProject) && ! $this->sourceProject->isEmpty()) { return ($this->getDatabasesDB)($database, $this->sourceProject); } + return ($this->getDatabasesDB)($database); } From 8de7b419298dfc57091e86c4b19bdd76a1842df5 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 20 Apr 2026 11:06:07 +0530 Subject: [PATCH 08/20] updated --- src/Appwrite/Platform/Workers/Migrations.php | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index 7a16b02642..ba88ee0c8e 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -198,14 +198,11 @@ class Migrations extends Action $useAppwriteApiSource = false; if ($credentials['projectId']) { $this->sourceProject = $this->dbForPlatform->getDocument('projects', $credentials['projectId']); - if ($this->sourceProject->isEmpty()) { - if ($source === SourceAppwrite::getName()) { - $useAppwriteApiSource = true; - } - } else { + if (! $this->sourceProject->isEmpty()) { $sourceRegion = $this->sourceProject->getAttribute('region', 'default'); $destinationRegion = $this->project->getAttribute('region', 'default'); $useAppwriteApiSource = $source === SourceAppwrite::getName() + && $destination === DestinationAppwrite::getName() && $sourceRegion !== $destinationRegion; if (! $useAppwriteApiSource) { $projectDB = call_user_func($this->getProjectDB, $this->sourceProject); From a9fb17b0676be4cb2007dc8a5e2fdc03205c8eae Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 20 Apr 2026 11:39:50 +0530 Subject: [PATCH 09/20] added telemetry for message arriaval delay --- app/realtime.php | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/app/realtime.php b/app/realtime.php index 5631a7f860..4f1a81b33a 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -394,6 +394,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, Console::success('Worker ' . $workerId . ' started successfully'); $telemetry = getTelemetry($workerId); + $realtimeDelayBuckets = [100, 250, 500, 750, 1000, 1500, 2000, 3000, 5000, 7500, 10000, 15000, 30000]; $register->set('telemetry', fn () => $telemetry); $register->set('telemetry.connectionCounter', fn () => $telemetry->createUpDownCounter('realtime.server.open_connections')); $register->set('telemetry.connectionCreatedCounter', fn () => $telemetry->createCounter('realtime.server.connection.created')); @@ -401,7 +402,12 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $register->set('telemetry.deliveryDelayHistogram', fn () => $telemetry->createHistogram( name: 'realtime.server.delivery_delay', unit: 'ms', - advisory: ['ExplicitBucketBoundaries' => [100, 250, 500, 750, 1000, 1500, 2000, 3000, 5000, 7500, 10000, 15000, 30000]], + advisory: ['ExplicitBucketBoundaries' => $realtimeDelayBuckets], + )); + $register->set('telemetry.arrivalDelayHistogram', fn () => $telemetry->createHistogram( + name: 'realtime.server.arrival_delay', + unit: 'ms', + advisory: ['ExplicitBucketBoundaries' => $realtimeDelayBuckets], )); $attempts = 0; @@ -519,6 +525,21 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $pubsub->subscribe(['realtime'], function (mixed $redis, string $channel, string $payload) use ($server, $workerId, $stats, $register, $realtime) { $event = json_decode($payload, true); + $eventTimestamp = $event['data']['timestamp'] ?? null; + if (\is_string($eventTimestamp)) { + try { + $eventDate = new \DateTimeImmutable($eventTimestamp); + $now = new \DateTimeImmutable('now', new \DateTimeZone('UTC')); + $eventTimestampMs = (float) $eventDate->format('U.u') * 1000; + $nowTimestampMs = (float) $now->format('U.u') * 1000; + $arrivalDelayMs = (int) \max(0, $nowTimestampMs - $eventTimestampMs); + + $register->get('telemetry.arrivalDelayHistogram')->record($arrivalDelayMs); + } catch (\Throwable) { + // Ignore invalid timestamp payloads. + } + } + if ($event['permissionsChanged'] && isset($event['userId'])) { $projectId = $event['project']; $userId = $event['userId']; From 2bff4192ee159365b6ede9fe5cc1f0b65542e0ac Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 20 Apr 2026 11:45:05 +0530 Subject: [PATCH 10/20] added telemetry for the connected clients --- app/realtime.php | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/app/realtime.php b/app/realtime.php index 4f1a81b33a..96fb46b0b1 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -396,6 +396,8 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $telemetry = getTelemetry($workerId); $realtimeDelayBuckets = [100, 250, 500, 750, 1000, 1500, 2000, 3000, 5000, 7500, 10000, 15000, 30000]; $register->set('telemetry', fn () => $telemetry); + $register->set('telemetry.workerCounter', fn () => $telemetry->createUpDownCounter('realtime.server.active_workers')); + $register->set('telemetry.workerClientCounter', fn () => $telemetry->createUpDownCounter('realtime.server.worker_clients')); $register->set('telemetry.connectionCounter', fn () => $telemetry->createUpDownCounter('realtime.server.open_connections')); $register->set('telemetry.connectionCreatedCounter', fn () => $telemetry->createCounter('realtime.server.connection.created')); $register->set('telemetry.messageSentCounter', fn () => $telemetry->createCounter('realtime.server.message.sent')); @@ -409,6 +411,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, unit: 'ms', advisory: ['ExplicitBucketBoundaries' => $realtimeDelayBuckets], )); + $register->get('telemetry.workerCounter')->add(1); $attempts = 0; $start = time(); @@ -661,6 +664,16 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, Console::error('Failed to restart pub/sub...'); }); +$server->onWorkerStop(function (int $workerId) use ($register) { + Console::warning('Worker ' . $workerId . ' stopping'); + + try { + $register->get('telemetry.workerCounter')->add(-1); + } catch (\Throwable $th) { + Console::error('Realtime onWorkerStop telemetry error: ' . $th->getMessage()); + } +}); + $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $register, $stats, &$realtime, $registerConnectionResources) { global $container; $request = new Request($request); @@ -749,6 +762,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $updateStats = static function (string $projectId, ?string $teamId, string $payloadJson) use ($register, $stats): void { $register->get('telemetry.connectionCounter')->add(1); + $register->get('telemetry.workerClientCounter')->add(1); $register->get('telemetry.connectionCreatedCounter')->add(1); $stats->set($projectId, [ @@ -1161,6 +1175,7 @@ $server->onClose(function (int $connection) use ($realtime, $stats, $register) { if (array_key_exists($connection, $realtime->connections)) { $stats->decr($realtime->connections[$connection]['projectId'], 'connectionsTotal'); $register->get('telemetry.connectionCounter')->add(-1); + $register->get('telemetry.workerClientCounter')->add(-1); $projectId = $realtime->connections[$connection]['projectId']; From 12f76d74b16876c05db57bbfb7fec9663f98db57 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 20 Apr 2026 11:52:43 +0530 Subject: [PATCH 11/20] added subscriptions telemetry and worker labelling to connections and subscriptions --- app/realtime.php | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 96fb46b0b1..591294dfa9 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -395,9 +395,12 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $telemetry = getTelemetry($workerId); $realtimeDelayBuckets = [100, 250, 500, 750, 1000, 1500, 2000, 3000, 5000, 7500, 10000, 15000, 30000]; + $workerTelemetryAttributes = ['workerId' => (string) $workerId]; $register->set('telemetry', fn () => $telemetry); + $register->set('telemetry.workerAttributes', fn () => $workerTelemetryAttributes); $register->set('telemetry.workerCounter', fn () => $telemetry->createUpDownCounter('realtime.server.active_workers')); $register->set('telemetry.workerClientCounter', fn () => $telemetry->createUpDownCounter('realtime.server.worker_clients')); + $register->set('telemetry.workerSubscriptionCounter', fn () => $telemetry->createUpDownCounter('realtime.server.worker_subscriptions')); $register->set('telemetry.connectionCounter', fn () => $telemetry->createUpDownCounter('realtime.server.open_connections')); $register->set('telemetry.connectionCreatedCounter', fn () => $telemetry->createCounter('realtime.server.connection.created')); $register->set('telemetry.messageSentCounter', fn () => $telemetry->createCounter('realtime.server.message.sent')); @@ -549,6 +552,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, if ($realtime->hasSubscriber($projectId, 'user:' . $userId)) { $connection = array_key_first(reset($realtime->subscriptions[$projectId]['user:' . $userId])); + $subscriptionsBefore = \count($realtime->getSubscriptionMetadata($connection)); $consoleDatabase = getConsoleDB(); $project = $consoleDatabase->getAuthorization()->skip(fn () => $consoleDatabase->getDocument('projects', $projectId)); $database = getProjectDB($project); @@ -579,6 +583,12 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, if ($authorization !== null) { $realtime->connections[$connection]['authorization'] = $authorization; } + + $subscriptionsAfter = \count($realtime->getSubscriptionMetadata($connection)); + $subscriptionDelta = $subscriptionsAfter - $subscriptionsBefore; + if ($subscriptionDelta !== 0) { + $register->get('telemetry.workerSubscriptionCounter')->add($subscriptionDelta, $register->get('telemetry.workerAttributes')); + } } } @@ -762,7 +772,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $updateStats = static function (string $projectId, ?string $teamId, string $payloadJson) use ($register, $stats): void { $register->get('telemetry.connectionCounter')->add(1); - $register->get('telemetry.workerClientCounter')->add(1); + $register->get('telemetry.workerClientCounter')->add(1, $register->get('telemetry.workerAttributes')); $register->get('telemetry.connectionCreatedCounter')->add(1); $stats->set($projectId, [ @@ -794,6 +804,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, ]); $realtime->subscribe($project->getId(), $connection, '', $roles, [], [], $user->getId()); + $register->get('telemetry.workerSubscriptionCounter')->add(1, $register->get('telemetry.workerAttributes')); $realtime->connections[$connection]['authorization'] = $authorization; $server->send([$connection], $connectedPayloadJson); $updateStats($project->getId(), $project->getAttribute('teamId'), $connectedPayloadJson); @@ -827,6 +838,9 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $mapping[$index] = $subscriptionId; } + if (!empty($subscriptions)) { + $register->get('telemetry.workerSubscriptionCounter')->add(\count($subscriptions), $register->get('telemetry.workerAttributes')); + } $realtime->connections[$connection]['authorization'] = $authorization; @@ -881,7 +895,7 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, } }); -$server->onMessage(function (int $connection, string $message) use ($server, $realtime, $containerId) { +$server->onMessage(function (int $connection, string $message) use ($server, $realtime, $containerId, $register) { $project = null; $authorization = null; try { @@ -995,6 +1009,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $authorization = $realtime->connections[$connection]['authorization'] ?? null; $projectId = $realtime->connections[$connection]['projectId'] ?? null; + $subscriptionsBefore = \count($realtime->getSubscriptionMetadata($connection)); $meta = $realtime->getSubscriptionMetadata($connection); $realtime->unsubscribe($connection); @@ -1019,6 +1034,12 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $realtime->connections[$connection]['authorization'] = $authorization; } + $subscriptionsAfter = \count($realtime->getSubscriptionMetadata($connection)); + $subscriptionDelta = $subscriptionsAfter - $subscriptionsBefore; + if ($subscriptionDelta !== 0) { + $register->get('telemetry.workerSubscriptionCounter')->add($subscriptionDelta, $register->get('telemetry.workerAttributes')); + } + $user = $response->output($user, Response::MODEL_ACCOUNT); $authResponsePayloadJson = json_encode([ @@ -1063,6 +1084,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re // bulk validation + parsing before subscribing $parsedPayloads = []; + $subscriptionsBefore = \count($realtime->getSubscriptionMetadata($connection)); foreach ($message['data'] as $payload) { if (!\is_array($payload)) { throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Each subscribe payload must be an object.'); @@ -1107,6 +1129,11 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re // subscribe() overwrites the connection entry; restore auth so later onMessage uses the same context. $realtime->connections[$connection]['authorization'] = $authorization; + $subscriptionsAfter = \count($realtime->getSubscriptionMetadata($connection)); + $subscriptionDelta = $subscriptionsAfter - $subscriptionsBefore; + if ($subscriptionDelta !== 0) { + $register->get('telemetry.workerSubscriptionCounter')->add($subscriptionDelta, $register->get('telemetry.workerAttributes')); + } $responsePayload = json_encode([ 'type' => 'response', @@ -1175,7 +1202,11 @@ $server->onClose(function (int $connection) use ($realtime, $stats, $register) { if (array_key_exists($connection, $realtime->connections)) { $stats->decr($realtime->connections[$connection]['projectId'], 'connectionsTotal'); $register->get('telemetry.connectionCounter')->add(-1); - $register->get('telemetry.workerClientCounter')->add(-1); + $register->get('telemetry.workerClientCounter')->add(-1, $register->get('telemetry.workerAttributes')); + $subscriptionsBeforeClose = \count($realtime->getSubscriptionMetadata($connection)); + if ($subscriptionsBeforeClose > 0) { + $register->get('telemetry.workerSubscriptionCounter')->add(-$subscriptionsBeforeClose, $register->get('telemetry.workerAttributes')); + } $projectId = $realtime->connections[$connection]['projectId']; From 4b94d14f1eb9ad1b8b16dc8109830ccf33fcde45 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 20 Apr 2026 12:13:53 +0530 Subject: [PATCH 12/20] updated time metric --- app/realtime.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 591294dfa9..4621c6543e 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -534,7 +534,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $eventTimestamp = $event['data']['timestamp'] ?? null; if (\is_string($eventTimestamp)) { try { - $eventDate = new \DateTimeImmutable($eventTimestamp); + $eventDate = new \DateTimeImmutable($eventTimestamp, new \DateTimeZone('UTC')); $now = new \DateTimeImmutable('now', new \DateTimeZone('UTC')); $eventTimestampMs = (float) $eventDate->format('U.u') * 1000; $nowTimestampMs = (float) $now->format('U.u') * 1000; @@ -634,7 +634,7 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $updatedAt = $event['data']['payload']['$updatedAt'] ?? null; if (\is_string($updatedAt)) { try { - $updatedAtDate = new \DateTimeImmutable($updatedAt); + $updatedAtDate = new \DateTimeImmutable($updatedAt, new \DateTimeZone('UTC')); $now = new \DateTimeImmutable('now', new \DateTimeZone('UTC')); $updatedAtTimestampMs = (float) $updatedAtDate->format('U.u') * 1000; $nowTimestampMs = (float) $now->format('U.u') * 1000; From 62f7f25cb58973478e12a06c2104f7a2f561c15f Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 20 Apr 2026 12:18:38 +0530 Subject: [PATCH 13/20] updated --- src/Appwrite/Platform/Workers/Migrations.php | 26 ++++++++++++-------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index ba88ee0c8e..339084727d 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -196,17 +196,23 @@ class Migrations extends Action /** @var Database|null $projectDB */ $projectDB = null; $useAppwriteApiSource = false; - if ($credentials['projectId']) { + if ($source === SourceAppwrite::getName() && empty($credentials['projectId'])) { + throw new \Exception('Source projectId is required for Appwrite migrations'); + } + + if (! empty($credentials['projectId'])) { $this->sourceProject = $this->dbForPlatform->getDocument('projects', $credentials['projectId']); - if (! $this->sourceProject->isEmpty()) { - $sourceRegion = $this->sourceProject->getAttribute('region', 'default'); - $destinationRegion = $this->project->getAttribute('region', 'default'); - $useAppwriteApiSource = $source === SourceAppwrite::getName() - && $destination === DestinationAppwrite::getName() - && $sourceRegion !== $destinationRegion; - if (! $useAppwriteApiSource) { - $projectDB = call_user_func($this->getProjectDB, $this->sourceProject); - } + if ($this->sourceProject->isEmpty()) { + throw new \Exception('Source project not found for provided projectId'); + } + + $sourceRegion = $this->sourceProject->getAttribute('region', 'default'); + $destinationRegion = $this->project->getAttribute('region', 'default'); + $useAppwriteApiSource = $source === SourceAppwrite::getName() + && $destination === DestinationAppwrite::getName() + && $sourceRegion !== $destinationRegion; + if (! $useAppwriteApiSource) { + $projectDB = call_user_func($this->getProjectDB, $this->sourceProject); } } $getDatabasesDB = fn (Document $database): Database => From e9ea39a822d0cc7cba91b130cadfd03342d47bd3 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 20 Apr 2026 17:37:45 +0530 Subject: [PATCH 14/20] Enhance Realtime adapter: support union of channels/roles on subscription and add unsubscribeSubscription method --- src/Appwrite/Messaging/Adapter/Realtime.php | 101 ++++++++++++++++++-- 1 file changed, 95 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Messaging/Adapter/Realtime.php b/src/Appwrite/Messaging/Adapter/Realtime.php index f1d806bcc5..3dd0cda328 100644 --- a/src/Appwrite/Messaging/Adapter/Realtime.php +++ b/src/Appwrite/Messaging/Adapter/Realtime.php @@ -114,14 +114,24 @@ class Realtime extends MessagingAdapter } } - // Keep userId from onOpen/authentication when provided. - // Fallback to existing stored value for subsequent subscribe upserts. - $this->connections[$identifier] = [ + // Union channels/roles across all subscriptions on the connection; overwriting would + // leave getSubscriptionMetadata and full unsubscribe operating on stale state. + $existing = $this->connections[$identifier] ?? []; + $existingChannels = $existing['channels'] ?? []; + $existingRoles = $existing['roles'] ?? []; + + $entry = [ 'projectId' => $projectId, - 'roles' => $roles, - 'userId' => $userId ?? ($this->connections[$identifier]['userId'] ?? ''), - 'channels' => $channels + 'roles' => \array_values(\array_unique(\array_merge($existingRoles, $roles))), + 'userId' => $userId ?? ($existing['userId'] ?? ''), + 'channels' => \array_values(\array_unique(\array_merge($existingChannels, $channels))), ]; + + if (\array_key_exists('authorization', $existing)) { + $entry['authorization'] = $existing['authorization']; + } + + $this->connections[$identifier] = $entry; } /** @@ -206,6 +216,85 @@ class Realtime extends MessagingAdapter } } + /** + * Removes a single subscription from a connection, keeping the connection alive so + * the client can resubscribe. Idempotent — returns true only when something was removed. + * + * @param mixed $connection + * @param string $subscriptionId + * @return bool + */ + public function unsubscribeSubscription(mixed $connection, string $subscriptionId): bool + { + $projectId = $this->connections[$connection]['projectId'] ?? ''; + if ($projectId === '' || !isset($this->subscriptions[$projectId])) { + return false; + } + + $removed = false; + + foreach ($this->subscriptions[$projectId] as $role => $byChannel) { + foreach ($byChannel as $channel => $byConnection) { + if (!isset($byConnection[$connection][$subscriptionId])) { + continue; + } + + unset($this->subscriptions[$projectId][$role][$channel][$connection][$subscriptionId]); + $removed = true; + + if (empty($this->subscriptions[$projectId][$role][$channel][$connection])) { + unset($this->subscriptions[$projectId][$role][$channel][$connection]); + } + if (empty($this->subscriptions[$projectId][$role][$channel])) { + unset($this->subscriptions[$projectId][$role][$channel]); + } + } + if (empty($this->subscriptions[$projectId][$role])) { + unset($this->subscriptions[$projectId][$role]); + } + } + + if (empty($this->subscriptions[$projectId])) { + unset($this->subscriptions[$projectId]); + } + + if ($removed) { + $this->recomputeConnectionState($connection); + } + + return $removed; + } + + /** + * Recomputes the cached channels/roles on the connection entry from the subscriptions tree. + * Called after per-subscription removal so stale entries do not linger for later reads. + * + * @param mixed $connection + * @return void + */ + private function recomputeConnectionState(mixed $connection): void + { + if (!isset($this->connections[$connection])) { + return; + } + + $projectId = $this->connections[$connection]['projectId'] ?? ''; + $channels = []; + $roles = []; + + foreach ($this->subscriptions[$projectId] ?? [] as $role => $byChannel) { + foreach ($byChannel as $channel => $byConnection) { + if (isset($byConnection[$connection])) { + $roles[$role] = true; + $channels[$channel] = true; + } + } + } + + $this->connections[$connection]['roles'] = \array_keys($roles); + $this->connections[$connection]['channels'] = \array_keys($channels); + } + /** * Checks if Channel has a subscriber. * @param string $projectId From 9f6517764978cafeb991d0d23caacb43aa3e64ec Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 20 Apr 2026 17:37:54 +0530 Subject: [PATCH 15/20] Add unsubscribe functionality to Realtime adapter --- app/realtime.php | 50 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 5631a7f860..32c71fa9a3 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -1070,9 +1070,6 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $realtime->subscribe($projectId, $connection, $subscriptionId, $roles, $channels, $queries); } - // subscribe() overwrites the connection entry; restore auth so later onMessage uses the same context. - $realtime->connections[$connection]['authorization'] = $authorization; - $responsePayload = json_encode([ 'type' => 'response', 'data' => [ @@ -1102,6 +1099,53 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re break; + case 'unsubscribe': + if (!\is_array($message['data']) || !\array_is_list($message['data'])) { + throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Payload is not valid.'); + } + + $unsubscribeResults = []; + foreach ($message['data'] as $payload) { + if ( + !\is_array($payload) + || !\array_key_exists('subscriptionId', $payload) + || !\is_string($payload['subscriptionId']) + || $payload['subscriptionId'] === '' + ) { + throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Each unsubscribe payload must include a non-empty subscriptionId.'); + } + + $subscriptionId = $payload['subscriptionId']; + $wasRemoved = $realtime->unsubscribeSubscription($connection, $subscriptionId); + $unsubscribeResults[] = [ + 'subscriptionId' => $subscriptionId, + 'removed' => $wasRemoved, + ]; + } + + $unsubscribeResponsePayload = json_encode([ + 'type' => 'response', + 'data' => [ + 'to' => 'unsubscribe', + 'success' => true, + 'subscriptions' => $unsubscribeResults, + ], + ]); + + $server->send([$connection], $unsubscribeResponsePayload); + + if ($project !== null && !$project->isEmpty()) { + $unsubscribeOutboundBytes = \strlen($unsubscribeResponsePayload); + + if ($unsubscribeOutboundBytes > 0) { + triggerStats([ + METRIC_REALTIME_OUTBOUND => $unsubscribeOutboundBytes, + ], $project->getId()); + } + } + + break; + default: throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Message type is not valid.'); } From 78eeac6d14753bafd8fb5bb92632611f1e31fef1 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 20 Apr 2026 17:38:01 +0530 Subject: [PATCH 16/20] Add unsubscribe functionality and enhance subscription handling in Realtime tests --- ...altimeCustomClientQueryTestWithMessage.php | 270 ++++++++++++++++++ tests/unit/Messaging/MessagingTest.php | 187 ++++++++++++ 2 files changed, 457 insertions(+) diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php b/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php index edce428e0f..5dce7056e8 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php @@ -164,6 +164,20 @@ class RealtimeCustomClientQueryTestWithMessage extends Scope return $response; } + /** + * @param array> $payloadEntries + * @return array + */ + private function sendUnsubscribeMessage(WebSocketClient $client, array $payloadEntries): array + { + $client->send(\json_encode([ + 'type' => 'unsubscribe', + 'data' => $payloadEntries, + ])); + + return \json_decode($client->receive(), true); + } + /** * subscriptionId: update with id from connected, create by omitting id, explicit new id, * duplicate id in one bulk (last wins), mixed bulk, idempotent repeat, empty queries → select-all. @@ -293,6 +307,262 @@ class RealtimeCustomClientQueryTestWithMessage extends Scope $client->close(); } + /** + * Update a subscription's queries/channels by reusing its subscriptionId. + * Verifies the update takes effect on live event filtering (not just the response echo), + * sibling subscriptions are untouched, unknown ids upsert as new, empty queries fall + * back to select-all, and a removed id can be recreated by subscribing again. + */ + public function testUpdateSubscriptionAndEdgeCases(): void + { + $user = $this->getUser(); + $userId = $user['$id'] ?? ''; + $session = $user['session'] ?? ''; + $projectId = $this->getProject()['$id']; + $headers = [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ]; + + $queryString = \http_build_query(['project' => $projectId]); + $client = new WebSocketClient( + 'ws://appwrite.test/v1/realtime?' . $queryString, + [ + 'headers' => $headers, + 'timeout' => 10, + ] + ); + $connected = \json_decode($client->receive(), true); + $this->assertEquals('connected', $connected['type'] ?? null); + + $triggerAccountEvent = function () use ($projectId, $session): void { + $this->client->call(Client::METHOD_PATCH, '/account/name', \array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ]), ['name' => 'Update Sub Test ' . \uniqid()]); + }; + + // subA matches current user, subB never matches + $created = $this->sendSubscribeMessage($client, [ + [ + 'channels' => ['account'], + 'queries' => [Query::equal('$id', [$userId])->toString()], + ], + [ + 'channels' => ['account'], + 'queries' => [Query::equal('$id', ['no-match-initial'])->toString()], + ], + ]); + $subA = $created['data']['subscriptions'][0]['subscriptionId']; + $subB = $created['data']['subscriptions'][1]['subscriptionId']; + $this->assertNotSame($subA, $subB); + + $triggerAccountEvent(); + $event = \json_decode($client->receive(), true); + $this->assertEquals('event', $event['type']); + $this->assertSame([$subA], $event['data']['subscriptions']); + + // Swap: A -> non-matching, B -> matching. Same ids returned, server-side filter swaps. + $swap = $this->sendSubscribeMessage($client, [ + [ + 'subscriptionId' => $subA, + 'channels' => ['account'], + 'queries' => [Query::equal('$id', ['no-match-swapped'])->toString()], + ], + [ + 'subscriptionId' => $subB, + 'channels' => ['account'], + 'queries' => [Query::equal('$id', [$userId])->toString()], + ], + ]); + $this->assertSame($subA, $swap['data']['subscriptions'][0]['subscriptionId']); + $this->assertSame($subB, $swap['data']['subscriptions'][1]['subscriptionId']); + + $triggerAccountEvent(); + $event = \json_decode($client->receive(), true); + $this->assertEquals('event', $event['type']); + $this->assertSame([$subB], $event['data']['subscriptions']); + + // Sibling isolation: updating only subA must leave subB's matching filter intact. + $isolation = $this->sendSubscribeMessage($client, [[ + 'subscriptionId' => $subA, + 'channels' => ['account'], + 'queries' => [Query::equal('$id', [$userId])->toString()], + ]]); + $this->assertSame($subA, $isolation['data']['subscriptions'][0]['subscriptionId']); + + $triggerAccountEvent(); + $event = \json_decode($client->receive(), true); + $this->assertEquals('event', $event['type']); + $this->assertEqualsCanonicalizing([$subA, $subB], $event['data']['subscriptions']); + + // Empty queries on update -> select-all; subA still matches every event on the channel. + $empty = $this->sendSubscribeMessage($client, [[ + 'subscriptionId' => $subA, + 'channels' => ['account'], + 'queries' => [], + ]]); + $this->assertSame($subA, $empty['data']['subscriptions'][0]['subscriptionId']); + + $triggerAccountEvent(); + $event = \json_decode($client->receive(), true); + $this->assertEquals('event', $event['type']); + $this->assertEqualsCanonicalizing([$subA, $subB], $event['data']['subscriptions']); + + // Unknown subscriptionId upserts as a new subscription. + $ghostId = ID::unique(); + $ghost = $this->sendSubscribeMessage($client, [[ + 'subscriptionId' => $ghostId, + 'channels' => ['account'], + 'queries' => [Query::equal('$id', [$userId])->toString()], + ]]); + $this->assertSame($ghostId, $ghost['data']['subscriptions'][0]['subscriptionId']); + $this->assertNotSame($subA, $ghostId); + $this->assertNotSame($subB, $ghostId); + + $triggerAccountEvent(); + $event = \json_decode($client->receive(), true); + $this->assertEquals('event', $event['type']); + $this->assertEqualsCanonicalizing([$subA, $subB, $ghostId], $event['data']['subscriptions']); + + // Update after unsubscribe: subscribing with the removed id recreates it. + $unsub = $this->sendUnsubscribeMessage($client, [['subscriptionId' => $subA]]); + $this->assertTrue($unsub['data']['subscriptions'][0]['removed']); + + $triggerAccountEvent(); + $event = \json_decode($client->receive(), true); + $this->assertEquals('event', $event['type']); + $this->assertEqualsCanonicalizing([$subB, $ghostId], $event['data']['subscriptions']); + + $recreated = $this->sendSubscribeMessage($client, [[ + 'subscriptionId' => $subA, + 'channels' => ['account'], + 'queries' => [Query::equal('$id', [$userId])->toString()], + ]]); + $this->assertSame($subA, $recreated['data']['subscriptions'][0]['subscriptionId']); + + $triggerAccountEvent(); + $event = \json_decode($client->receive(), true); + $this->assertEquals('event', $event['type']); + $this->assertEqualsCanonicalizing([$subA, $subB, $ghostId], $event['data']['subscriptions']); + + $client->close(); + } + + public function testUnsubscribeRemovesOnlyMatchingSubscription(): void + { + $user = $this->getUser(); + $userId = $user['$id'] ?? ''; + $session = $user['session'] ?? ''; + $projectId = $this->getProject()['$id']; + $headers = [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ]; + + $queryString = \http_build_query(['project' => $projectId]); + $client = new WebSocketClient( + 'ws://appwrite.test/v1/realtime?' . $queryString, + [ + 'headers' => $headers, + 'timeout' => 10, + ] + ); + + $connected = \json_decode($client->receive(), true); + $this->assertEquals('connected', $connected['type'] ?? null); + + // Two subscriptions on the `account` channel, both matching the current user + $r1 = $this->sendSubscribeMessage($client, [[ + 'channels' => ['account'], + 'queries' => [Query::equal('$id', [$userId])->toString()], + ]]); + $subA = $r1['data']['subscriptions'][0]['subscriptionId']; + + $r2 = $this->sendSubscribeMessage($client, [[ + 'channels' => ['account'], + 'queries' => [Query::select(['*'])->toString()], + ]]); + $subB = $r2['data']['subscriptions'][0]['subscriptionId']; + + $this->assertNotSame($subA, $subB); + + // Trigger an event -- both subscriptions should match + $name = 'Unsubscribe Test ' . \uniqid(); + $this->client->call(Client::METHOD_PATCH, '/account/name', \array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ]), ['name' => $name]); + + $event = \json_decode($client->receive(), true); + $this->assertEquals('event', $event['type']); + $this->assertEqualsCanonicalizing([$subA, $subB], $event['data']['subscriptions']); + + // Unsubscribe subA only + $unsubA = $this->sendUnsubscribeMessage($client, [['subscriptionId' => $subA]]); + $this->assertEquals('response', $unsubA['type']); + $this->assertEquals('unsubscribe', $unsubA['data']['to']); + $this->assertTrue($unsubA['data']['success']); + $this->assertCount(1, $unsubA['data']['subscriptions']); + $this->assertSame($subA, $unsubA['data']['subscriptions'][0]['subscriptionId']); + $this->assertTrue($unsubA['data']['subscriptions'][0]['removed']); + + // Trigger another event -- only subB should match now + $name = 'Unsubscribe Test ' . \uniqid(); + $this->client->call(Client::METHOD_PATCH, '/account/name', \array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ]), ['name' => $name]); + + $event = \json_decode($client->receive(), true); + $this->assertEquals('event', $event['type']); + $this->assertSame([$subB], $event['data']['subscriptions']); + + // Idempotent: unsubscribing subA again reports removed=false + $unsubAgain = $this->sendUnsubscribeMessage($client, [['subscriptionId' => $subA]]); + $this->assertTrue($unsubAgain['data']['success']); + $this->assertFalse($unsubAgain['data']['subscriptions'][0]['removed']); + + // Connection is still alive -- ping still works + $client->send(\json_encode(['type' => 'ping'])); + $pong = \json_decode($client->receive(), true); + $this->assertEquals('pong', $pong['type']); + + // Invalid payloads are rejected + $errNonString = $this->sendUnsubscribeMessage($client, [['subscriptionId' => 123]]); + $this->assertEquals('error', $errNonString['type']); + $this->assertStringContainsString('subscriptionId', $errNonString['data']['message']); + + $errEmpty = $this->sendUnsubscribeMessage($client, [['subscriptionId' => '']]); + $this->assertEquals('error', $errEmpty['type']); + + $errMissing = $this->sendUnsubscribeMessage($client, [['channels' => ['foo']]]); + $this->assertEquals('error', $errMissing['type']); + + $errNonList = $this->sendUnsubscribeMessage($client, ['subscriptionId' => $subB]); + $this->assertEquals('error', $errNonList['type']); + + // Bulk unsubscribe: remaining subB plus a never-existed id -- response mirrors input order + $bulk = $this->sendUnsubscribeMessage($client, [ + ['subscriptionId' => $subB], + ['subscriptionId' => 'does-not-exist'], + ]); + $this->assertTrue($bulk['data']['success']); + $this->assertCount(2, $bulk['data']['subscriptions']); + $this->assertSame($subB, $bulk['data']['subscriptions'][0]['subscriptionId']); + $this->assertTrue($bulk['data']['subscriptions'][0]['removed']); + $this->assertSame('does-not-exist', $bulk['data']['subscriptions'][1]['subscriptionId']); + $this->assertFalse($bulk['data']['subscriptions'][1]['removed']); + + $client->close(); + } + public function testInvalidQueryShouldNotSubscribe(): void { $user = $this->getUser(); diff --git a/tests/unit/Messaging/MessagingTest.php b/tests/unit/Messaging/MessagingTest.php index 4b2474c760..f48be46202 100644 --- a/tests/unit/Messaging/MessagingTest.php +++ b/tests/unit/Messaging/MessagingTest.php @@ -147,6 +147,193 @@ class MessagingTest extends TestCase $this->assertEmpty($realtime->subscriptions); } + public function testSubscribeUnionsChannelsAndRoles(): void + { + $realtime = new Realtime(); + + $realtime->subscribe( + '1', + 1, + 'sub-a', + [Role::user(ID::custom('123'))->toString()], + ['documents'], + ); + + $realtime->subscribe( + '1', + 1, + 'sub-b', + [Role::users()->toString()], + ['files'], + ); + + $connection = $realtime->connections[1]; + + $this->assertContains('documents', $connection['channels']); + $this->assertContains('files', $connection['channels']); + $this->assertContains(Role::user(ID::custom('123'))->toString(), $connection['roles']); + $this->assertContains(Role::users()->toString(), $connection['roles']); + $this->assertCount(2, $connection['channels']); + $this->assertCount(2, $connection['roles']); + } + + public function testUnsubscribeSubscriptionRemovesOnlyOneSubscription(): void + { + $realtime = new Realtime(); + + $realtime->subscribe( + '1', + 1, + 'sub-a', + [Role::user(ID::custom('123'))->toString()], + ['documents'], + ); + + $realtime->subscribe( + '1', + 1, + 'sub-b', + [Role::users()->toString()], + ['files'], + ); + + $removed = $realtime->unsubscribeSubscription(1, 'sub-a'); + + $this->assertTrue($removed); + $this->assertArrayHasKey(1, $realtime->connections); + + // sub-a is fully cleaned from the tree + $this->assertArrayNotHasKey( + Role::user(ID::custom('123'))->toString(), + $realtime->subscriptions['1'] + ); + + // sub-b still delivers + $event = [ + 'project' => '1', + 'roles' => [Role::users()->toString()], + 'data' => [ + 'channels' => ['files'], + ], + ]; + $receivers = array_keys($realtime->getSubscribers($event)); + $this->assertEquals([1], $receivers); + + // Channels recomputed: sub-a's channel is gone + $this->assertSame(['files'], $realtime->connections[1]['channels']); + + // Roles are connection-level auth context — union of both subscribe calls preserved + $this->assertContains(Role::user(ID::custom('123'))->toString(), $realtime->connections[1]['roles']); + $this->assertContains(Role::users()->toString(), $realtime->connections[1]['roles']); + } + + public function testUnsubscribeSubscriptionIsIdempotent(): void + { + $realtime = new Realtime(); + + $realtime->subscribe( + '1', + 1, + 'sub-a', + [Role::users()->toString()], + ['documents'], + ); + + $this->assertFalse($realtime->unsubscribeSubscription(1, 'does-not-exist')); + $this->assertFalse($realtime->unsubscribeSubscription(99, 'sub-a')); + + // Original sub is untouched + $event = [ + 'project' => '1', + 'roles' => [Role::users()->toString()], + 'data' => [ + 'channels' => ['documents'], + ], + ]; + $this->assertEquals([1], array_keys($realtime->getSubscribers($event))); + } + + public function testUnsubscribeSubscriptionKeepsConnectionWhenLastSubRemoved(): void + { + $realtime = new Realtime(); + + $realtime->subscribe( + '1', + 1, + 'sub-a', + [Role::users()->toString()], + ['documents'], + ); + + $this->assertTrue($realtime->unsubscribeSubscription(1, 'sub-a')); + + $this->assertArrayHasKey(1, $realtime->connections); + $this->assertSame([], $realtime->connections[1]['channels']); + // Roles preserved so a later resubscribe on the same connection still has auth context + $this->assertSame([Role::users()->toString()], $realtime->connections[1]['roles']); + $this->assertArrayNotHasKey('1', $realtime->subscriptions); + } + + public function testResubscribeAfterUnsubscribingLastSubDelivers(): void + { + $realtime = new Realtime(); + + $realtime->subscribe( + '1', + 1, + 'sub-a', + [Role::users()->toString()], + ['documents'], + ); + + $this->assertTrue($realtime->unsubscribeSubscription(1, 'sub-a')); + + // Simulate the message-based subscribe path reading stored roles + $storedRoles = $realtime->connections[1]['roles']; + $this->assertNotEmpty($storedRoles, 'connection roles must survive per-subscription removal'); + + $realtime->subscribe('1', 1, 'sub-b', $storedRoles, ['files']); + + $event = [ + 'project' => '1', + 'roles' => [Role::users()->toString()], + 'data' => [ + 'channels' => ['files'], + ], + ]; + $this->assertEquals([1], array_keys($realtime->getSubscribers($event))); + } + + public function testSubscribeAfterOnOpenEmptySentinelPreservesUnion(): void + { + $realtime = new Realtime(); + + // Mirrors the onOpen empty-channels path: subscribe with '' id, empty channels + $realtime->subscribe( + '1', + 1, + '', + [Role::users()->toString()], + [], + [], + 'user-123', + ); + + // Now a real subscription comes in via the subscribe message type + $realtime->subscribe( + '1', + 1, + 'sub-a', + [Role::user(ID::custom('user-123'))->toString()], + ['documents'], + ); + + $this->assertSame('user-123', $realtime->connections[1]['userId']); + $this->assertContains('documents', $realtime->connections[1]['channels']); + $this->assertContains(Role::users()->toString(), $realtime->connections[1]['roles']); + $this->assertContains(Role::user(ID::custom('user-123'))->toString(), $realtime->connections[1]['roles']); + } + public function testConvertChannelsGuest(): void { $user = new Document([ From b2233193d50aba158c24e8bd20bee2bf24d573e8 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 20 Apr 2026 18:19:12 +0530 Subject: [PATCH 17/20] updated --- src/Appwrite/Messaging/Adapter/Realtime.php | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Appwrite/Messaging/Adapter/Realtime.php b/src/Appwrite/Messaging/Adapter/Realtime.php index 3dd0cda328..eeb1387674 100644 --- a/src/Appwrite/Messaging/Adapter/Realtime.php +++ b/src/Appwrite/Messaging/Adapter/Realtime.php @@ -266,8 +266,13 @@ class Realtime extends MessagingAdapter } /** - * Recomputes the cached channels/roles on the connection entry from the subscriptions tree. - * Called after per-subscription removal so stale entries do not linger for later reads. + * Recomputes the cached channels on the connection entry from the subscriptions tree. + * Called after per-subscription removal so stale channel entries do not linger for later reads. + * + * Roles are deliberately NOT recomputed here. They represent the connection's authorization + * context (set at onOpen, replaced on `authentication` / permission-change) and must survive + * per-subscription removal — otherwise a client that unsubscribes every subscription and then + * resubscribes would subscribe with an empty roles array and silently receive nothing. * * @param mixed $connection * @return void @@ -280,18 +285,15 @@ class Realtime extends MessagingAdapter $projectId = $this->connections[$connection]['projectId'] ?? ''; $channels = []; - $roles = []; - foreach ($this->subscriptions[$projectId] ?? [] as $role => $byChannel) { + foreach ($this->subscriptions[$projectId] ?? [] as $byChannel) { foreach ($byChannel as $channel => $byConnection) { if (isset($byConnection[$connection])) { - $roles[$role] = true; $channels[$channel] = true; } } } - $this->connections[$connection]['roles'] = \array_keys($roles); $this->connections[$connection]['channels'] = \array_keys($channels); } From b8385fe92769c3ba382c110d4ba511700bc37fdb Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 20 Apr 2026 18:27:48 +0530 Subject: [PATCH 18/20] updated --- app/realtime.php | 9 +++++++-- ...altimeCustomClientQueryTestWithMessage.php | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 32c71fa9a3..bd493dc64f 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -1104,7 +1104,9 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Payload is not valid.'); } - $unsubscribeResults = []; + // Validate every payload before executing any removal so an invalid entry + // later in the batch does not leave earlier entries half-applied on the server. + $validatedIds = []; foreach ($message['data'] as $payload) { if ( !\is_array($payload) @@ -1114,8 +1116,11 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re ) { throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Each unsubscribe payload must include a non-empty subscriptionId.'); } + $validatedIds[] = $payload['subscriptionId']; + } - $subscriptionId = $payload['subscriptionId']; + $unsubscribeResults = []; + foreach ($validatedIds as $subscriptionId) { $wasRemoved = $realtime->unsubscribeSubscription($connection, $subscriptionId); $unsubscribeResults[] = [ 'subscriptionId' => $subscriptionId, diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php b/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php index 5dce7056e8..6376875157 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientQueryTestWithMessage.php @@ -548,6 +548,26 @@ class RealtimeCustomClientQueryTestWithMessage extends Scope $errNonList = $this->sendUnsubscribeMessage($client, ['subscriptionId' => $subB]); $this->assertEquals('error', $errNonList['type']); + // A batch with a valid id followed by an invalid one must be rejected atomically: + // the valid id must remain subscribed, not be quietly removed before validation fails. + $partial = $this->sendUnsubscribeMessage($client, [ + ['subscriptionId' => $subB], + ['subscriptionId' => 999], + ]); + $this->assertEquals('error', $partial['type']); + + $name = 'Partial Rejection Test ' . \uniqid(); + $this->client->call(Client::METHOD_PATCH, '/account/name', \array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ]), ['name' => $name]); + + $event = \json_decode($client->receive(), true); + $this->assertEquals('event', $event['type']); + $this->assertSame([$subB], $event['data']['subscriptions']); + // Bulk unsubscribe: remaining subB plus a never-existed id -- response mirrors input order $bulk = $this->sendUnsubscribeMessage($client, [ ['subscriptionId' => $subB], From abd6f0add2988621712eb2d226e51a811c3cf0ac Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 21 Apr 2026 10:52:21 +0530 Subject: [PATCH 19/20] updated --- app/realtime.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/realtime.php b/app/realtime.php index b5d8859d7b..667b035b9b 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -1166,6 +1166,8 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Payload is not valid.'); } + $subscriptionsBefore = \count($realtime->getSubscriptionMetadata($connection)); + // Validate every payload before executing any removal so an invalid entry // later in the batch does not leave earlier entries half-applied on the server. $validatedIds = []; @@ -1189,6 +1191,11 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re 'removed' => $wasRemoved, ]; } + $subscriptionsAfter = \count($realtime->getSubscriptionMetadata($connection)); + $subscriptionDelta = $subscriptionsAfter - $subscriptionsBefore; + if ($subscriptionDelta !== 0) { + $register->get('telemetry.workerSubscriptionCounter')->add($subscriptionDelta, $register->get('telemetry.workerAttributes')); + } $unsubscribeResponsePayload = json_encode([ 'type' => 'response', From 5df65d541766ab5455810876811d4c34348d78f7 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 21 Apr 2026 10:58:34 +0530 Subject: [PATCH 20/20] updated --- app/realtime.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/realtime.php b/app/realtime.php index 667b035b9b..552823336f 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -804,7 +804,6 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, ]); $realtime->subscribe($project->getId(), $connection, '', $roles, [], [], $user->getId()); - $register->get('telemetry.workerSubscriptionCounter')->add(1, $register->get('telemetry.workerAttributes')); $realtime->connections[$connection]['authorization'] = $authorization; $server->send([$connection], $connectedPayloadJson); $updateStats($project->getId(), $project->getAttribute('teamId'), $connectedPayloadJson);