From 7644e0fe48eb0a2dbba50d6db71d4c41a8560d50 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 3 Mar 2026 18:48:37 +0530 Subject: [PATCH 01/57] Add realtime metrics for connections, messages, and bandwidth in project usage --- app/controllers/api/project.php | 34 ++++ app/init/constants.php | 6 + app/realtime.php | 107 ++++++++++- .../Utopia/Response/Model/UsageProject.php | 39 ++++ tests/e2e/General/UsageTest.php | 172 ++++++++++++++++++ 5 files changed, 355 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index d24519e3fb..1fd33a5db0 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -72,6 +72,10 @@ Http::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, + METRIC_REALTIME_CONNECTIONS, + METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, + METRIC_REALTIME_INBOUND, + METRIC_REALTIME_OUTBOUND, ], 'period' => [ METRIC_NETWORK_REQUESTS, @@ -85,6 +89,10 @@ Http::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, + METRIC_REALTIME_CONNECTIONS, + METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, + METRIC_REALTIME_INBOUND, + METRIC_REALTIME_OUTBOUND, ] ]; @@ -347,6 +355,26 @@ Http::get('/v1/project/usage') ]; } + // Realtime bandwidth = realtime.inbound + realtime.outbound (per bucket) + $realtimeProjectBandwidth = []; + foreach ($usage[METRIC_REALTIME_INBOUND] as $item) { + $realtimeProjectBandwidth[$item['date']] ??= 0; + $realtimeProjectBandwidth[$item['date']] += $item['value']; + } + + foreach ($usage[METRIC_REALTIME_OUTBOUND] as $item) { + $realtimeProjectBandwidth[$item['date']] ??= 0; + $realtimeProjectBandwidth[$item['date']] += $item['value']; + } + + $realtimeBandwidth = []; + foreach ($realtimeProjectBandwidth as $date => $value) { + $realtimeBandwidth[] = [ + 'date' => $date, + 'value' => $value + ]; + } + $response->dynamic(new Document([ 'requests' => ($usage[METRIC_NETWORK_REQUESTS]), 'network' => $network, @@ -367,10 +395,16 @@ Http::get('/v1/project/usage') 'deploymentsStorageTotal' => $total[METRIC_DEPLOYMENTS_STORAGE], 'databasesReadsTotal' => $total[METRIC_DATABASES_OPERATIONS_READS], 'databasesWritesTotal' => $total[METRIC_DATABASES_OPERATIONS_WRITES], + 'realtimeConnectionsTotal' => $total[METRIC_REALTIME_CONNECTIONS], + 'realtimeMessagesTotal' => $total[METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT], + 'realtimeBandwidthTotal' => ($total[METRIC_REALTIME_INBOUND] ?? 0) + ($total[METRIC_REALTIME_OUTBOUND] ?? 0), 'executionsBreakdown' => $executionsBreakdown, 'bucketsBreakdown' => $bucketsBreakdown, 'databasesReads' => $usage[METRIC_DATABASES_OPERATIONS_READS], 'databasesWrites' => $usage[METRIC_DATABASES_OPERATIONS_WRITES], + 'realtimeConnections' => $usage[METRIC_REALTIME_CONNECTIONS], + 'realtimeMessages' => $usage[METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT], + 'realtimeBandwidth' => $realtimeBandwidth, 'databasesStorageBreakdown' => $databasesStorageBreakdown, 'executionsMbSecondsBreakdown' => $executionsMbSecondsBreakdown, 'buildsMbSecondsBreakdown' => $buildsMbSecondsBreakdown, diff --git a/app/init/constants.php b/app/init/constants.php index c6a5bc853e..cd77b05746 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -361,6 +361,12 @@ const METRIC_AVATARS_SCREENSHOTS_GENERATED = 'avatars.screenshotsGenerated'; const METRIC_FUNCTIONS_RUNTIME = 'functions.runtimes.{runtime}'; const METRIC_SITES_FRAMEWORK = 'sites.frameworks.{framework}'; +// Realtime metrics +const METRIC_REALTIME_CONNECTIONS = 'realtime.connections'; +const METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT = 'realtime.messages.sent'; +const METRIC_REALTIME_INBOUND = 'realtime.inbound'; +const METRIC_REALTIME_OUTBOUND = 'realtime.outbound'; + // Resource types const RESOURCE_TYPE_PROJECTS = 'projects'; const RESOURCE_TYPE_FUNCTIONS = 'functions'; diff --git a/app/realtime.php b/app/realtime.php index 7ec24d03c8..8a066bcbab 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -1,5 +1,6 @@ getSequence()])) { + return $ctx['queueForStatsUsage'][$project->getSequence()]; + } + + global $register; + + /** @var Group $pools */ + $pools = $register->get('pools'); + + $queue = new StatsUsage(new BrokerPool(publisher: $pools->get('publisher'))); + $queue->setProject($project); + + return $ctx['queueForStatsUsage'][$project->getSequence()] = $queue; + } +} + $realtime = getRealtime(); /** @@ -545,20 +572,59 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, } $total = 0; + $outboundBytes = 0; + foreach ($groups as $group) { $data = $event['data']; $data['subscriptions'] = $group['subscriptions']; - $server->send($group['ids'], json_encode([ + $payloadJson = json_encode([ 'type' => 'event', 'data' => $data - ])); - $total += count($group['ids']); + ]); + + $server->send($group['ids'], $payloadJson); + + $count = count($group['ids']); + $total += $count; + $outboundBytes += strlen($payloadJson) * $count; } if ($total > 0) { $register->get('telemetry.messageSentCounter')->add($total); $stats->incr($event['project'], 'messages', $total); + + $projectId = $event['project'] ?? null; + + if (!empty($projectId)) { + try { + $consoleDB = getConsoleDB(); + /** @var Document $project */ + $project = $consoleDB->getAuthorization()->skip( + fn () => $consoleDB->getDocument('projects', $projectId) + ); + + if (!$project->isEmpty()) { + $queueForStatsUsage = getQueueForStatsUsageForProject($project); + + $queueForStatsUsage->addMetric( + METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, + $total + ); + + if ($outboundBytes > 0) { + $queueForStatsUsage->addMetric( + METRIC_REALTIME_OUTBOUND, + $outboundBytes + ); + } + + $queueForStatsUsage->trigger(); + } + } catch (Throwable $th) { + logError($th, 'realtimeUsageOutbound', tags: ['projectId' => $projectId]); + } + } } }); } catch (Throwable $th) { @@ -707,6 +773,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, ]); $stats->incr($project->getId(), 'connections'); $stats->incr($project->getId(), 'connectionsTotal'); + + try { + $queueForStatsUsage = getQueueForStatsUsageForProject($project); + $queueForStatsUsage + ->addMetric(METRIC_REALTIME_CONNECTIONS, 1) + ->trigger(); + } catch (\Throwable $th) { + logError($th, 'realtimeUsageConnections', project: $project); + } + } catch (Throwable $th) { logError($th, 'realtime', project: $project, user: $logUser, authorization: $authorization); @@ -748,6 +824,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $authorization = null; try { + $rawSize = \strlen($message); $response = new Response(new SwooleResponse()); $projectId = $realtime->connections[$connection]['projectId'] ?? null; @@ -786,6 +863,18 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many messages.'); } + // Record realtime inbound bytes for this project + if ($project !== null && !$project->isEmpty()) { + try { + $queueForStatsUsage = getQueueForStatsUsageForProject($project); + $queueForStatsUsage + ->addMetric(METRIC_REALTIME_INBOUND, $rawSize) + ->trigger(); + } catch (Throwable $th) { + logError($th, 'realtimeUsageInbound', project: $project); + } + } + $message = json_decode($message, true); if (is_null($message) || (!array_key_exists('type', $message) && !array_key_exists('data', $message))) { @@ -905,6 +994,18 @@ $server->onClose(function (int $connection) use ($realtime, $stats, $register) { if (array_key_exists($connection, $realtime->connections)) { $stats->decr($realtime->connections[$connection]['projectId'], 'connectionsTotal'); $register->get('telemetry.connectionCounter')->add(-1); + + $projectId = $realtime->connections[$connection]['projectId']; + + $consoleDB = getConsoleDB(); + $project = $consoleDB->getAuthorization()->skip( + fn () => $consoleDB->getDocument('projects', $projectId) + ); + + if (!$project->isEmpty()) { + $queue = getQueueForStatsUsageForProject($project); + $queue->addMetric(METRIC_REALTIME_CONNECTIONS, -1)->trigger(); + } } $realtime->unsubscribe($connection); diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index ee644aa845..c219a35f29 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -100,6 +100,24 @@ class UsageProject extends Model 'default' => 0, 'example' => 0, ]) + ->addRule('realtimeConnectionsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Current aggregated number of open Realtime connections.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('realtimeMessagesTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of Realtime messages sent to clients.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('realtimeBandwidthTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total consumed Realtime bandwidth (in bytes).', + 'default' => 0, + 'example' => 0, + ]) ->addRule('requests', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of requests per period.', @@ -114,6 +132,27 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) + ->addRule('realtimeConnections', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of open Realtime connections per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('realtimeMessages', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of Realtime messages sent to clients per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('realtimeBandwidth', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated consumed Realtime bandwidth (in bytes) per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ->addRule('users', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of users per period.', diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index e7b5b1d844..29229ae2bd 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -11,6 +11,7 @@ use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideServer; use Tests\E2E\Services\Functions\FunctionsBase; +use Tests\E2E\Services\Realtime\RealtimeBase; use Tests\E2E\Services\Sites\SitesBase; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -23,6 +24,7 @@ class UsageTest extends Scope use ProjectCustom; use SideServer; use FunctionsBase; + use RealtimeBase; use SitesBase { FunctionsBase::createDeployment insteadof SitesBase; FunctionsBase::setupDeployment insteadof SitesBase; @@ -1408,6 +1410,176 @@ class UsageTest extends Scope }); } + public function testRealtimeUsageMetrics(): void + { + $user = $this->getUser(); + $session = $user['session'] ?? ''; + $projectId = $this->getProject()['$id']; + + // Baseline realtime usage before opening a new connection + $baseline = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $connectionsBefore = $baseline['body']['realtimeConnectionsTotal'] ?? 0; + $messagesBefore = $baseline['body']['realtimeMessagesTotal'] ?? 0; + + $connectionCount = 3; + $clients = []; + + for ($i = 0; $i < $connectionCount; $i++) { + $client = $this->getWebsocket(['documents'], [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ], null); + + $connected = json_decode($client->receive(), true); + $this->assertEquals('connected', $connected['type']); + + $clients[] = $client; + } + + try { + $database = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Realtime Usage DB', + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Realtime Usage Collection', + 'permissions' => [ + Permission::create(Role::user($user['$id'])), + ], + 'documentSecurity' => true, + ]); + + $collectionId = $collection['body']['$id']; + + $attribute = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'key' => 'name', + 'size' => 256, + 'required' => true, + ]); + + $this->assertEquals(202, $attribute['headers']['status-code']); + + $this->assertEventually(function () use ($databaseId, $collectionId, $projectId) { + $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/name', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + $this->assertEquals('available', $response['body']['status']); + }, 30000, 250); + + $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders()), [ + 'documentId' => ID::unique(), + 'data' => [ + 'name' => 'Realtime Usage Doc', + ], + 'permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $this->assertEquals(201, $document['headers']['status-code']); + + $event = json_decode($clients[0]->receive(), true); + $this->assertEquals('event', $event['type']); + + // After creating a document we expect all connections to receive an event + $this->assertEventually(function () use ($connectionsBefore, $messagesBefore, $connectionCount) { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertEquals(200, $response['headers']['status-code']); + + $this->assertArrayHasKey('realtimeConnectionsTotal', $response['body']); + $this->assertArrayHasKey('realtimeMessagesTotal', $response['body']); + $this->assertArrayHasKey('realtimeBandwidthTotal', $response['body']); + $this->assertArrayHasKey('realtimeConnections', $response['body']); + $this->assertArrayHasKey('realtimeMessages', $response['body']); + $this->assertArrayHasKey('realtimeBandwidth', $response['body']); + + // We expect exactly $connectionCount additional open connections and $connectionCount additional message deliveries + $this->assertEquals($connectionsBefore + $connectionCount, $response['body']['realtimeConnectionsTotal']); + $this->assertEquals($messagesBefore + $connectionCount, $response['body']['realtimeMessagesTotal']); + $this->assertGreaterThan(0, $response['body']['realtimeBandwidthTotal']); + + $this->validateDates($response['body']['realtimeConnections']); + $this->validateDates($response['body']['realtimeMessages']); + $this->validateDates($response['body']['realtimeBandwidth']); + }, 60000, 2000); + + // Now close a single connection and ensure the counters reflect it + $clients[0]->close(); + + $this->assertEventually(function () use ($connectionsBefore, $messagesBefore, $connectionCount) { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertEquals(200, $response['headers']['status-code']); + + $this->assertArrayHasKey('realtimeConnectionsTotal', $response['body']); + $this->assertArrayHasKey('realtimeMessagesTotal', $response['body']); + $this->assertArrayHasKey('realtimeBandwidthTotal', $response['body']); + + // One of the connections is closed, so we expect one less open connection. + // Messages and bandwidth are cumulative and should not decrease. + $this->assertEquals($connectionsBefore + $connectionCount - 1, $response['body']['realtimeConnectionsTotal']); + $this->assertEquals($messagesBefore + $connectionCount, $response['body']['realtimeMessagesTotal']); + $this->assertGreaterThan(0, $response['body']['realtimeBandwidthTotal']); + }, 60000, 2000); + } finally { + foreach ($clients as $client) { + $client->close(); + } + } + } + public function tearDown(): void { $this->projectId = ''; From 82db411517641559f9ebbf52d2f3a809f74e8830 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 3 Mar 2026 19:36:49 +0530 Subject: [PATCH 02/57] updated --- app/realtime.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/realtime.php b/app/realtime.php index 8a066bcbab..33833d27ea 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -226,7 +226,7 @@ if (!function_exists('getTelemetry')) { } } -if (!function_exists('queueForStatsUsage')) { +if (!function_exists('getQueueForStatsUsageForProject')) { function getQueueForStatsUsageForProject(Document $project): StatsUsage { $ctx = Coroutine::getContext(); From 9fdd7c1c6e55181eea155a820352e870b5fe0e8d Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 5 Mar 2026 11:30:15 +0530 Subject: [PATCH 03/57] added try catch connection close metric --- app/realtime.php | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 33833d27ea..76a84e1d9d 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -997,14 +997,18 @@ $server->onClose(function (int $connection) use ($realtime, $stats, $register) { $projectId = $realtime->connections[$connection]['projectId']; - $consoleDB = getConsoleDB(); - $project = $consoleDB->getAuthorization()->skip( - fn () => $consoleDB->getDocument('projects', $projectId) - ); + try { + $consoleDB = getConsoleDB(); + $project = $consoleDB->getAuthorization()->skip( + fn () => $consoleDB->getDocument('projects', $projectId) + ); - if (!$project->isEmpty()) { - $queue = getQueueForStatsUsageForProject($project); - $queue->addMetric(METRIC_REALTIME_CONNECTIONS, -1)->trigger(); + if (!$project->isEmpty()) { + $queue = getQueueForStatsUsageForProject($project); + $queue->addMetric(METRIC_REALTIME_CONNECTIONS, -1)->trigger(); + } + } catch (Throwable $th) { + logError($th, 'realtimeUsageConnectionClose', tags: ['projectId' => $projectId]); } } $realtime->unsubscribe($connection); From b5c2cc971610df03dabe1045624eb20b2a2d2953 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 5 Mar 2026 18:10:09 +0530 Subject: [PATCH 04/57] Add triggerStats function for realtime usage metrics tracking --- app/realtime.php | 160 +++++++++++++++++++++++++++++------------------ 1 file changed, 100 insertions(+), 60 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 76a84e1d9d..8250a395a9 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -251,6 +251,43 @@ if (!function_exists('getQueueForStatsUsageForProject')) { } } +if (!function_exists('triggerStats')) { + /** + * Trigger realtime usage stats with a generic metric map. + * + * @param array $event Metrics in the form METRIC_CONSTANT => value + */ + function triggerStats(array $event, string $projectId): void + { + if (empty($projectId)) { + return; + } + + try { + $consoleDB = getConsoleDB(); + /** @var Document $project */ + $project = $consoleDB->getAuthorization()->skip( + fn () => $consoleDB->getDocument('projects', $projectId) + ); + + if ($project->isEmpty()) { + return; + } + + $queueForStatsUsage = getQueueForStatsUsageForProject($project); + + foreach ($event as $metric => $value) { + $queueForStatsUsage->addMetric($metric, $value); + } + + $queueForStatsUsage->trigger(); + $queueForStatsUsage->reset(); + } catch (Throwable $th) { + logError($th, 'realtimeStats', tags: ['projectId' => $projectId]); + } + } +} + $realtime = getRealtime(); /** @@ -597,33 +634,15 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $projectId = $event['project'] ?? null; if (!empty($projectId)) { - try { - $consoleDB = getConsoleDB(); - /** @var Document $project */ - $project = $consoleDB->getAuthorization()->skip( - fn () => $consoleDB->getDocument('projects', $projectId) - ); + $metrics = [ + METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT => $total, + ]; - if (!$project->isEmpty()) { - $queueForStatsUsage = getQueueForStatsUsageForProject($project); - - $queueForStatsUsage->addMetric( - METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, - $total - ); - - if ($outboundBytes > 0) { - $queueForStatsUsage->addMetric( - METRIC_REALTIME_OUTBOUND, - $outboundBytes - ); - } - - $queueForStatsUsage->trigger(); - } - } catch (Throwable $th) { - logError($th, 'realtimeUsageOutbound', tags: ['projectId' => $projectId]); + if ($outboundBytes > 0) { + $metrics[METRIC_REALTIME_OUTBOUND] = $outboundBytes; } + + triggerStats($metrics, $projectId); } } }); @@ -701,6 +720,19 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many requests'); } + // Record realtime inbound bytes for this project (WS handshake + query params) + try { + $rawSize = $request->getSize(); + } catch (Throwable) { + $rawSize = \strlen((string) $request->getURI()); + } + + if ($rawSize > 0) { + triggerStats([ + METRIC_REALTIME_INBOUND => $rawSize, + ], $project->getId()); + } + /* * Validate Client Domain - Check to avoid CSRF attack. * Adding Appwrite API domains to allow XDOMAIN communication. @@ -755,14 +787,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $user = empty($user->getId()) ? null : $response->output($user, Response::MODEL_ACCOUNT); - $server->send([$connection], json_encode([ + $connectedPayloadJson = json_encode([ 'type' => 'connected', 'data' => [ 'channels' => $names, 'subscriptions' => $mapping, 'user' => $user ] - ])); + ]); + + $server->send([$connection], $connectedPayloadJson); $register->get('telemetry.connectionCounter')->add(1); $register->get('telemetry.connectionCreatedCounter')->add(1); @@ -774,14 +808,10 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $stats->incr($project->getId(), 'connections'); $stats->incr($project->getId(), 'connectionsTotal'); - try { - $queueForStatsUsage = getQueueForStatsUsageForProject($project); - $queueForStatsUsage - ->addMetric(METRIC_REALTIME_CONNECTIONS, 1) - ->trigger(); - } catch (\Throwable $th) { - logError($th, 'realtimeUsageConnections', project: $project); - } + $connectedOutboundBytes = \strlen($connectedPayloadJson); + + triggerStats([METRIC_REALTIME_CONNECTIONS => 1, METRIC_REALTIME_OUTBOUND => $connectedOutboundBytes], $project->getId()); + } catch (Throwable $th) { logError($th, 'realtime', project: $project, user: $logUser, authorization: $authorization); @@ -865,14 +895,9 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re // Record realtime inbound bytes for this project if ($project !== null && !$project->isEmpty()) { - try { - $queueForStatsUsage = getQueueForStatsUsageForProject($project); - $queueForStatsUsage - ->addMetric(METRIC_REALTIME_INBOUND, $rawSize) - ->trigger(); - } catch (Throwable $th) { - logError($th, 'realtimeUsageInbound', project: $project); - } + triggerStats([ + METRIC_REALTIME_INBOUND => $rawSize, + ], $project->getId()); } $message = json_decode($message, true); @@ -883,9 +908,21 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re switch ($message['type']) { case 'ping': - $server->send([$connection], json_encode([ + $pongPayloadJson = json_encode([ 'type' => 'pong' - ])); + ]); + + $server->send([$connection], $pongPayloadJson); + + if ($project !== null && !$project->isEmpty()) { + $pongOutboundBytes = \strlen($pongPayloadJson); + + if ($pongOutboundBytes > 0) { + triggerStats([ + METRIC_REALTIME_OUTBOUND => $pongOutboundBytes, + ], $project->getId()); + } + } break; case 'authentication': @@ -946,14 +983,27 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re } $user = $response->output($user, Response::MODEL_ACCOUNT); - $server->send([$connection], json_encode([ + + $authResponsePayloadJson = json_encode([ 'type' => 'response', 'data' => [ 'to' => 'authentication', 'success' => true, 'user' => $user ] - ])); + ]); + + $server->send([$connection], $authResponsePayloadJson); + + if ($project !== null && !$project->isEmpty()) { + $authOutboundBytes = \strlen($authResponsePayloadJson); + + if ($authOutboundBytes > 0) { + triggerStats([ + METRIC_REALTIME_OUTBOUND => $authOutboundBytes, + ], $project->getId()); + } + } break; @@ -997,19 +1047,9 @@ $server->onClose(function (int $connection) use ($realtime, $stats, $register) { $projectId = $realtime->connections[$connection]['projectId']; - try { - $consoleDB = getConsoleDB(); - $project = $consoleDB->getAuthorization()->skip( - fn () => $consoleDB->getDocument('projects', $projectId) - ); - - if (!$project->isEmpty()) { - $queue = getQueueForStatsUsageForProject($project); - $queue->addMetric(METRIC_REALTIME_CONNECTIONS, -1)->trigger(); - } - } catch (Throwable $th) { - logError($th, 'realtimeUsageConnectionClose', tags: ['projectId' => $projectId]); - } + triggerStats([ + METRIC_REALTIME_CONNECTIONS => -1, + ], $projectId); } $realtime->unsubscribe($connection); From 03d7bccde2a4465d651a5575b4a3321990ced6fe Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 5 Mar 2026 18:14:16 +0530 Subject: [PATCH 05/57] updated tests --- tests/e2e/General/UsageTest.php | 55 +++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 29229ae2bd..2eb2673c8a 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1546,6 +1546,61 @@ class UsageTest extends Scope $this->validateDates($response['body']['realtimeBandwidth']); }, 60000, 2000); + // Capture a snapshot of usage after the broadcasted document event + $afterEventUsage = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertEquals(200, $afterEventUsage['headers']['status-code']); + + $connectionsAfterEvent = $afterEventUsage['body']['realtimeConnectionsTotal'] ?? 0; + $messagesAfterEvent = $afterEventUsage['body']['realtimeMessagesTotal'] ?? 0; + $bandwidthAfterEvent = $afterEventUsage['body']['realtimeBandwidthTotal'] ?? 0; + + // Send a ping over an existing connection to exercise the ping/pong + // realtime usage metrics path (inbound + outbound bytes) without + // generating additional "messages sent" usage. + $clients[1]->send(json_encode([ + 'type' => 'ping', + ])); + + $firstMessage = json_decode($clients[1]->receive(), true); + // Depending on timing, the first frame we see here can be either + // a broadcast "event" (from another operation) or the "pong" + // response. Both are valid, and in either case the ping/pong + // traffic still exercises the realtime usage metrics paths. + $this->assertContains($firstMessage['type'], ['event']); + + // We expect: + // - connections count to remain the same + // - messages count to remain the same (no new broadcast events) + // - bandwidth total to increase because of ping/pong traffic + $this->assertEventually(function () use ($connectionsAfterEvent, $messagesAfterEvent, $bandwidthAfterEvent) { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertEquals(200, $response['headers']['status-code']); + + $this->assertEquals($connectionsAfterEvent, $response['body']['realtimeConnectionsTotal']); + $this->assertEquals($messagesAfterEvent, $response['body']['realtimeMessagesTotal']); + $this->assertGreaterThan($bandwidthAfterEvent, $response['body']['realtimeBandwidthTotal']); + }, 60000, 2000); + // Now close a single connection and ensure the counters reflect it $clients[0]->close(); From ece5b8173277ff0d41cbbb0b56c798baf0a0f2fe Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 6 Mar 2026 15:27:30 +0530 Subject: [PATCH 06/57] updated test --- tests/e2e/General/UsageTest.php | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 2eb2673c8a..f51b04b9c6 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1571,12 +1571,23 @@ class UsageTest extends Scope 'type' => 'ping', ])); - $firstMessage = json_decode($clients[1]->receive(), true); - // Depending on timing, the first frame we see here can be either - // a broadcast "event" (from another operation) or the "pong" - // response. Both are valid, and in either case the ping/pong - // traffic still exercises the realtime usage metrics paths. - $this->assertContains($firstMessage['type'], ['event']); + // A broadcast document "event" can still be queued for this connection. + // Read until we see the pong response (and fail fast on unexpected frames). + $pong = null; + for ($i = 0; $i < 5; $i++) { + $frame = json_decode($clients[1]->receive(), true); + $this->assertIsArray($frame); + $this->assertArrayHasKey('type', $frame); + + if ($frame['type'] === 'pong') { + $pong = $frame; + break; + } + + $this->assertEquals('event', $frame['type']); + } + + $this->assertNotNull($pong, 'Expected to receive a pong frame after ping.'); // We expect: // - connections count to remain the same From 0ea196d21c3a76fcee5d04fc3024fba1d6d6eadd Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 6 Mar 2026 15:37:30 +0530 Subject: [PATCH 07/57] updated inbound raw size to the request size --- app/realtime.php | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 8250a395a9..9ab6601ed7 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -720,18 +720,11 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many requests'); } - // Record realtime inbound bytes for this project (WS handshake + query params) - try { - $rawSize = $request->getSize(); - } catch (Throwable) { - $rawSize = \strlen((string) $request->getURI()); - } + $rawSize = $request->getSize(); - if ($rawSize > 0) { - triggerStats([ - METRIC_REALTIME_INBOUND => $rawSize, - ], $project->getId()); - } + triggerStats([ + METRIC_REALTIME_INBOUND => $rawSize, + ], $project->getId()); /* * Validate Client Domain - Check to avoid CSRF attack. From ab0fa70bca60c26d191f4a43499f8908694e6ec4 Mon Sep 17 00:00:00 2001 From: eldadfux Date: Mon, 9 Mar 2026 19:31:44 +0100 Subject: [PATCH 08/57] Enhance project context validation in realtime message handling. Ensure that projectId is checked for emptiness before processing messages, and enforce project context requirement for non-ping messages. --- app/realtime.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/realtime.php b/app/realtime.php index e0591a2596..0239e70f22 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -763,7 +763,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $database = getConsoleDB(); $database->setAuthorization($authorization); - if ($projectId !== 'console') { + if (!empty($projectId) && $projectId !== 'console') { $project = $authorization->skip(fn () => $database->getDocument('projects', $projectId)); $database = getProjectDB($project); @@ -795,6 +795,11 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Message format is not valid.'); } + // Ping does not require project context; other messages do (e.g. after unsubscribe during auth) + if (empty($projectId) && ($message['type'] ?? '') !== 'ping') { + throw new Exception(Exception::REALTIME_POLICY_VIOLATION, 'Missing project context. Reconnect to the project first.'); + } + switch ($message['type']) { case 'ping': $server->send([$connection], json_encode([ From 39f3bc7b9dfc09d6c277b01a6efc61a43e30e637 Mon Sep 17 00:00:00 2001 From: eldadfux Date: Mon, 9 Mar 2026 20:08:41 +0100 Subject: [PATCH 09/57] Fix SDK namespace call --- app/controllers/general.php | 3 +++ app/http.php | 3 +++ 2 files changed, 6 insertions(+) diff --git a/app/controllers/general.php b/app/controllers/general.php index 57edd98bc4..f77aa3ec52 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -1399,6 +1399,9 @@ Http::error() $sdk = $route?->getLabel("sdk", false); $action = 'UNKNOWN_NAMESPACE.UNKNOWN.METHOD'; if (!empty($sdk)) { + if (\is_array($sdk)) { + $sdk = $sdk[0]; + } /** @var \Appwrite\SDK\Method $sdk */ $action = $sdk->getNamespace() . '.' . $sdk->getMethodName(); } elseif ($route === null) { diff --git a/app/http.php b/app/http.php index 7f771de130..1302940856 100644 --- a/app/http.php +++ b/app/http.php @@ -581,6 +581,9 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool $action = 'UNKNOWN_NAMESPACE.UNKNOWN.METHOD'; if (!empty($sdk)) { + if (\is_array($sdk)) { + $sdk = $sdk[0]; + } /** @var Appwrite\SDK\Method $sdk */ $action = $sdk->getNamespace() . '.' . $sdk->getMethodName(); } elseif ($route === null) { From a0167d6c6c18560cd8d6d4b8762437c7358916aa Mon Sep 17 00:00:00 2001 From: eldadfux Date: Mon, 9 Mar 2026 20:17:32 +0100 Subject: [PATCH 10/57] Fix for when vcs comment is empty --- src/Appwrite/Vcs/Comment.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Vcs/Comment.php b/src/Appwrite/Vcs/Comment.php index 90f9c8f95d..e6d6996748 100644 --- a/src/Appwrite/Vcs/Comment.php +++ b/src/Appwrite/Vcs/Comment.php @@ -251,7 +251,7 @@ class Comment $json = \base64_decode($state); $builds = \json_decode($json, true); - $this->builds = $builds; + $this->builds = \is_array($builds) ? $builds : []; return $this; } From 95b3db0228c65b65a128d4ab0454ae17d7430bbe Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 9 Mar 2026 20:40:34 +0000 Subject: [PATCH 11/57] fix: storage health error swallowing --- .../Health/Http/Health/Storage/Get.php | 36 ++++++------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Health/Http/Health/Storage/Get.php b/src/Appwrite/Platform/Modules/Health/Http/Health/Storage/Get.php index 52468cab5a..6757f6a334 100644 --- a/src/Appwrite/Platform/Modules/Health/Http/Health/Storage/Get.php +++ b/src/Appwrite/Platform/Modules/Health/Http/Health/Storage/Get.php @@ -2,7 +2,6 @@ namespace Appwrite\Platform\Modules\Health\Http\Health\Storage; -use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; @@ -58,34 +57,21 @@ class Get extends Action $checkStart = \microtime(true); foreach ($devices as $device) { - $uniqueFileName = \uniqid('health', true); - $filePath = $device->getPath($uniqueFileName); + $path = $device->getPath(\uniqid('health', true)); - if (!$device->write($filePath, 'test', 'text/plain')) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed writing test file to ' . $device->getRoot()); - } - - $readError = null; try { - if ($device->read($filePath) !== 'test') { - $readError = new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed reading test file from ' . $device->getRoot()); + if (!$device->write($path, 'test', 'text/plain')) { + throw new \Exception("Failed writing test file to {$device->getRoot()}"); + } + + $content = $device->read($path); + if ($content !== 'test') { + throw new \Exception("Failed reading test file from {$device->getRoot()}: content mismatch"); } - } catch (\Throwable $e) { - $readError = $e; } finally { - // Always attempt to clean up test file - if (!$device->delete($filePath)) { - if ($readError !== null) { - // If read already failed, wrap delete error but preserve original - \error_log('Failed deleting test file from ' . $device->getRoot() . ' during read error recovery'); - } else { - throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed deleting test file from ' . $device->getRoot()); - } - } - // Re-throw read error if it occurred - if ($readError !== null) { - throw $readError; - } + try { + $device->delete($path); + } catch (\Throwable) {} } } From 4d7aa6d8ab241ea5e43917e459b682072d6b0767 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 9 Mar 2026 20:47:07 +0000 Subject: [PATCH 12/57] style: format storage health check braces Co-Authored-By: Claude Opus 4.6 --- .../Platform/Modules/Health/Http/Health/Storage/Get.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Health/Http/Health/Storage/Get.php b/src/Appwrite/Platform/Modules/Health/Http/Health/Storage/Get.php index 6757f6a334..93c9483959 100644 --- a/src/Appwrite/Platform/Modules/Health/Http/Health/Storage/Get.php +++ b/src/Appwrite/Platform/Modules/Health/Http/Health/Storage/Get.php @@ -71,7 +71,8 @@ class Get extends Action } finally { try { $device->delete($path); - } catch (\Throwable) {} + } catch (\Throwable) { + } } } From eccc39a4669db5afb52e35928e071524329a2a05 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 10 Mar 2026 12:15:25 +0530 Subject: [PATCH 13/57] refactor: remove realtime metrics from project usage endpoints and related tests --- app/controllers/api/project.php | 34 --- app/realtime.php | 59 +---- .../Utopia/Response/Model/UsageProject.php | 39 --- tests/e2e/General/UsageTest.php | 236 ------------------ 4 files changed, 1 insertion(+), 367 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 1fd33a5db0..d24519e3fb 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -72,10 +72,6 @@ Http::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, - METRIC_REALTIME_CONNECTIONS, - METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, - METRIC_REALTIME_INBOUND, - METRIC_REALTIME_OUTBOUND, ], 'period' => [ METRIC_NETWORK_REQUESTS, @@ -89,10 +85,6 @@ Http::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, - METRIC_REALTIME_CONNECTIONS, - METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, - METRIC_REALTIME_INBOUND, - METRIC_REALTIME_OUTBOUND, ] ]; @@ -355,26 +347,6 @@ Http::get('/v1/project/usage') ]; } - // Realtime bandwidth = realtime.inbound + realtime.outbound (per bucket) - $realtimeProjectBandwidth = []; - foreach ($usage[METRIC_REALTIME_INBOUND] as $item) { - $realtimeProjectBandwidth[$item['date']] ??= 0; - $realtimeProjectBandwidth[$item['date']] += $item['value']; - } - - foreach ($usage[METRIC_REALTIME_OUTBOUND] as $item) { - $realtimeProjectBandwidth[$item['date']] ??= 0; - $realtimeProjectBandwidth[$item['date']] += $item['value']; - } - - $realtimeBandwidth = []; - foreach ($realtimeProjectBandwidth as $date => $value) { - $realtimeBandwidth[] = [ - 'date' => $date, - 'value' => $value - ]; - } - $response->dynamic(new Document([ 'requests' => ($usage[METRIC_NETWORK_REQUESTS]), 'network' => $network, @@ -395,16 +367,10 @@ Http::get('/v1/project/usage') 'deploymentsStorageTotal' => $total[METRIC_DEPLOYMENTS_STORAGE], 'databasesReadsTotal' => $total[METRIC_DATABASES_OPERATIONS_READS], 'databasesWritesTotal' => $total[METRIC_DATABASES_OPERATIONS_WRITES], - 'realtimeConnectionsTotal' => $total[METRIC_REALTIME_CONNECTIONS], - 'realtimeMessagesTotal' => $total[METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT], - 'realtimeBandwidthTotal' => ($total[METRIC_REALTIME_INBOUND] ?? 0) + ($total[METRIC_REALTIME_OUTBOUND] ?? 0), 'executionsBreakdown' => $executionsBreakdown, 'bucketsBreakdown' => $bucketsBreakdown, 'databasesReads' => $usage[METRIC_DATABASES_OPERATIONS_READS], 'databasesWrites' => $usage[METRIC_DATABASES_OPERATIONS_WRITES], - 'realtimeConnections' => $usage[METRIC_REALTIME_CONNECTIONS], - 'realtimeMessages' => $usage[METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT], - 'realtimeBandwidth' => $realtimeBandwidth, 'databasesStorageBreakdown' => $databasesStorageBreakdown, 'executionsMbSecondsBreakdown' => $executionsMbSecondsBreakdown, 'buildsMbSecondsBreakdown' => $buildsMbSecondsBreakdown, diff --git a/app/realtime.php b/app/realtime.php index 9ab6601ed7..1e58832203 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -1,6 +1,5 @@ getSequence()])) { - return $ctx['queueForStatsUsage'][$project->getSequence()]; - } - - global $register; - - /** @var Group $pools */ - $pools = $register->get('pools'); - - $queue = new StatsUsage(new BrokerPool(publisher: $pools->get('publisher'))); - $queue->setProject($project); - - return $ctx['queueForStatsUsage'][$project->getSequence()] = $queue; - } -} - if (!function_exists('triggerStats')) { - /** - * Trigger realtime usage stats with a generic metric map. - * - * @param array $event Metrics in the form METRIC_CONSTANT => value - */ function triggerStats(array $event, string $projectId): void { - if (empty($projectId)) { - return; - } - - try { - $consoleDB = getConsoleDB(); - /** @var Document $project */ - $project = $consoleDB->getAuthorization()->skip( - fn () => $consoleDB->getDocument('projects', $projectId) - ); - - if ($project->isEmpty()) { - return; - } - - $queueForStatsUsage = getQueueForStatsUsageForProject($project); - - foreach ($event as $metric => $value) { - $queueForStatsUsage->addMetric($metric, $value); - } - - $queueForStatsUsage->trigger(); - $queueForStatsUsage->reset(); - } catch (Throwable $th) { - logError($th, 'realtimeStats', tags: ['projectId' => $projectId]); - } + return; } } diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index c219a35f29..ee644aa845 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -100,24 +100,6 @@ class UsageProject extends Model 'default' => 0, 'example' => 0, ]) - ->addRule('realtimeConnectionsTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Current aggregated number of open Realtime connections.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('realtimeMessagesTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total number of Realtime messages sent to clients.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('realtimeBandwidthTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total consumed Realtime bandwidth (in bytes).', - 'default' => 0, - 'example' => 0, - ]) ->addRule('requests', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of requests per period.', @@ -132,27 +114,6 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) - ->addRule('realtimeConnections', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of open Realtime connections per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - ->addRule('realtimeMessages', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of Realtime messages sent to clients per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - ->addRule('realtimeBandwidth', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated consumed Realtime bandwidth (in bytes) per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) ->addRule('users', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of users per period.', diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index f51b04b9c6..cabbdf6913 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1410,242 +1410,6 @@ class UsageTest extends Scope }); } - public function testRealtimeUsageMetrics(): void - { - $user = $this->getUser(); - $session = $user['session'] ?? ''; - $projectId = $this->getProject()['$id']; - - // Baseline realtime usage before opening a new connection - $baseline = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $connectionsBefore = $baseline['body']['realtimeConnectionsTotal'] ?? 0; - $messagesBefore = $baseline['body']['realtimeMessagesTotal'] ?? 0; - - $connectionCount = 3; - $clients = []; - - for ($i = 0; $i < $connectionCount; $i++) { - $client = $this->getWebsocket(['documents'], [ - 'origin' => 'http://localhost', - 'cookie' => 'a_session_' . $projectId . '=' . $session, - ], null); - - $connected = json_decode($client->receive(), true); - $this->assertEquals('connected', $connected['type']); - - $clients[] = $client; - } - - try { - $database = $this->client->call(Client::METHOD_POST, '/databases', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - 'x-appwrite-key' => $this->getProject()['apiKey'], - ], [ - 'databaseId' => ID::unique(), - 'name' => 'Realtime Usage DB', - ]); - - $databaseId = $database['body']['$id']; - - $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - 'x-appwrite-key' => $this->getProject()['apiKey'], - ], [ - 'collectionId' => ID::unique(), - 'name' => 'Realtime Usage Collection', - 'permissions' => [ - Permission::create(Role::user($user['$id'])), - ], - 'documentSecurity' => true, - ]); - - $collectionId = $collection['body']['$id']; - - $attribute = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - 'x-appwrite-key' => $this->getProject()['apiKey'], - ], [ - 'key' => 'name', - 'size' => 256, - 'required' => true, - ]); - - $this->assertEquals(202, $attribute['headers']['status-code']); - - $this->assertEventually(function () use ($databaseId, $collectionId, $projectId) { - $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/name', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - 'x-appwrite-key' => $this->getProject()['apiKey'], - ]); - $this->assertEquals('available', $response['body']['status']); - }, 30000, 250); - - $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - ], $this->getHeaders()), [ - 'documentId' => ID::unique(), - 'data' => [ - 'name' => 'Realtime Usage Doc', - ], - 'permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - ]); - - $this->assertEquals(201, $document['headers']['status-code']); - - $event = json_decode($clients[0]->receive(), true); - $this->assertEquals('event', $event['type']); - - // After creating a document we expect all connections to receive an event - $this->assertEventually(function () use ($connectionsBefore, $messagesBefore, $connectionCount) { - $response = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $this->assertEquals(200, $response['headers']['status-code']); - - $this->assertArrayHasKey('realtimeConnectionsTotal', $response['body']); - $this->assertArrayHasKey('realtimeMessagesTotal', $response['body']); - $this->assertArrayHasKey('realtimeBandwidthTotal', $response['body']); - $this->assertArrayHasKey('realtimeConnections', $response['body']); - $this->assertArrayHasKey('realtimeMessages', $response['body']); - $this->assertArrayHasKey('realtimeBandwidth', $response['body']); - - // We expect exactly $connectionCount additional open connections and $connectionCount additional message deliveries - $this->assertEquals($connectionsBefore + $connectionCount, $response['body']['realtimeConnectionsTotal']); - $this->assertEquals($messagesBefore + $connectionCount, $response['body']['realtimeMessagesTotal']); - $this->assertGreaterThan(0, $response['body']['realtimeBandwidthTotal']); - - $this->validateDates($response['body']['realtimeConnections']); - $this->validateDates($response['body']['realtimeMessages']); - $this->validateDates($response['body']['realtimeBandwidth']); - }, 60000, 2000); - - // Capture a snapshot of usage after the broadcasted document event - $afterEventUsage = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $this->assertEquals(200, $afterEventUsage['headers']['status-code']); - - $connectionsAfterEvent = $afterEventUsage['body']['realtimeConnectionsTotal'] ?? 0; - $messagesAfterEvent = $afterEventUsage['body']['realtimeMessagesTotal'] ?? 0; - $bandwidthAfterEvent = $afterEventUsage['body']['realtimeBandwidthTotal'] ?? 0; - - // Send a ping over an existing connection to exercise the ping/pong - // realtime usage metrics path (inbound + outbound bytes) without - // generating additional "messages sent" usage. - $clients[1]->send(json_encode([ - 'type' => 'ping', - ])); - - // A broadcast document "event" can still be queued for this connection. - // Read until we see the pong response (and fail fast on unexpected frames). - $pong = null; - for ($i = 0; $i < 5; $i++) { - $frame = json_decode($clients[1]->receive(), true); - $this->assertIsArray($frame); - $this->assertArrayHasKey('type', $frame); - - if ($frame['type'] === 'pong') { - $pong = $frame; - break; - } - - $this->assertEquals('event', $frame['type']); - } - - $this->assertNotNull($pong, 'Expected to receive a pong frame after ping.'); - - // We expect: - // - connections count to remain the same - // - messages count to remain the same (no new broadcast events) - // - bandwidth total to increase because of ping/pong traffic - $this->assertEventually(function () use ($connectionsAfterEvent, $messagesAfterEvent, $bandwidthAfterEvent) { - $response = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $this->assertEquals(200, $response['headers']['status-code']); - - $this->assertEquals($connectionsAfterEvent, $response['body']['realtimeConnectionsTotal']); - $this->assertEquals($messagesAfterEvent, $response['body']['realtimeMessagesTotal']); - $this->assertGreaterThan($bandwidthAfterEvent, $response['body']['realtimeBandwidthTotal']); - }, 60000, 2000); - - // Now close a single connection and ensure the counters reflect it - $clients[0]->close(); - - $this->assertEventually(function () use ($connectionsBefore, $messagesBefore, $connectionCount) { - $response = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $this->assertEquals(200, $response['headers']['status-code']); - - $this->assertArrayHasKey('realtimeConnectionsTotal', $response['body']); - $this->assertArrayHasKey('realtimeMessagesTotal', $response['body']); - $this->assertArrayHasKey('realtimeBandwidthTotal', $response['body']); - - // One of the connections is closed, so we expect one less open connection. - // Messages and bandwidth are cumulative and should not decrease. - $this->assertEquals($connectionsBefore + $connectionCount - 1, $response['body']['realtimeConnectionsTotal']); - $this->assertEquals($messagesBefore + $connectionCount, $response['body']['realtimeMessagesTotal']); - $this->assertGreaterThan(0, $response['body']['realtimeBandwidthTotal']); - }, 60000, 2000); - } finally { - foreach ($clients as $client) { - $client->close(); - } - } - } - public function tearDown(): void { $this->projectId = ''; From 7f4cba276e63700c75052bade5e41a62d11e6414 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 10 Mar 2026 12:19:46 +0530 Subject: [PATCH 14/57] updated tests --- tests/e2e/General/UsageTest.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index cabbdf6913..e7b5b1d844 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -11,7 +11,6 @@ use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideServer; use Tests\E2E\Services\Functions\FunctionsBase; -use Tests\E2E\Services\Realtime\RealtimeBase; use Tests\E2E\Services\Sites\SitesBase; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -24,7 +23,6 @@ class UsageTest extends Scope use ProjectCustom; use SideServer; use FunctionsBase; - use RealtimeBase; use SitesBase { FunctionsBase::createDeployment insteadof SitesBase; FunctionsBase::setupDeployment insteadof SitesBase; From ee2e41fa45fc358ce2266f4b4a2b6367b227e408 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Tue, 10 Mar 2026 09:20:57 +0000 Subject: [PATCH 15/57] feat: add messaging resource migration support --- composer.json | 2 +- composer.lock | 149 ++-- src/Appwrite/Platform/Workers/Migrations.php | 10 + .../Utopia/Response/Model/MigrationReport.php | 24 + .../Services/Migrations/MigrationsBase.php | 838 ++++++++++++++++++ 5 files changed, 947 insertions(+), 76 deletions(-) diff --git a/composer.json b/composer.json index d7f0ae66b5..abe51e500b 100644 --- a/composer.json +++ b/composer.json @@ -70,7 +70,7 @@ "utopia-php/locale": "0.8.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.20.*", - "utopia-php/migration": "1.6.*", + "utopia-php/migration": "1.7.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "1.*", "utopia-php/span": "1.1.*", diff --git a/composer.lock b/composer.lock index 86093899d0..738db8f287 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1cc64e07484256225f56bd525674c3b8", + "content-hash": "b99693284208ff3d006260a089a4f7b9", "packages": [ { "name": "adhocore/jwt", @@ -4058,16 +4058,16 @@ }, { "name": "utopia-php/domains", - "version": "1.0.5", + "version": "1.0.2", "source": { "type": "git", "url": "https://github.com/utopia-php/domains.git", - "reference": "0edf6bb2b07f30db849a267027077bf5abb994c6" + "reference": "b4896a6746f0fbe29dfd5e32f7790bd94c1af1e6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/domains/zipball/0edf6bb2b07f30db849a267027077bf5abb994c6", - "reference": "0edf6bb2b07f30db849a267027077bf5abb994c6", + "url": "https://api.github.com/repos/utopia-php/domains/zipball/b4896a6746f0fbe29dfd5e32f7790bd94c1af1e6", + "reference": "b4896a6746f0fbe29dfd5e32f7790bd94c1af1e6", "shasum": "" }, "require": { @@ -4114,9 +4114,9 @@ ], "support": { "issues": "https://github.com/utopia-php/domains/issues", - "source": "https://github.com/utopia-php/domains/tree/1.0.5" + "source": "https://github.com/utopia-php/domains/tree/1.0.2" }, - "time": "2026-03-03T09:20:50+00:00" + "time": "2026-02-25T08:18:25+00:00" }, { "name": "utopia-php/dsn", @@ -4517,16 +4517,16 @@ }, { "name": "utopia-php/migration", - "version": "1.6.3", + "version": "1.7.0", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "c2d016944cb029fa5ff822ceee704785a06ef289" + "reference": "97583ae502e40621ea91a71de19d053c5ae2e558" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/c2d016944cb029fa5ff822ceee704785a06ef289", - "reference": "c2d016944cb029fa5ff822ceee704785a06ef289", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/97583ae502e40621ea91a71de19d053c5ae2e558", + "reference": "97583ae502e40621ea91a71de19d053c5ae2e558", "shasum": "" }, "require": { @@ -4566,9 +4566,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/1.6.3" + "source": "https://github.com/utopia-php/migration/tree/1.7.0" }, - "time": "2026-03-04T07:08:22+00:00" + "time": "2026-03-10T06:36:27+00:00" }, { "name": "utopia-php/mongo", @@ -5215,16 +5215,16 @@ }, { "name": "utopia-php/vcs", - "version": "2.0.1", + "version": "2.0.0", "source": { "type": "git", "url": "https://github.com/utopia-php/vcs.git", - "reference": "92a1650824ba0c5e6a1bc46e622ac87c50a08920" + "reference": "058049326e04a2a0c2f0ce8ad00c7e84825aba14" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/vcs/zipball/92a1650824ba0c5e6a1bc46e622ac87c50a08920", - "reference": "92a1650824ba0c5e6a1bc46e622ac87c50a08920", + "url": "https://api.github.com/repos/utopia-php/vcs/zipball/058049326e04a2a0c2f0ce8ad00c7e84825aba14", + "reference": "058049326e04a2a0c2f0ce8ad00c7e84825aba14", "shasum": "" }, "require": { @@ -5258,9 +5258,9 @@ ], "support": { "issues": "https://github.com/utopia-php/vcs/issues", - "source": "https://github.com/utopia-php/vcs/tree/2.0.1" + "source": "https://github.com/utopia-php/vcs/tree/2.0.0" }, - "time": "2026-02-27T12:18:49+00:00" + "time": "2026-02-25T11:36:45+00:00" }, { "name": "utopia-php/websocket", @@ -5438,16 +5438,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "1.11.6", + "version": "1.11.1", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "f80e302d000cdc2f98b4bb5ff2fc3bd0bdff7b38" + "reference": "6ff411f26f2750eea05c7598c14bb3a2ada898cb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/f80e302d000cdc2f98b4bb5ff2fc3bd0bdff7b38", - "reference": "f80e302d000cdc2f98b4bb5ff2fc3bd0bdff7b38", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/6ff411f26f2750eea05c7598c14bb3a2ada898cb", + "reference": "6ff411f26f2750eea05c7598c14bb3a2ada898cb", "shasum": "" }, "require": { @@ -5483,22 +5483,22 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/1.11.6" + "source": "https://github.com/appwrite/sdk-generator/tree/1.11.1" }, - "time": "2026-03-09T07:12:51+00:00" + "time": "2026-02-25T07:15:19+00:00" }, { "name": "brianium/paratest", - "version": "v7.19.1", + "version": "v7.19.0", "source": { "type": "git", "url": "https://github.com/paratestphp/paratest.git", - "reference": "95b03194f4cdf5c83175ceead673e21cb66465e7" + "reference": "7c6c29af7c4b406b49ce0c6b0a3a81d3684474e6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/paratestphp/paratest/zipball/95b03194f4cdf5c83175ceead673e21cb66465e7", - "reference": "95b03194f4cdf5c83175ceead673e21cb66465e7", + "url": "https://api.github.com/repos/paratestphp/paratest/zipball/7c6c29af7c4b406b49ce0c6b0a3a81d3684474e6", + "reference": "7c6c29af7c4b406b49ce0c6b0a3a81d3684474e6", "shasum": "" }, "require": { @@ -5512,7 +5512,7 @@ "phpunit/php-code-coverage": "^12.5.3 || ^13.0.1", "phpunit/php-file-iterator": "^6.0.1 || ^7", "phpunit/php-timer": "^8 || ^9", - "phpunit/phpunit": "^12.5.14 || ^13.0.5", + "phpunit/phpunit": "^12.5.9 || ^13", "sebastian/environment": "^8.0.3 || ^9", "symfony/console": "^7.4.4 || ^8.0.4", "symfony/process": "^7.4.5 || ^8.0.5" @@ -5522,10 +5522,10 @@ "ext-pcntl": "*", "ext-pcov": "*", "ext-posix": "*", - "phpstan/phpstan": "^2.1.40", - "phpstan/phpstan-deprecation-rules": "^2.0.4", - "phpstan/phpstan-phpunit": "^2.0.16", - "phpstan/phpstan-strict-rules": "^2.0.10", + "phpstan/phpstan": "^2.1.38", + "phpstan/phpstan-deprecation-rules": "^2.0.3", + "phpstan/phpstan-phpunit": "^2.0.12", + "phpstan/phpstan-strict-rules": "^2.0.8", "symfony/filesystem": "^7.4.0 || ^8.0.1" }, "bin": [ @@ -5566,7 +5566,7 @@ ], "support": { "issues": "https://github.com/paratestphp/paratest/issues", - "source": "https://github.com/paratestphp/paratest/tree/v7.19.1" + "source": "https://github.com/paratestphp/paratest/tree/v7.19.0" }, "funding": [ { @@ -5578,7 +5578,7 @@ "type": "paypal" } ], - "time": "2026-02-25T14:53:45+00:00" + "time": "2026-02-06T10:53:26+00:00" }, { "name": "czproject/git-php", @@ -6398,16 +6398,16 @@ }, { "name": "phpbench/phpbench", - "version": "1.5.1", + "version": "1.4.3", "source": { "type": "git", "url": "https://github.com/phpbench/phpbench.git", - "reference": "9a28fd0833f11171b949843c6fd663eb69b6d14c" + "reference": "b641dde59d969ea42eed70a39f9b51950bc96878" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpbench/phpbench/zipball/9a28fd0833f11171b949843c6fd663eb69b6d14c", - "reference": "9a28fd0833f11171b949843c6fd663eb69b6d14c", + "url": "https://api.github.com/repos/phpbench/phpbench/zipball/b641dde59d969ea42eed70a39f9b51950bc96878", + "reference": "b641dde59d969ea42eed70a39f9b51950bc96878", "shasum": "" }, "require": { @@ -6418,7 +6418,7 @@ "ext-reflection": "*", "ext-spl": "*", "ext-tokenizer": "*", - "php": "^8.2", + "php": "^8.1", "phpbench/container": "^2.2", "psr/log": "^1.1 || ^2.0 || ^3.0", "seld/jsonlint": "^1.1", @@ -6438,9 +6438,8 @@ "phpstan/extension-installer": "^1.1", "phpstan/phpstan": "^1.0", "phpstan/phpstan-phpunit": "^1.0", - "phpunit/phpunit": "^11.5", + "phpunit/phpunit": "^10.4 || ^11.0", "rector/rector": "^1.2", - "sebastian/exporter": "^6.3.2", "symfony/error-handler": "^6.1 || ^7.0 || ^8.0", "symfony/var-dumper": "^6.1 || ^7.0 || ^8.0" }, @@ -6485,7 +6484,7 @@ ], "support": { "issues": "https://github.com/phpbench/phpbench/issues", - "source": "https://github.com/phpbench/phpbench/tree/1.5.1" + "source": "https://github.com/phpbench/phpbench/tree/1.4.3" }, "funding": [ { @@ -6493,15 +6492,15 @@ "type": "github" } ], - "time": "2026-03-05T08:18:58+00:00" + "time": "2025-11-06T19:07:31+00:00" }, { "name": "phpstan/phpstan", - "version": "1.12.33", + "version": "1.12.32", "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/37982d6fc7cbb746dda7773530cda557cdf119e1", - "reference": "37982d6fc7cbb746dda7773530cda557cdf119e1", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/2770dcdf5078d0b0d53f94317e06affe88419aa8", + "reference": "2770dcdf5078d0b0d53f94317e06affe88419aa8", "shasum": "" }, "require": { @@ -6546,7 +6545,7 @@ "type": "github" } ], - "time": "2026-02-28T20:30:03+00:00" + "time": "2025-09-30T10:16:31+00:00" }, { "name": "phpunit/php-code-coverage", @@ -8096,16 +8095,16 @@ }, { "name": "symfony/console", - "version": "v8.0.7", + "version": "v8.0.4", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "15ed9008a4ebe2d6a78e4937f74e0c13ef2e618a" + "reference": "ace03c4cf9805080ff40cbeec69fca180c339a3b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/15ed9008a4ebe2d6a78e4937f74e0c13ef2e618a", - "reference": "15ed9008a4ebe2d6a78e4937f74e0c13ef2e618a", + "url": "https://api.github.com/repos/symfony/console/zipball/ace03c4cf9805080ff40cbeec69fca180c339a3b", + "reference": "ace03c4cf9805080ff40cbeec69fca180c339a3b", "shasum": "" }, "require": { @@ -8162,7 +8161,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v8.0.7" + "source": "https://github.com/symfony/console/tree/v8.0.4" }, "funding": [ { @@ -8182,20 +8181,20 @@ "type": "tidelift" } ], - "time": "2026-03-06T14:06:22+00:00" + "time": "2026-01-13T13:06:50+00:00" }, { "name": "symfony/filesystem", - "version": "v8.0.6", + "version": "v8.0.1", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "7bf9162d7a0dff98d079b72948508fa48018a770" + "reference": "d937d400b980523dc9ee946bb69972b5e619058d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/7bf9162d7a0dff98d079b72948508fa48018a770", - "reference": "7bf9162d7a0dff98d079b72948508fa48018a770", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/d937d400b980523dc9ee946bb69972b5e619058d", + "reference": "d937d400b980523dc9ee946bb69972b5e619058d", "shasum": "" }, "require": { @@ -8232,7 +8231,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v8.0.6" + "source": "https://github.com/symfony/filesystem/tree/v8.0.1" }, "funding": [ { @@ -8252,20 +8251,20 @@ "type": "tidelift" } ], - "time": "2026-02-25T16:59:43+00:00" + "time": "2025-12-01T09:13:36+00:00" }, { "name": "symfony/finder", - "version": "v8.0.6", + "version": "v8.0.5", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "441404f09a54de6d1bd6ad219e088cdf4c91f97c" + "reference": "8bd576e97c67d45941365bf824e18dc8538e6eb0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/441404f09a54de6d1bd6ad219e088cdf4c91f97c", - "reference": "441404f09a54de6d1bd6ad219e088cdf4c91f97c", + "url": "https://api.github.com/repos/symfony/finder/zipball/8bd576e97c67d45941365bf824e18dc8538e6eb0", + "reference": "8bd576e97c67d45941365bf824e18dc8538e6eb0", "shasum": "" }, "require": { @@ -8300,7 +8299,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v8.0.6" + "source": "https://github.com/symfony/finder/tree/v8.0.5" }, "funding": [ { @@ -8320,7 +8319,7 @@ "type": "tidelift" } ], - "time": "2026-01-29T09:41:02+00:00" + "time": "2026-01-26T15:08:38+00:00" }, { "name": "symfony/options-resolver", @@ -8790,16 +8789,16 @@ }, { "name": "symfony/string", - "version": "v8.0.6", + "version": "v8.0.4", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "6c9e1108041b5dce21a9a4984b531c4923aa9ec4" + "reference": "758b372d6882506821ed666032e43020c4f57194" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/6c9e1108041b5dce21a9a4984b531c4923aa9ec4", - "reference": "6c9e1108041b5dce21a9a4984b531c4923aa9ec4", + "url": "https://api.github.com/repos/symfony/string/zipball/758b372d6882506821ed666032e43020c4f57194", + "reference": "758b372d6882506821ed666032e43020c4f57194", "shasum": "" }, "require": { @@ -8856,7 +8855,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v8.0.6" + "source": "https://github.com/symfony/string/tree/v8.0.4" }, "funding": [ { @@ -8876,7 +8875,7 @@ "type": "tidelift" } ], - "time": "2026-02-09T10:14:57+00:00" + "time": "2026-01-12T12:37:40+00:00" }, { "name": "textalk/websocket", @@ -9108,7 +9107,7 @@ ], "aliases": [], "minimum-stability": "dev", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": true, "prefer-lowest": false, "platform": { @@ -9132,5 +9131,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.9.0" + "plugin-api-version": "2.6.0" } diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index d87edaf788..c4cb9ce415 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -317,6 +317,16 @@ class Migrations extends Action 'sites.write', 'tokens.read', 'tokens.write', + 'providers.read', + 'providers.write', + 'topics.read', + 'topics.write', + 'subscribers.read', + 'subscribers.write', + 'messages.read', + 'messages.write', + 'targets.read', + 'targets.write', ] ]); diff --git a/src/Appwrite/Utopia/Response/Model/MigrationReport.php b/src/Appwrite/Utopia/Response/Model/MigrationReport.php index 7ebc22d22e..850e4b5ae9 100644 --- a/src/Appwrite/Utopia/Response/Model/MigrationReport.php +++ b/src/Appwrite/Utopia/Response/Model/MigrationReport.php @@ -59,6 +59,30 @@ class MigrationReport extends Model 'default' => 0, 'example' => 5, ]) + ->addRule(Resource::TYPE_PROVIDER, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of providers to be migrated.', + 'default' => 0, + 'example' => 5, + ]) + ->addRule(Resource::TYPE_TOPIC, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of topics to be migrated.', + 'default' => 0, + 'example' => 10, + ]) + ->addRule(Resource::TYPE_SUBSCRIBER, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of subscribers to be migrated.', + 'default' => 0, + 'example' => 100, + ]) + ->addRule(Resource::TYPE_MESSAGE, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of messages to be migrated.', + 'default' => 0, + 'example' => 50, + ]) ->addRule('size', [ 'type' => self::TYPE_INTEGER, 'description' => 'Size of files to be migrated in mb.', diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 606f9e8127..d5fe7753a4 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1694,4 +1694,842 @@ trait MigrationsBase 'x-appwrite-key' => $this->getProject()['apiKey'] ]); } + + /** + * Messaging + */ + public function testAppwriteMigrationMessagingProvider(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid', + 'apiKey' => 'my-apikey', + 'from' => 'migration@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $this->assertNotEmpty($provider['body']['$id']); + + $providerId = $provider['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertEquals([Resource::TYPE_PROVIDER], $result['resources']); + $this->assertArrayHasKey(Resource::TYPE_PROVIDER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_PROVIDER]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($providerId, $response['body']['$id']); + $this->assertEquals('Migration Sendgrid', $response['body']['name']); + $this->assertEquals('email', $response['body']['type']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingProviderSMTP(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/smtp', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration SMTP', + 'host' => 'smtp.test.com', + 'port' => 587, + 'from' => 'migration-smtp@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_PROVIDER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_PROVIDER]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($providerId, $response['body']['$id']); + $this->assertEquals('Migration SMTP', $response['body']['name']); + $this->assertEquals('email', $response['body']['type']); + $this->assertEquals('smtp', $response['body']['provider']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingProviderTwilio(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/twilio', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Twilio', + 'from' => '+15551234567', + 'accountSid' => 'test-account-sid', + 'authToken' => 'test-auth-token', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_PROVIDER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_PROVIDER]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($providerId, $response['body']['$id']); + $this->assertEquals('Migration Twilio', $response['body']['name']); + $this->assertEquals('sms', $response['body']['type']); + $this->assertEquals('twilio', $response['body']['provider']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingTopic(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Topic', + 'apiKey' => 'my-apikey', + 'from' => 'migration-topic@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $this->assertNotEmpty($topic['body']['$id']); + + $topicId = $topic['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_TOPIC, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_TOPIC]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($topicId, $response['body']['$id']); + $this->assertEquals('Migration Topic', $response['body']['name']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingSubscriber(): void + { + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-sub@test.com', + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $this->assertEquals(1, \count($user['body']['targets'])); + $targetId = $user['body']['targets'][0]['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Subscriber', + 'apiKey' => 'my-apikey', + 'from' => uniqid() . '-migration-sub@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Subscriber Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $subscriber = $this->client->call(Client::METHOD_POST, '/messaging/topics/' . $topicId . '/subscribers', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'subscriberId' => ID::unique(), + 'targetId' => $targetId, + ]); + + $this->assertEquals(201, $subscriber['headers']['status-code']); + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_SUBSCRIBER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($topicId, $response['body']['$id']); + $this->assertGreaterThanOrEqual(1, $response['body']['emailTotal']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingMessage(): void + { + $this->getDestinationProject(true); + + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-msg@test.com', + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $this->assertEquals(1, \count($user['body']['targets'])); + $targetId = $user['body']['targets'][0]['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Message', + 'apiKey' => 'my-apikey', + 'from' => 'migration-msg@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Message Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $message = $this->client->call(Client::METHOD_POST, '/messaging/messages/email', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'messageId' => ID::unique(), + 'targets' => [$targetId], + 'topics' => [$topicId], + 'subject' => 'Migration Test Email', + 'content' => 'This is a migration test email', + 'draft' => true, + ]); + + $this->assertEquals(201, $message['headers']['status-code']); + $this->assertNotEmpty($message['body']['$id']); + + $messageId = $message['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + Resource::TYPE_MESSAGE, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_MESSAGE, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_MESSAGE]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($messageId, $response['body']['$id']); + $this->assertEquals('draft', $response['body']['status']); + $this->assertEquals('Migration Test Email', $response['body']['data']['subject']); + $this->assertEquals('This is a migration test email', $response['body']['data']['content']); + $this->assertContains($topicId, $response['body']['topics']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingSmsMessage(): void + { + $this->getDestinationProject(true); + + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-sms@test.com', + 'phone' => '+1' . str_pad((string) rand(200000000, 999999999), 10, '0', STR_PAD_LEFT), + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $this->assertGreaterThanOrEqual(1, \count($user['body']['targets'])); + + $smsTarget = null; + foreach ($user['body']['targets'] as $target) { + if ($target['providerType'] === 'sms') { + $smsTarget = $target; + break; + } + } + $this->assertNotNull($smsTarget); + $targetId = $smsTarget['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/twilio', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Twilio SMS Msg', + 'from' => '+15559876543', + 'accountSid' => 'test-account-sid', + 'authToken' => 'test-auth-token', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration SMS Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $message = $this->client->call(Client::METHOD_POST, '/messaging/messages/sms', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'messageId' => ID::unique(), + 'targets' => [$targetId], + 'topics' => [$topicId], + 'content' => 'Migration SMS test content', + 'draft' => true, + ]); + + $this->assertEquals(201, $message['headers']['status-code']); + $messageId = $message['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + Resource::TYPE_MESSAGE, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_MESSAGE, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_MESSAGE]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($messageId, $response['body']['$id']); + $this->assertEquals('draft', $response['body']['status']); + $this->assertEquals('Migration SMS test content', $response['body']['data']['content']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingScheduledMessage(): void + { + $this->getDestinationProject(true); + + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-sched@test.com', + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $targetId = $user['body']['targets'][0]['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Scheduled', + 'apiKey' => 'my-apikey', + 'from' => 'migration-sched@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Scheduled Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $subscriber = $this->client->call(Client::METHOD_POST, '/messaging/topics/' . $topicId . '/subscribers', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'subscriberId' => ID::unique(), + 'targetId' => $targetId, + ]); + + $this->assertEquals(201, $subscriber['headers']['status-code']); + + // Create a scheduled message with a future date using topics only + // Direct targets use source IDs which won't resolve in the destination via API + $futureDate = (new \DateTime('+1 year'))->format(\DateTime::ATOM); + $message = $this->client->call(Client::METHOD_POST, '/messaging/messages/email', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'messageId' => ID::unique(), + 'topics' => [$topicId], + 'subject' => 'Migration Scheduled Email', + 'content' => 'This is a scheduled migration test email', + 'scheduledAt' => $futureDate, + ]); + + $this->assertEquals(201, $message['headers']['status-code']); + $messageId = $message['body']['$id']; + $this->assertEquals('scheduled', $message['body']['status']); + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + Resource::TYPE_MESSAGE, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_MESSAGE, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_MESSAGE]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($messageId, $response['body']['$id']); + $this->assertEquals('scheduled', $response['body']['status']); + $this->assertEquals('Migration Scheduled Email', $response['body']['data']['subject']); + $this->assertEquals( + (new \DateTime($futureDate))->getTimestamp(), + (new \DateTime($response['body']['scheduledAt']))->getTimestamp(), + ); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } } From f4b8992cdee45ab90729ad20a8170e6f374c3753 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 10 Mar 2026 12:19:18 +0000 Subject: [PATCH 16/57] Enable SMTP keep-alive to reuse connections across mail jobs Reduces job processing time by avoiding repeated TCP connect, TLS handshake, and SMTP AUTH on every email sent. Co-Authored-By: Claude Opus 4.6 --- app/init/registers.php | 1 + src/Appwrite/Platform/Workers/Mails.php | 1 + 2 files changed, 2 insertions(+) diff --git a/app/init/registers.php b/app/init/registers.php index 26a9329270..7b68c2af9a 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -420,6 +420,7 @@ $register->set('smtp', function () { $mail->Password = $password; $mail->SMTPSecure = System::getEnv('_APP_SMTP_SECURE', ''); $mail->SMTPAutoTLS = false; + $mail->SMTPKeepAlive = true; $mail->CharSet = 'UTF-8'; $mail->Timeout = 10; /* Connection timeout */ $mail->getSMTPInstance()->Timelimit = 30; /* Timeout for each individual SMTP command (e.g. HELO, EHLO, etc.) */ diff --git a/src/Appwrite/Platform/Workers/Mails.php b/src/Appwrite/Platform/Workers/Mails.php index 72f7cddd06..f144c58e1b 100644 --- a/src/Appwrite/Platform/Workers/Mails.php +++ b/src/Appwrite/Platform/Workers/Mails.php @@ -214,6 +214,7 @@ class Mails extends Action $mail->Password = $password; $mail->SMTPSecure = $smtp['secure']; $mail->SMTPAutoTLS = false; + $mail->SMTPKeepAlive = true; $mail->CharSet = 'UTF-8'; $mail->Timeout = 10; /* Connection timeout */ $mail->getSMTPInstance()->Timelimit = 30; /* Timeout for each individual SMTP command (e.g. HELO, EHLO, etc.) */ From 1be142e4becf9aaf1278853f9e8da9ed4791da2f Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Tue, 10 Mar 2026 12:33:30 +0000 Subject: [PATCH 17/57] test: use database fix branch for preserveDates datetime format --- composer.json | 2 +- composer.lock | 27 ++++++++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/composer.json b/composer.json index abe51e500b..c2d2749a74 100644 --- a/composer.json +++ b/composer.json @@ -58,7 +58,7 @@ "utopia-php/compression": "0.1.*", "utopia-php/config": "1.*", "utopia-php/console": "0.1.*", - "utopia-php/database": "5.*", + "utopia-php/database": "dev-fix-preserve-dates-datetime-format as 5.3.7", "utopia-php/detector": "0.2.*", "utopia-php/domains": "1.*", "utopia-php/emails": "0.6.*", diff --git a/composer.lock b/composer.lock index 738db8f287..3caf85b8cb 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b99693284208ff3d006260a089a4f7b9", + "content-hash": "09f5b42e8589d72026a0b3731fc4f2af", "packages": [ { "name": "adhocore/jwt", @@ -3850,16 +3850,16 @@ }, { "name": "utopia-php/database", - "version": "5.3.7", + "version": "dev-fix-preserve-dates-datetime-format", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "438cc82af2981cd41ad200dd9b0df5bf00f3046a" + "reference": "49aa3181862c1370c17c2156a3b3228dca48df1e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/438cc82af2981cd41ad200dd9b0df5bf00f3046a", - "reference": "438cc82af2981cd41ad200dd9b0df5bf00f3046a", + "url": "https://api.github.com/repos/utopia-php/database/zipball/49aa3181862c1370c17c2156a3b3228dca48df1e", + "reference": "49aa3181862c1370c17c2156a3b3228dca48df1e", "shasum": "" }, "require": { @@ -3902,9 +3902,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/5.3.7" + "source": "https://github.com/utopia-php/database/tree/fix-preserve-dates-datetime-format" }, - "time": "2026-03-09T04:28:56+00:00" + "time": "2026-03-10T12:28:50+00:00" }, { "name": "utopia-php/detector", @@ -9105,9 +9105,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-fix-preserve-dates-datetime-format", + "alias": "5.3.7", + "alias_normalized": "5.3.7.0" + } + ], "minimum-stability": "dev", - "stability-flags": [], + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": true, "prefer-lowest": false, "platform": { From 33cafdfc0e8e036ddc30d4cc4ba3a20590feda72 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 10 Mar 2026 22:17:26 +0000 Subject: [PATCH 18/57] Initial plan From 1113bffb78ebf8c4caaf0e54dabf463a1951efbb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 10 Mar 2026 22:19:06 +0000 Subject: [PATCH 19/57] fix: use message timestamp for receivedAt in StatsUsage worker Co-authored-by: stnguyen90 <1477010+stnguyen90@users.noreply.github.com> --- src/Appwrite/Platform/Workers/StatsUsage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 018c192647..801790f188 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -160,7 +160,7 @@ class StatsUsage extends Action } $this->stats[$projectId]['project'] = $project; - $this->stats[$projectId]['receivedAt'] = DateTime::now(); + $this->stats[$projectId]['receivedAt'] = DateTime::format(new \DateTime('@' . $message->getTimestamp())); foreach ($payload['metrics'] ?? [] as $metric) { $this->keys++; if (!isset($this->stats[$projectId]['keys'][$metric['key']])) { From 4bf3c72196498987bdc1aab54ba117622c370c9b Mon Sep 17 00:00:00 2001 From: eldadfux Date: Wed, 11 Mar 2026 06:25:08 +0100 Subject: [PATCH 20/57] fix: allow users to update phone number to empty without causing duplicate errors --- app/controllers/api/users.php | 13 +++--- tests/e2e/Services/Users/UsersBase.php | 56 +++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index d0e5e19a51..9d04018b10 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1560,12 +1560,15 @@ Http::patch('/v1/users/:userId/phone') $oldPhone = $user->getAttribute('phone'); + // Store null instead of empty string so unique constraint allows multiple users without phone + $phoneValue = $number !== '' ? $number : null; + $user - ->setAttribute('phone', $number) + ->setAttribute('phone', $phoneValue) ->setAttribute('phoneVerification', false) ; - if (\strlen($number) !== 0) { + if ($number !== '') { $target = $dbForProject->findOne('targets', [ Query::equal('identifier', [$number]), ]); @@ -1577,7 +1580,7 @@ Http::patch('/v1/users/:userId/phone') try { $user = $dbForProject->updateDocument('users', $user->getId(), new Document([ - 'phone' => $user->getAttribute('phone'), + 'phone' => $phoneValue, 'phoneVerification' => $user->getAttribute('phoneVerification'), ])); /** @@ -1586,14 +1589,14 @@ Http::patch('/v1/users/:userId/phone') $oldTarget = $user->find('identifier', $oldPhone, 'targets'); if ($oldTarget instanceof Document && !$oldTarget->isEmpty()) { - if (\strlen($number) !== 0) { + if ($number !== '') { $dbForProject->updateDocument('targets', $oldTarget->getId(), new Document(['identifier' => $number])); $oldTarget->setAttribute('identifier', $number); } else { $dbForProject->deleteDocument('targets', $oldTarget->getId()); } } else { - if (\strlen($number) !== 0) { + if ($number !== '') { $target = $dbForProject->createDocument('targets', new Document([ '$permissions' => [ Permission::read(Role::user($user->getId())), diff --git a/tests/e2e/Services/Users/UsersBase.php b/tests/e2e/Services/Users/UsersBase.php index 5c7b289722..866ee591a2 100644 --- a/tests/e2e/Services/Users/UsersBase.php +++ b/tests/e2e/Services/Users/UsersBase.php @@ -1596,7 +1596,7 @@ trait UsersBase ]); $this->assertEquals($user['headers']['status-code'], 200); - $this->assertEquals($user['body']['phone'], $updatedNumber); + $this->assertEmpty($user['body']['phone'] ?? ''); $user = $this->client->call(Client::METHOD_GET, '/users/' . $data['userId'], array_merge([ 'content-type' => 'application/json', @@ -1604,7 +1604,7 @@ trait UsersBase ], $this->getHeaders())); $this->assertEquals($user['headers']['status-code'], 200); - $this->assertEquals($user['body']['phone'], $updatedNumber); + $this->assertEmpty($user['body']['phone'] ?? ''); $updatedNumber = "+910000000000"; //dummy number $user = $this->client->call(Client::METHOD_PATCH, '/users/' . $data['userId'] . '/phone', array_merge([ @@ -1648,6 +1648,58 @@ trait UsersBase static::$userNumberUpdated = true; } + public function testUpdateTwoUsersPhoneToEmpty(): void + { + $projectId = $this->getProject()['$id']; + $headers = array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders()); + + // Create two users with distinct valid phone numbers + $user1 = $this->client->call(Client::METHOD_POST, '/users', $headers, [ + 'userId' => ID::unique(), + 'email' => 'user1-phone-empty-test@appwrite.io', + 'password' => 'password', + 'name' => 'User One', + 'phone' => '+16175551201', + ]); + $this->assertEquals(201, $user1['headers']['status-code']); + $this->assertEquals('+16175551201', $user1['body']['phone']); + + $user2 = $this->client->call(Client::METHOD_POST, '/users', $headers, [ + 'userId' => ID::unique(), + 'email' => 'user2-phone-empty-test@appwrite.io', + 'password' => 'password', + 'name' => 'User Two', + 'phone' => '+16175551202', + ]); + $this->assertEquals(201, $user2['headers']['status-code']); + $this->assertEquals('+16175551202', $user2['body']['phone']); + + // Update first user's phone to empty - must succeed + $response1 = $this->client->call(Client::METHOD_PATCH, '/users/' . $user1['body']['$id'] . '/phone', $headers, [ + 'number' => '', + ]); + $this->assertEquals(200, $response1['headers']['status-code'], 'First user phone should update to empty'); + $this->assertEmpty($response1['body']['phone'] ?? ''); + + // Update second user's phone to empty - must succeed (would fail with duplicate if empty was stored as '') + $response2 = $this->client->call(Client::METHOD_PATCH, '/users/' . $user2['body']['$id'] . '/phone', $headers, [ + 'number' => '', + ]); + $this->assertEquals(200, $response2['headers']['status-code'], 'Second user phone should update to empty without duplicate error'); + $this->assertEmpty($response2['body']['phone'] ?? ''); + + // Verify both users have empty phone via GET + $get1 = $this->client->call(Client::METHOD_GET, '/users/' . $user1['body']['$id'], $headers); + $get2 = $this->client->call(Client::METHOD_GET, '/users/' . $user2['body']['$id'], $headers); + $this->assertEquals(200, $get1['headers']['status-code']); + $this->assertEquals(200, $get2['headers']['status-code']); + $this->assertEmpty($get1['body']['phone'] ?? ''); + $this->assertEmpty($get2['body']['phone'] ?? ''); + } + public function testUpdateUserNumberSearch(): void { $data = $this->ensureUserNumberUpdated(); From 0ebda89adf98c475d3ea35157be4a4a60a621aa5 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 11 Mar 2026 05:56:44 +0000 Subject: [PATCH 21/57] update utopia-php/database to released 5.3.8 --- composer.json | 2 +- composer.lock | 29 ++++++++++------------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/composer.json b/composer.json index c2d2749a74..abe51e500b 100644 --- a/composer.json +++ b/composer.json @@ -58,7 +58,7 @@ "utopia-php/compression": "0.1.*", "utopia-php/config": "1.*", "utopia-php/console": "0.1.*", - "utopia-php/database": "dev-fix-preserve-dates-datetime-format as 5.3.7", + "utopia-php/database": "5.*", "utopia-php/detector": "0.2.*", "utopia-php/domains": "1.*", "utopia-php/emails": "0.6.*", diff --git a/composer.lock b/composer.lock index 3caf85b8cb..0e25d3bc5f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "09f5b42e8589d72026a0b3731fc4f2af", + "content-hash": "b99693284208ff3d006260a089a4f7b9", "packages": [ { "name": "adhocore/jwt", @@ -3850,16 +3850,16 @@ }, { "name": "utopia-php/database", - "version": "dev-fix-preserve-dates-datetime-format", + "version": "5.3.8", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "49aa3181862c1370c17c2156a3b3228dca48df1e" + "reference": "4920bb60afb98d4bd81f4d331765716ae1d40255" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/49aa3181862c1370c17c2156a3b3228dca48df1e", - "reference": "49aa3181862c1370c17c2156a3b3228dca48df1e", + "url": "https://api.github.com/repos/utopia-php/database/zipball/4920bb60afb98d4bd81f4d331765716ae1d40255", + "reference": "4920bb60afb98d4bd81f4d331765716ae1d40255", "shasum": "" }, "require": { @@ -3902,9 +3902,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/fix-preserve-dates-datetime-format" + "source": "https://github.com/utopia-php/database/tree/5.3.8" }, - "time": "2026-03-10T12:28:50+00:00" + "time": "2026-03-11T01:03:34+00:00" }, { "name": "utopia-php/detector", @@ -9105,18 +9105,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-fix-preserve-dates-datetime-format", - "alias": "5.3.7", - "alias_normalized": "5.3.7.0" - } - ], + "aliases": [], "minimum-stability": "dev", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": {}, "prefer-stable": true, "prefer-lowest": false, "platform": { @@ -9140,5 +9131,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.9.0" } From 7d15d7e1f57dc3bc52b0c8f6b9d41b4cab12b60e Mon Sep 17 00:00:00 2001 From: Hemachandar <132386067+hmacr@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:15:50 +0530 Subject: [PATCH 22/57] Rename time-travel task (#11523) * Rename time-travel task * rename --- Dockerfile | 2 +- bin/{time-travel => task-time-travel} | 0 tests/e2e/Services/Functions/FunctionsConsoleClientTest.php | 2 +- tests/e2e/Services/Sites/SitesConsoleClientTest.php | 4 ++-- 4 files changed, 4 insertions(+), 4 deletions(-) rename bin/{time-travel => task-time-travel} (100%) diff --git a/Dockerfile b/Dockerfile index 266d4501d0..210c2bc3d9 100755 --- a/Dockerfile +++ b/Dockerfile @@ -70,7 +70,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/sdks && \ chmod +x /usr/local/bin/specs && \ chmod +x /usr/local/bin/ssl && \ - chmod +x /usr/local/bin/time-travel && \ + chmod +x /usr/local/bin/task-time-travel && \ chmod +x /usr/local/bin/screenshot && \ chmod +x /usr/local/bin/test && \ chmod +x /usr/local/bin/upgrade && \ diff --git a/bin/time-travel b/bin/task-time-travel similarity index 100% rename from bin/time-travel rename to bin/task-time-travel diff --git a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php index cc18d14a8e..06044d9984 100644 --- a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php @@ -686,7 +686,7 @@ class FunctionsConsoleClientTest extends Scope $stdout = ''; $stderr = ''; - $code = Console::execute("docker exec appwrite time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); + $code = Console::execute("docker exec appwrite task-time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); $this->assertSame(0, $code, "Time-travel command failed with code $code: $stderr ($stdout)"); $stdout = ''; diff --git a/tests/e2e/Services/Sites/SitesConsoleClientTest.php b/tests/e2e/Services/Sites/SitesConsoleClientTest.php index 2a94dded5f..2e0e1a892d 100644 --- a/tests/e2e/Services/Sites/SitesConsoleClientTest.php +++ b/tests/e2e/Services/Sites/SitesConsoleClientTest.php @@ -180,12 +180,12 @@ class SitesConsoleClientTest extends Scope $stdout = ''; $stderr = ''; - $code = Console::execute("docker exec appwrite-task-maintenance time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); + $code = Console::execute("docker exec appwrite task-time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); $this->assertSame(0, $code, "Time-travel command failed with code $code: $stderr ($stdout)"); $stdout = ''; $stderr = ''; - $code = Console::execute("docker exec appwrite-task-maintenance maintenance --type=trigger", '', $stdout, $stderr); + $code = Console::execute("docker exec appwrite maintenance --type=trigger", '', $stdout, $stderr); $this->assertSame(0, $code, "Maintenance command failed with code $code: $stderr ($stdout)"); $this->assertEventually(function () use ($siteId) { From e557a5fc6e47293a82d5257e3d2672e37c946e1b Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 19:41:49 +0000 Subject: [PATCH 23/57] Add Docker Hub login and parallel image pulls to CI Login to Docker Hub in all test jobs to avoid rate limits, add `docker compose pull --quiet` to parallelize image downloads before `docker compose up`, and replace sed-based .env overrides with native GitHub Actions env fields. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 74 +++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0cff6288e2..8e937b261a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,6 +53,12 @@ jobs: with: submodules: recursive + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -97,10 +103,17 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -142,10 +155,17 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -249,12 +269,19 @@ jobs: echo "_APP_DB_PORT=5432" >> $GITHUB_ENV fi + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 env: _APP_BROWSER_HOST: http://invalid-browser/v1 run: | docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -356,10 +383,17 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -434,11 +468,19 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 + env: + _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -499,11 +541,19 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 + env: + _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -562,11 +612,19 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 + env: + _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -636,11 +694,19 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 + env: + _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." From 4e1b710503ba1ee880842a394645b1ea57aba8ee Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 19:59:23 +0000 Subject: [PATCH 24/57] Fix Docker Hub login credentials to match repo config Use `vars.DOCKERHUB_USERNAME` and `secrets.DOCKERHUB_TOKEN` to match the existing publish and release workflows. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8e937b261a..55254ec35a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -56,8 +56,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -106,8 +106,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -158,8 +158,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -272,8 +272,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -386,8 +386,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -471,8 +471,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -544,8 +544,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -615,8 +615,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -697,8 +697,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 From 0b416e44753a0a7b00c869ae43e69b0cc8145876 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:01:25 +0000 Subject: [PATCH 25/57] Skip locally-built images during docker compose pull Add --ignore-buildable flag so docker compose pull skips images with build directives (appwrite-dev) instead of trying to pull them from Docker Hub. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 55254ec35a..982e19f832 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -113,7 +113,7 @@ jobs: timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -165,7 +165,7 @@ jobs: timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -281,7 +281,7 @@ jobs: _APP_BROWSER_HOST: http://invalid-browser/v1 run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -393,7 +393,7 @@ jobs: timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -480,7 +480,7 @@ jobs: _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -553,7 +553,7 @@ jobs: _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -624,7 +624,7 @@ jobs: _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -706,7 +706,7 @@ jobs: _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." From d22642590f76a3903ccca6a45be98305ea3a5f3b Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:07:13 +0000 Subject: [PATCH 26/57] Add build directive to all appwrite-dev services for --ignore-buildable Add x-build YAML anchor and apply it to all services using the appwrite-dev image so docker compose pull --ignore-buildable correctly skips them instead of trying to pull from Docker Hub. Co-Authored-By: Claude Opus 4.6 --- docker-compose.yml | 60 ++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4a38757737..c744a9c5a5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,15 @@ x-logging: &x-logging max-file: "5" max-size: "10m" +x-build: &x-build + build: + context: . + target: development + args: + DEBUG: false + TESTING: true + VERSION: dev + services: traefik: image: traefik:3.6 @@ -50,15 +59,8 @@ services: appwrite: container_name: appwrite - <<: *x-logging + <<: [*x-logging, *x-build] image: appwrite-dev - build: - context: . - target: development - args: - DEBUG: false - TESTING: true - VERSION: dev ports: - 9501:80 networks: @@ -260,7 +262,7 @@ services: appwrite-realtime: entrypoint: realtime - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-realtime image: appwrite-dev restart: unless-stopped @@ -312,7 +314,7 @@ services: appwrite-worker-audits: entrypoint: worker-audits - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-audits image: appwrite-dev networks: @@ -343,7 +345,7 @@ services: appwrite-worker-webhooks: entrypoint: worker-webhooks - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-webhooks image: appwrite-dev networks: @@ -378,7 +380,7 @@ services: appwrite-worker-deletes: entrypoint: worker-deletes - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-deletes image: appwrite-dev networks: @@ -443,7 +445,7 @@ services: appwrite-worker-databases: entrypoint: worker-databases - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-databases image: appwrite-dev networks: @@ -476,7 +478,7 @@ services: appwrite-worker-builds: entrypoint: worker-builds - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-builds image: appwrite-dev networks: @@ -551,7 +553,7 @@ services: appwrite-worker-screenshots: entrypoint: worker-screenshots - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-screenshots image: appwrite-dev networks: @@ -614,7 +616,7 @@ services: appwrite-worker-certificates: entrypoint: worker-certificates - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-certificates image: appwrite-dev networks: @@ -656,7 +658,7 @@ services: appwrite-worker-executions: entrypoint: worker-executions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-executions image: appwrite-dev networks: @@ -686,7 +688,7 @@ services: appwrite-worker-functions: entrypoint: worker-functions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-functions image: appwrite-dev networks: @@ -730,7 +732,7 @@ services: appwrite-worker-mails: entrypoint: worker-mails - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-mails image: appwrite-dev networks: @@ -772,7 +774,7 @@ services: appwrite-worker-messaging: entrypoint: worker-messaging - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-messaging restart: unless-stopped image: appwrite-dev @@ -829,7 +831,7 @@ services: appwrite-worker-migrations: entrypoint: worker-migrations - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-migrations restart: unless-stopped image: appwrite-dev @@ -874,7 +876,7 @@ services: appwrite-task-maintenance: entrypoint: maintenance - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-maintenance image: appwrite-dev networks: @@ -920,7 +922,7 @@ services: appwrite-task-interval: entrypoint: interval - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-interval image: appwrite-dev networks: @@ -961,7 +963,7 @@ services: appwrite-task-stats-resources: container_name: appwrite-task-stats-resources entrypoint: stats-resources - <<: *x-logging + <<: [*x-logging, *x-build] image: appwrite-dev networks: - appwrite @@ -993,7 +995,7 @@ services: appwrite-worker-stats-resources: entrypoint: worker-stats-resources - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-stats-resources image: appwrite-dev networks: @@ -1026,7 +1028,7 @@ services: appwrite-worker-stats-usage: entrypoint: worker-stats-usage - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-stats-usage image: appwrite-dev networks: @@ -1059,7 +1061,7 @@ services: appwrite-task-scheduler-functions: entrypoint: schedule-functions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-scheduler-functions image: appwrite-dev networks: @@ -1089,7 +1091,7 @@ services: appwrite-task-scheduler-executions: entrypoint: schedule-executions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-scheduler-executions image: appwrite-dev networks: @@ -1118,7 +1120,7 @@ services: appwrite-task-scheduler-messages: entrypoint: schedule-messages - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-scheduler-messages image: appwrite-dev networks: From 6aac7ea6ad9f25ed2e1fc15c1ace0bc682b7b169 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:17:03 +0000 Subject: [PATCH 27/57] Update GitHub Actions to latest versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - actions/cache: v4 → v5 - docker/setup-buildx-action: v3 → v4 Fixes Node.js 20 deprecation warnings. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 982e19f832..3ccf652ccb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -60,7 +60,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Build Appwrite uses: docker/build-push-action@v6 @@ -79,7 +79,7 @@ jobs: VERSION=dev - name: Cache Docker Image - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -97,7 +97,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -149,7 +149,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -243,7 +243,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -377,7 +377,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -462,7 +462,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -535,7 +535,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -606,7 +606,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -688,7 +688,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar From 3613a645d049c5c2705c7412837a91b3f24fd938 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:19:04 +0000 Subject: [PATCH 28/57] Remove php-retry action and run test commands directly Replace itznotabug/php-retry with native run steps and timeout-minutes. Also remove pull-requests: write permission that was only needed by php-retry to post PR comments. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 108 ++++++++---------------------------- 1 file changed, 22 insertions(+), 86 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3ccf652ccb..d252c72a42 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -90,7 +90,6 @@ jobs: needs: setup permissions: contents: read - pull-requests: write steps: - name: checkout @@ -124,18 +123,11 @@ jobs: run: docker compose exec -T appwrite vars - name: Run Unit Tests - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/unit - command: >- - docker compose exec - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/unit + timeout-minutes: 15 + run: >- + docker compose exec + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/unit e2e_general_test: name: E2E General Test @@ -143,7 +135,6 @@ jobs: needs: setup permissions: contents: read - pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -181,18 +172,11 @@ jobs: done - name: Run General Tests - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/General - command: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/General --debug + timeout-minutes: 15 + run: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/General --debug - name: Failure Logs if: failure() @@ -206,7 +190,6 @@ jobs: needs: setup permissions: contents: read - pull-requests: write strategy: fail-fast: false matrix: @@ -297,15 +280,8 @@ jobs: done - name: Run ${{ matrix.service }} tests with Project table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 20 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/${{ matrix.service }} - command: | + timeout-minutes: 20 + run: | echo "Using project tables" SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" @@ -339,7 +315,6 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read - pull-requests: write strategy: fail-fast: false matrix: @@ -409,15 +384,8 @@ jobs: done - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 20 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/${{ matrix.service }} - command: | + timeout-minutes: 20 + run: | if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then echo "Using shared tables V1" export _APP_DATABASE_SHARED_TABLES=database_db_main @@ -456,7 +424,6 @@ jobs: needs: setup permissions: contents: read - pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -488,15 +455,8 @@ jobs: done - name: Run Projects tests in dedicated table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Projects - command: | + timeout-minutes: 15 + run: | echo "Using project tables" export _APP_DATABASE_SHARED_TABLES= export _APP_DATABASE_SHARED_TABLES_V1= @@ -522,7 +482,6 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read - pull-requests: write strategy: fail-fast: false matrix: @@ -561,15 +520,8 @@ jobs: done - name: Run Projects tests in ${{ matrix.tables-mode }} table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Projects - command: | + timeout-minutes: 15 + run: | if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then echo "Using shared tables V1" export _APP_DATABASE_SHARED_TABLES=database_db_main @@ -600,7 +552,6 @@ jobs: needs: setup permissions: contents: read - pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -640,15 +591,8 @@ jobs: done - name: Run Site tests with browser connected in dedicated table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Sites - command: | + timeout-minutes: 15 + run: | echo "Keeping original value of _APP_BROWSER_HOST" echo "Using project tables" export _APP_DATABASE_SHARED_TABLES= @@ -675,7 +619,6 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read - pull-requests: write strategy: fail-fast: false matrix: @@ -722,15 +665,8 @@ jobs: done - name: Run Site tests with browser connected in ${{ matrix.tables-mode }} table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Sites - command: | + timeout-minutes: 15 + run: | echo "Keeping original value of _APP_BROWSER_HOST" if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then echo "Using shared tables V1" From dc9a1c03d18bc6f3ef5f1452f9ab89ef00e9d0cb Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:20:23 +0000 Subject: [PATCH 29/57] Replace shell exports with GitHub Actions env fields Use step-level env: fields and GHA expressions for conditional values instead of shell export statements and if/elif blocks. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 103 ++++++++++++++---------------------- 1 file changed, 39 insertions(+), 64 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d252c72a42..e78a23575c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -385,17 +385,10 @@ jobs: - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode timeout-minutes: 20 + env: + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | - if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then - echo "Using shared tables V1" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1=database_db_main - elif [ "${{ matrix.tables-mode }}" == "Shared V2" ]; then - echo "Using shared tables V2" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1= - fi - SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" # Services that rely on sequential test method execution (shared static state) @@ -456,16 +449,15 @@ jobs: - name: Run Projects tests in dedicated table mode timeout-minutes: 15 - run: | - echo "Using project tables" - export _APP_DATABASE_SHARED_TABLES= - export _APP_DATABASE_SHARED_TABLES_V1= - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled + env: + _APP_DATABASE_SHARED_TABLES: "" + _APP_DATABASE_SHARED_TABLES_V1: "" + run: >- + docker compose exec -T + -e _APP_DATABASE_SHARED_TABLES + -e _APP_DATABASE_SHARED_TABLES_V1 + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled - name: Failure Logs if: failure() @@ -521,22 +513,15 @@ jobs: - name: Run Projects tests in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 - run: | - if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then - echo "Using shared tables V1" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1=database_db_main - elif [ "${{ matrix.tables-mode }}" == "Shared V2" ]; then - echo "Using shared tables V2" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1= - fi - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled + env: + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} + run: >- + docker compose exec -T + -e _APP_DATABASE_SHARED_TABLES + -e _APP_DATABASE_SHARED_TABLES_V1 + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled - name: Failure Logs if: failure() @@ -592,17 +577,15 @@ jobs: - name: Run Site tests with browser connected in dedicated table mode timeout-minutes: 15 - run: | - echo "Keeping original value of _APP_BROWSER_HOST" - echo "Using project tables" - export _APP_DATABASE_SHARED_TABLES= - export _APP_DATABASE_SHARED_TABLES_V1= - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots + env: + _APP_DATABASE_SHARED_TABLES: "" + _APP_DATABASE_SHARED_TABLES_V1: "" + run: >- + docker compose exec -T + -e _APP_DATABASE_SHARED_TABLES + -e _APP_DATABASE_SHARED_TABLES_V1 + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots - name: Failure Logs if: failure() @@ -666,23 +649,15 @@ jobs: - name: Run Site tests with browser connected in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 - run: | - echo "Keeping original value of _APP_BROWSER_HOST" - if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then - echo "Using shared tables V1" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1=database_db_main - elif [ "${{ matrix.tables-mode }}" == "Shared V2" ]; then - echo "Using shared tables V2" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1= - fi - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots + env: + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} + run: >- + docker compose exec -T + -e _APP_DATABASE_SHARED_TABLES + -e _APP_DATABASE_SHARED_TABLES_V1 + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots - name: Failure Logs if: failure() From d30df31b826f24962f27089feefc99fbdb503818 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:25:47 +0000 Subject: [PATCH 30/57] Move compose-time env vars to docker compose up step _APP_DATABASE_SHARED_TABLES and _APP_DATABASE_SHARED_TABLES_V1 are read by the server at boot time via System::getEnv(), not by the test runner. Passing them via docker compose exec -e had no effect on the already-running Swoole server. Move them to the Load and Start Appwrite step so they're set at docker compose up time. Keep _APP_E2E_RESPONSE_FORMAT on exec since it's read by the test runner process (tests/e2e/Scopes/Scope.php). Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 46 +++++++++++-------------------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e78a23575c..201a47c8ba 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -262,6 +262,8 @@ jobs: timeout-minutes: 3 env: _APP_BROWSER_HOST: http://invalid-browser/v1 + _APP_DATABASE_SHARED_TABLES: "" + _APP_DATABASE_SHARED_TABLES_V1: "" run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -282,7 +284,6 @@ jobs: - name: Run ${{ matrix.service }} tests with Project table mode timeout-minutes: 20 run: | - echo "Using project tables" SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" # Services that rely on sequential test method execution (shared static state) @@ -291,14 +292,7 @@ jobs: Databases|Functions|Realtime) FUNCTIONAL_FLAG="" ;; esac - echo "Running with paratest (parallel) for: ${{ matrix.service }} ${FUNCTIONAL_FLAG:+(functional)}" docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES="" \ - -e _APP_DATABASE_SHARED_TABLES_V1="" \ - -e _APP_DB_ADAPTER="${{ env._APP_DB_ADAPTER }}" \ - -e _APP_DB_HOST="${{ env._APP_DB_HOST }}" \ - -e _APP_DB_PORT="${{ env._APP_DB_PORT }}" \ - -e _APP_DB_SCHEMA=appwrite \ -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ appwrite vendor/bin/paratest --processes $(nproc) $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --exclude-group ciIgnore --log-junit tests/e2e/Services/${{ matrix.service }}/junit.xml @@ -366,6 +360,9 @@ jobs: - name: Load and Start Appwrite timeout-minutes: 3 + env: + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -385,9 +382,6 @@ jobs: - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode timeout-minutes: 20 - env: - _APP_DATABASE_SHARED_TABLES: database_db_main - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" @@ -398,8 +392,6 @@ jobs: esac docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ appwrite vendor/bin/paratest --processes $(nproc) $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --exclude-group ciIgnore --log-junit tests/e2e/Services/${{ matrix.service }}/junit.xml @@ -438,6 +430,8 @@ jobs: timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled + _APP_DATABASE_SHARED_TABLES: "" + _APP_DATABASE_SHARED_TABLES_V1: "" run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -449,13 +443,8 @@ jobs: - name: Run Projects tests in dedicated table mode timeout-minutes: 15 - env: - _APP_DATABASE_SHARED_TABLES: "" - _APP_DATABASE_SHARED_TABLES_V1: "" run: >- docker compose exec -T - -e _APP_DATABASE_SHARED_TABLES - -e _APP_DATABASE_SHARED_TABLES_V1 -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled @@ -502,6 +491,8 @@ jobs: timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -513,13 +504,8 @@ jobs: - name: Run Projects tests in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 - env: - _APP_DATABASE_SHARED_TABLES: database_db_main - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: >- docker compose exec -T - -e _APP_DATABASE_SHARED_TABLES - -e _APP_DATABASE_SHARED_TABLES_V1 -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled @@ -558,6 +544,8 @@ jobs: timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled + _APP_DATABASE_SHARED_TABLES: "" + _APP_DATABASE_SHARED_TABLES_V1: "" run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -577,13 +565,8 @@ jobs: - name: Run Site tests with browser connected in dedicated table mode timeout-minutes: 15 - env: - _APP_DATABASE_SHARED_TABLES: "" - _APP_DATABASE_SHARED_TABLES_V1: "" run: >- docker compose exec -T - -e _APP_DATABASE_SHARED_TABLES - -e _APP_DATABASE_SHARED_TABLES_V1 -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots @@ -630,6 +613,8 @@ jobs: timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -649,13 +634,8 @@ jobs: - name: Run Site tests with browser connected in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 - env: - _APP_DATABASE_SHARED_TABLES: database_db_main - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: >- docker compose exec -T - -e _APP_DATABASE_SHARED_TABLES - -e _APP_DATABASE_SHARED_TABLES_V1 -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots From 534dc55f17780a49b7d800316f7bbcb67ad5a9e7 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:35:42 +0000 Subject: [PATCH 31/57] Remove unnecessary abuse enabled from screenshot tests The screenshot tests have no abuse-related code. Abuse was only enabled on these jobs as a side effect of the original sed command applying to all jobs below e2e_service_test. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 201a47c8ba..721c25cadf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -543,7 +543,6 @@ jobs: - name: Load and Start Appwrite timeout-minutes: 3 env: - _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: "" _APP_DATABASE_SHARED_TABLES_V1: "" run: | @@ -612,7 +611,6 @@ jobs: - name: Load and Start Appwrite timeout-minutes: 3 env: - _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | From ecca0d80363fcd88030989d22c6a6729241f15e0 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:48:39 +0000 Subject: [PATCH 32/57] Increase Load and Start Appwrite timeout from 3 to 10 minutes The 3-minute timeout was too tight with the added docker compose pull step for downloading third-party images. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 721c25cadf..5c20ca094d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -109,7 +109,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -153,7 +153,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -259,7 +259,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_BROWSER_HOST: http://invalid-browser/v1 _APP_DATABASE_SHARED_TABLES: "" @@ -359,7 +359,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} @@ -427,7 +427,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: "" @@ -488,7 +488,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: database_db_main @@ -541,7 +541,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_DATABASE_SHARED_TABLES: "" _APP_DATABASE_SHARED_TABLES_V1: "" @@ -609,7 +609,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} From 0bbcc3f570455e241776460669a5683d74e3676a Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 21:06:44 +0000 Subject: [PATCH 33/57] Move dev tools to docker-compose.override.yml Move adminer, redis-insight, mongo-express, and graphql-explorer to an override file. CI sets COMPOSE_FILE=docker-compose.yml explicitly so these are excluded from test runs, reducing the number of images to pull from 14 to 10. Local docker compose auto-loads both files so dev workflow is unchanged. Co-Authored-By: Claude Opus 4.6 --- docker-compose.override.yml | 144 ++++++++++++++++++++++++++++++++++++ docker-compose.yml | 142 +---------------------------------- 2 files changed, 147 insertions(+), 139 deletions(-) create mode 100644 docker-compose.override.yml diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000000..29182f01d2 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,144 @@ +# Dev tools for local development only. +# This file is automatically loaded by `docker compose` alongside docker-compose.yml. +# CI sets COMPOSE_FILE=docker-compose.yml explicitly, so these services are excluded from test runs. + +services: + appwrite-mongo-express: + profiles: ["mongodb"] + image: mongo-express + container_name: appwrite-mongo-express + networks: + - appwrite + ports: + - "8082:8081" + environment: + ME_CONFIG_MONGODB_URL: "mongodb://root:${_APP_DB_ROOT_PASS}@appwrite-mongodb:27017/?replicaSet=rs0&directConnection=true" + ME_CONFIG_BASICAUTH_USERNAME: ${_APP_DB_USER} + ME_CONFIG_BASICAUTH_PASSWORD: ${_APP_DB_PASS} + depends_on: + - mongodb + + adminer: + image: adminer + container_name: appwrite-adminer + restart: always + ports: + - 9506:8080 + networks: + - appwrite + - gateway + environment: + - ADMINER_DESIGN=pepa-linha + - ADMINER_DEFAULT_SERVER=mariadb + - ADMINER_DEFAULT_USERNAME=root + - ADMINER_DEFAULT_PASSWORD=rootsecretpassword + - ADMINER_DEFAULT_DB=appwrite + configs: + - source: adminer-index.php + target: /var/www/html/index.php + mode: 0755 + labels: + - "traefik.enable=true" + - "traefik.constraint-label-stack=appwrite" + - "traefik.docker.network=gateway" + - "traefik.http.services.appwrite_adminer.loadbalancer.server.port=8080" + - "traefik.http.routers.appwrite_adminer_http.entrypoints=appwrite_web" + - "traefik.http.routers.appwrite_adminer_http.rule=Host(`mysql.localhost`)" + - "traefik.http.routers.appwrite_adminer_http.service=appwrite_adminer" + - "traefik.http.routers.appwrite_adminer_https.entrypoints=appwrite_websecure" + - "traefik.http.routers.appwrite_adminer_https.rule=Host(`mysql.localhost`)" + - "traefik.http.routers.appwrite_adminer_https.service=appwrite_adminer" + - "traefik.http.routers.appwrite_adminer_https.tls=true" + + redis-insight: + image: redis/redisinsight:latest + restart: unless-stopped + networks: + - appwrite + - gateway + environment: + - RI_PRE_SETUP_DATABASES_PATH=/mnt/connections.json + configs: + - source: redisinsight-connections.json + target: /mnt/connections.json + mode: 0755 + labels: + - "traefik.enable=true" + - "traefik.constraint-label-stack=appwrite" + - "traefik.docker.network=gateway" + - "traefik.http.services.appwrite_redisinsight.loadbalancer.server.port=5540" + - "traefik.http.routers.appwrite_redisinsight_http.entrypoints=appwrite_web" + - "traefik.http.routers.appwrite_redisinsight_http.rule=Host(`redis.localhost`)" + - "traefik.http.routers.appwrite_redisinsight_http.service=appwrite_redisinsight" + - "traefik.http.routers.appwrite_redisinsight_https.entrypoints=appwrite_websecure" + - "traefik.http.routers.appwrite_redisinsight_https.rule=Host(`redis.localhost`)" + - "traefik.http.routers.appwrite_redisinsight_https.service=appwrite_redisinsight" + - "traefik.http.routers.appwrite_redisinsight_https.tls=true" + ports: + - "8081:5540" + + graphql-explorer: + container_name: appwrite-graphql-explorer + image: appwrite/altair:0.3.0 + restart: unless-stopped + networks: + - appwrite + ports: + - "9509:3000" + environment: + - SERVER_URL=http://localhost/v1/graphql + +configs: + redisinsight-connections.json: + content: | + [ + { + "compressor": "NONE", + "id": "104dc90a-21ef-4d5e-8912-b30baabb152f", + "host": "redis", + "port": 6379, + "name": "redis:6379", + "db": 0, + "username": "default", + "password": null, + "connectionType": "STANDALONE", + "nameFromProvider": null, + "provider": "REDIS", + "lastConnection": "2025-10-16T09:22:02.591Z", + "modules": [ + { + "name": "ReJSON", + "version": 20808, + "semanticVersion": "2.8.8" + }, + { + "name": "search", + "version": 21015, + "semanticVersion": "2.10.15" + } + ], + "tls": false, + "tlsServername": null, + "verifyServerCert": null, + "caCert": null, + "clientCert": null, + "ssh": false, + "sshOptions": null, + "forceStandalone": false, + "tags": [] + } + ] + + adminer-index.php: + content: | + $$_ENV['ADMINER_DEFAULT_SERVER'], + 'driver' => 'server', /* seems to autodetect the driver from server settings */ + 'username' => $$_ENV['ADMINER_DEFAULT_USERNAME'], + 'password' => $$_ENV['ADMINER_DEFAULT_PASSWORD'], + 'db' => $$_ENV['ADMINER_DEFAULT_DB'], + ]; + } + include './adminer.php'; diff --git a/docker-compose.yml b/docker-compose.yml index c744a9c5a5..0013fdfa64 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1277,20 +1277,7 @@ services: retries: 10 start_period: 30s - appwrite-mongo-express: - profiles: ["mongodb"] - image: mongo-express - container_name: appwrite-mongo-express - networks: - - appwrite - ports: - - "8082:8081" - environment: - ME_CONFIG_MONGODB_URL: "mongodb://root:${_APP_DB_ROOT_PASS}@appwrite-mongodb:27017/?replicaSet=rs0&directConnection=true" - ME_CONFIG_BASICAUTH_USERNAME: ${_APP_DB_USER} - ME_CONFIG_BASICAUTH_PASSWORD: ${_APP_DB_PASS} - depends_on: - - mongodb + postgresql: profiles: ["postgresql"] @@ -1398,78 +1385,8 @@ services: networks: - appwrite - adminer: - image: adminer - container_name: appwrite-adminer - <<: *x-logging - restart: always - ports: - - 9506:8080 - networks: - - appwrite - - gateway - environment: - - ADMINER_DESIGN=pepa-linha - - ADMINER_DEFAULT_SERVER=mariadb - - ADMINER_DEFAULT_USERNAME=root - - ADMINER_DEFAULT_PASSWORD=rootsecretpassword - - ADMINER_DEFAULT_DB=appwrite - configs: - - source: adminer-index.php - target: /var/www/html/index.php - mode: 0755 - labels: - - "traefik.enable=true" - - "traefik.constraint-label-stack=appwrite" - - "traefik.docker.network=gateway" - - "traefik.http.services.appwrite_adminer.loadbalancer.server.port=8080" - - "traefik.http.routers.appwrite_adminer_http.entrypoints=appwrite_web" - - "traefik.http.routers.appwrite_adminer_http.rule=Host(`mysql.localhost`)" - - "traefik.http.routers.appwrite_adminer_http.service=appwrite_adminer" - - "traefik.http.routers.appwrite_adminer_https.entrypoints=appwrite_websecure" - - "traefik.http.routers.appwrite_adminer_https.rule=Host(`mysql.localhost`)" - - "traefik.http.routers.appwrite_adminer_https.service=appwrite_adminer" - - "traefik.http.routers.appwrite_adminer_https.tls=true" - - redis-insight: - image: redis/redisinsight:latest - restart: unless-stopped - networks: - - appwrite - - gateway - environment: - - RI_PRE_SETUP_DATABASES_PATH=/mnt/connections.json - configs: - - source: redisinsight-connections.json - target: /mnt/connections.json - mode: 0755 - labels: - - "traefik.enable=true" - - "traefik.constraint-label-stack=appwrite" - - "traefik.docker.network=gateway" - - "traefik.http.services.appwrite_redisinsight.loadbalancer.server.port=5540" - - "traefik.http.routers.appwrite_redisinsight_http.entrypoints=appwrite_web" - - "traefik.http.routers.appwrite_redisinsight_http.rule=Host(`redis.localhost`)" - - "traefik.http.routers.appwrite_redisinsight_http.service=appwrite_redisinsight" - - "traefik.http.routers.appwrite_redisinsight_https.entrypoints=appwrite_websecure" - - "traefik.http.routers.appwrite_redisinsight_https.rule=Host(`redis.localhost`)" - - "traefik.http.routers.appwrite_redisinsight_https.service=appwrite_redisinsight" - - "traefik.http.routers.appwrite_redisinsight_https.tls=true" - ports: - - "8081:5540" - - graphql-explorer: - container_name: appwrite-graphql-explorer - image: appwrite/altair:0.3.0 - restart: unless-stopped - networks: - - appwrite - ports: - - "9509:3000" - environment: - - SERVER_URL=http://localhost/v1/graphql - - # Dev Tools End ------------------------------------------------------------------------------------------ + # Dev tools (adminer, redis-insight, mongo-express, graphql-explorer) + # are defined in docker-compose.override.yml networks: gateway: @@ -1482,60 +1399,7 @@ networks: runtimes: name: runtimes -configs: - redisinsight-connections.json: - content: | - [ - { - "compressor": "NONE", - "id": "104dc90a-21ef-4d5e-8912-b30baabb152f", - "host": "redis", - "port": 6379, - "name": "redis:6379", - "db": 0, - "username": "default", - "password": null, - "connectionType": "STANDALONE", - "nameFromProvider": null, - "provider": "REDIS", - "lastConnection": "2025-10-16T09:22:02.591Z", - "modules": [ - { - "name": "ReJSON", - "version": 20808, - "semanticVersion": "2.8.8" - }, - { - "name": "search", - "version": 21015, - "semanticVersion": "2.10.15" - } - ], - "tls": false, - "tlsServername": null, - "verifyServerCert": null, - "caCert": null, - "clientCert": null, - "ssh": false, - "sshOptions": null, - "forceStandalone": false, - "tags": [] - } - ] - adminer-index.php: - content: | - $$_ENV['ADMINER_DEFAULT_SERVER'], - 'driver' => 'server', /* seems to autodetect the driver from server settings */ - 'username' => $$_ENV['ADMINER_DEFAULT_USERNAME'], - 'password' => $$_ENV['ADMINER_DEFAULT_PASSWORD'], - 'db' => $$_ENV['ADMINER_DEFAULT_DB'], - ]; - } - include './adminer.php'; volumes: appwrite-mariadb: From 33ce469ab0206b161b60983fabca86d666fdc839 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 21:44:15 +0000 Subject: [PATCH 34/57] Add Docker healthcheck and use --wait instead of polling loop Replace the manual shell polling loop (until doctor > /dev/null) with a proper Docker healthcheck on the appwrite service and `docker compose up --wait`, which blocks until healthchecks pass. Also reverts the timeout back to 3 minutes now that image pulls are cached. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 64 ++++++++++--------------------------- docker-compose.yml | 5 +++ 2 files changed, 21 insertions(+), 48 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5c20ca094d..cebcd2a62c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -109,15 +109,11 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Environment Variables run: docker compose exec -T appwrite vars @@ -153,15 +149,11 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Wait for Open Runtimes timeout-minutes: 3 @@ -259,7 +251,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_BROWSER_HOST: http://invalid-browser/v1 _APP_DATABASE_SHARED_TABLES: "" @@ -267,11 +259,7 @@ jobs: run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Wait for Open Runtimes timeout-minutes: 3 @@ -359,18 +347,14 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Wait for Open Runtimes timeout-minutes: 3 @@ -427,7 +411,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: "" @@ -435,11 +419,7 @@ jobs: run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Run Projects tests in dedicated table mode timeout-minutes: 15 @@ -488,7 +468,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: database_db_main @@ -496,11 +476,7 @@ jobs: run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Run Projects tests in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 @@ -541,18 +517,14 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_DATABASE_SHARED_TABLES: "" _APP_DATABASE_SHARED_TABLES_V1: "" run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Wait for Open Runtimes timeout-minutes: 3 @@ -609,18 +581,14 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Wait for Open Runtimes timeout-minutes: 3 diff --git a/docker-compose.yml b/docker-compose.yml index 0013fdfa64..583b558e60 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,6 +61,11 @@ services: container_name: appwrite <<: [*x-logging, *x-build] image: appwrite-dev + healthcheck: + test: ["CMD", "doctor"] + interval: 5s + timeout: 5s + retries: 12 ports: - 9501:80 networks: From 38798c899396ee48b3151185ecaa632495c3f939 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 21:58:30 +0000 Subject: [PATCH 35/57] Add healthchecks for MariaDB, PostgreSQL, and Redis Add proper healthchecks to infrastructure services and use condition: service_healthy for redis in appwrite's depends_on so it waits for Redis to be ready before starting. Co-Authored-By: Claude Opus 4.6 --- docker-compose.yml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 583b558e60..c0b0560a7f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,10 +108,10 @@ services: - ./dev:/usr/src/code/dev depends_on: - - ${_APP_DB_HOST:-mongodb} - - redis - - coredns - # - clamav + redis: + condition: service_healthy + coredns: + condition: service_started entrypoint: - php - -e @@ -1244,6 +1244,11 @@ services: - MYSQL_PASSWORD=${_APP_DB_PASS} - MARIADB_AUTO_UPGRADE=1 command: "mysqld --innodb-flush-method=fsync" + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 5s + timeout: 5s + retries: 12 mongodb: profiles: ["mongodb"] @@ -1303,6 +1308,11 @@ services: - POSTGRES_USER=${_APP_DB_USER} - POSTGRES_PASSWORD=${_APP_DB_PASS} command: "postgres" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${_APP_DB_USER}"] + interval: 5s + timeout: 5s + retries: 12 redis: image: redis:7.4.7-alpine @@ -1319,6 +1329,11 @@ services: - appwrite volumes: - appwrite-redis:/data:rw + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 12 coredns: # DNS server for testing purposes (Proxy APIs) image: coredns/coredns:1.12.4 From fd28ad8a6681c9cf8d678cc80c4c29bfc9e5b00d Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 21:59:33 +0000 Subject: [PATCH 36/57] Remove --debug flag from test commands for quieter CI output Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cebcd2a62c..39c9aee9ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -168,8 +168,7 @@ jobs: run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/General --debug - + appwrite test /usr/src/code/tests/e2e/General - name: Failure Logs if: failure() run: | From 16929bc42074ef739151d4021aa9d92ee91082f7 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 22:00:00 +0000 Subject: [PATCH 37/57] Remove remaining --debug flags from test commands Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 39c9aee9ca..879b72327d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -425,7 +425,7 @@ jobs: run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled + appwrite test /usr/src/code/tests/e2e/Services/Projects --group=abuseEnabled - name: Failure Logs if: failure() @@ -482,7 +482,7 @@ jobs: run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled + appwrite test /usr/src/code/tests/e2e/Services/Projects --group=abuseEnabled - name: Failure Logs if: failure() @@ -538,7 +538,7 @@ jobs: run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots + appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots - name: Failure Logs if: failure() @@ -602,7 +602,7 @@ jobs: run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots + appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots - name: Failure Logs if: failure() From 4cbe50193a163f3e9e44e97c520a46a200515e5f Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 22:24:52 +0000 Subject: [PATCH 38/57] Increase Load and Start Appwrite timeout to 5 minutes The docker compose pull step alone can take over 2.5 minutes on CI (e.g. openruntimes-executor, traefik), leaving no time for docker compose up --wait within 3 minutes. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 879b72327d..73c136bb93 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -109,7 +109,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -149,7 +149,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable From 7dfe44cb363a8a4350e4339fbc504f794143959f Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 22:44:42 +0000 Subject: [PATCH 39/57] Run abuse-enabled tests across entire test suite, not just Projects The abuseEnabled jobs previously only ran tests in Services/Projects, missing the Account abuse test and any future abuseEnabled tests in other services. Also rename jobs to "Abuse" for consistency. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 73c136bb93..c13370f3f0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -387,7 +387,7 @@ jobs: docker compose logs openruntimes-executor e2e_abuse_enabled: - name: E2E Service Test (Abuse enabled) + name: E2E Service Test (Abuse) runs-on: ubuntu-latest needs: setup permissions: @@ -420,12 +420,12 @@ jobs: docker compose pull --quiet --ignore-buildable docker compose up -d --quiet-pull --wait - - name: Run Projects tests in dedicated table mode + - name: Run abuse-enabled tests in dedicated table mode timeout-minutes: 15 run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Projects --group=abuseEnabled + appwrite test /usr/src/code/tests/e2e --group=abuseEnabled - name: Failure Logs if: failure() @@ -436,7 +436,7 @@ jobs: docker compose logs openruntimes-executor e2e_abuse_enabled_shared_mode: - name: E2E Shared Mode Service Test (Abuse enabled) + name: E2E Shared Mode Service Test (Abuse) runs-on: ubuntu-latest needs: [ setup, check_database_changes ] if: needs.check_database_changes.outputs.database_changed == 'true' @@ -477,12 +477,12 @@ jobs: docker compose pull --quiet --ignore-buildable docker compose up -d --quiet-pull --wait - - name: Run Projects tests in ${{ matrix.tables-mode }} table mode + - name: Run abuse-enabled tests in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Projects --group=abuseEnabled + appwrite test /usr/src/code/tests/e2e --group=abuseEnabled - name: Failure Logs if: failure() From 89419344c2b3f045d5ce8c048eb1dd2a7782c4de Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 23:20:51 +0000 Subject: [PATCH 40/57] Restore MariaDB defaults, fix remaining timeouts, filter listener span logs - Change .env defaults back from MongoDB to MariaDB - Bump all remaining "Load and Start Appwrite" timeouts from 3 to 5 minutes - Filter listener.* span logs to only export on error Co-Authored-By: Claude Opus 4.6 --- .env | 8 ++++---- .github/workflows/tests.yml | 12 ++++++------ app/init/span.php | 7 ++++++- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.env b/.env index 0df9cb42f4..3c8bef5b68 100644 --- a/.env +++ b/.env @@ -39,10 +39,10 @@ _APP_REDIS_HOST=redis _APP_REDIS_PORT=6379 _APP_REDIS_PASS= _APP_REDIS_USER= -COMPOSE_PROFILES=mongodb -_APP_DB_ADAPTER=mongodb -_APP_DB_HOST=mongodb -_APP_DB_PORT=27017 +COMPOSE_PROFILES=mariadb +_APP_DB_ADAPTER=mariadb +_APP_DB_HOST=mariadb +_APP_DB_PORT=3306 _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c13370f3f0..9f2556f3e5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -250,7 +250,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_BROWSER_HOST: http://invalid-browser/v1 _APP_DATABASE_SHARED_TABLES: "" @@ -346,7 +346,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} @@ -410,7 +410,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: "" @@ -467,7 +467,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: database_db_main @@ -516,7 +516,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_DATABASE_SHARED_TABLES: "" _APP_DATABASE_SHARED_TABLES_V1: "" @@ -580,7 +580,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} diff --git a/app/init/span.php b/app/init/span.php index 76f37f5300..8afa01b2df 100644 --- a/app/init/span.php +++ b/app/init/span.php @@ -5,4 +5,9 @@ use Utopia\Span\Span; use Utopia\Span\Storage; Span::setStorage(new Storage\Coroutine()); -Span::addExporter(new Exporter\Pretty()); +Span::addExporter(new Exporter\Pretty(), function (Span $span): bool { + if (\str_starts_with($span->getAction(), 'listener.')) { + return $span->getError() !== null; + } + return true; +}); From 8cb36835cc4eaa92345bae1b2413e7aa89b11d41 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 23:41:26 +0000 Subject: [PATCH 41/57] Restore php-retry action for flaky test resilience Re-add itznotabug/php-retry@v3 wrapping all test steps with max_attempts: 2 and retry_wait_seconds: 300. Also restore pull-requests: write permission needed by the action. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 135 ++++++++++++++++++++++++++---------- 1 file changed, 100 insertions(+), 35 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9f2556f3e5..20d3c71a9a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -90,6 +90,7 @@ jobs: needs: setup permissions: contents: read + pull-requests: write steps: - name: checkout @@ -119,11 +120,18 @@ jobs: run: docker compose exec -T appwrite vars - name: Run Unit Tests - timeout-minutes: 15 - run: >- - docker compose exec - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/unit + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/unit + command: >- + docker compose exec + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/unit e2e_general_test: name: E2E General Test @@ -131,6 +139,7 @@ jobs: needs: setup permissions: contents: read + pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -164,11 +173,19 @@ jobs: done - name: Run General Tests - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/General + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/General + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/General + - name: Failure Logs if: failure() run: | @@ -181,6 +198,7 @@ jobs: needs: setup permissions: contents: read + pull-requests: write strategy: fail-fast: false matrix: @@ -228,7 +246,7 @@ jobs: run: | DB_ADAPTER_LOWER=$(echo "${{ matrix.db_adapter }}" | tr 'A-Z' 'a-z') echo "COMPOSE_PROFILES=${DB_ADAPTER_LOWER}" >> $GITHUB_ENV - + if [ "${{ matrix.db_adapter }}" = "MARIADB" ]; then echo "_APP_DB_ADAPTER=mariadb" >> $GITHUB_ENV echo "_APP_DB_HOST=mariadb" >> $GITHUB_ENV @@ -269,8 +287,15 @@ jobs: done - name: Run ${{ matrix.service }} tests with Project table mode - timeout-minutes: 20 - run: | + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 20 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/Services/${{ matrix.service }} + command: | SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" # Services that rely on sequential test method execution (shared static state) @@ -296,6 +321,7 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read + pull-requests: write strategy: fail-fast: false matrix: @@ -364,8 +390,15 @@ jobs: done - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode - timeout-minutes: 20 - run: | + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 20 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/Services/${{ matrix.service }} + command: | SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" # Services that rely on sequential test method execution (shared static state) @@ -392,6 +425,7 @@ jobs: needs: setup permissions: contents: read + pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -421,11 +455,18 @@ jobs: docker compose up -d --quiet-pull --wait - name: Run abuse-enabled tests in dedicated table mode - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e --group=abuseEnabled + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e --group=abuseEnabled - name: Failure Logs if: failure() @@ -442,6 +483,7 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read + pull-requests: write strategy: fail-fast: false matrix: @@ -478,11 +520,18 @@ jobs: docker compose up -d --quiet-pull --wait - name: Run abuse-enabled tests in ${{ matrix.tables-mode }} table mode - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e --group=abuseEnabled + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e --group=abuseEnabled - name: Failure Logs if: failure() @@ -498,6 +547,7 @@ jobs: needs: setup permissions: contents: read + pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -534,11 +584,18 @@ jobs: done - name: Run Site tests with browser connected in dedicated table mode - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/Services/Sites + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots - name: Failure Logs if: failure() @@ -555,6 +612,7 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read + pull-requests: write strategy: fail-fast: false matrix: @@ -598,11 +656,18 @@ jobs: done - name: Run Site tests with browser connected in ${{ matrix.tables-mode }} table mode - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/Services/Sites + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots - name: Failure Logs if: failure() From 9f4ba3a4a285042b8e79327be96d81a0ac51db09 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 23:47:03 +0000 Subject: [PATCH 42/57] Reduce php-retry wait time from 300s to 60s Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 20d3c71a9a..1b112eebbd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -123,7 +123,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -176,7 +176,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -290,7 +290,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 20 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -393,7 +393,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 20 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -458,7 +458,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -523,7 +523,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -587,7 +587,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -659,7 +659,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} From f09b9a994eaefed1ec506db792facc823d707858 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 23:59:31 +0000 Subject: [PATCH 43/57] Revert .env defaults back to MongoDB Co-Authored-By: Claude Opus 4.6 --- .env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 3c8bef5b68..0df9cb42f4 100644 --- a/.env +++ b/.env @@ -39,10 +39,10 @@ _APP_REDIS_HOST=redis _APP_REDIS_PORT=6379 _APP_REDIS_PASS= _APP_REDIS_USER= -COMPOSE_PROFILES=mariadb -_APP_DB_ADAPTER=mariadb -_APP_DB_HOST=mariadb -_APP_DB_PORT=3306 +COMPOSE_PROFILES=mongodb +_APP_DB_ADAPTER=mongodb +_APP_DB_HOST=mongodb +_APP_DB_PORT=27017 _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password From 20c65bac37959979a4d3e76703fdd69ef4f99742 Mon Sep 17 00:00:00 2001 From: eldadfux Date: Thu, 12 Mar 2026 09:02:16 +0100 Subject: [PATCH 44/57] Allow null branch --- src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php index 47195a3eb5..ba99cefb42 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php @@ -75,7 +75,7 @@ class Create extends Action ->callback($this->action(...)); } - public function action(string $domain, string $siteId, string $branch, Response $response, Document $project, Certificate $queueForCertificates, Event $queueForEvents, Database $dbForPlatform, Database $dbForProject, array $platform, Log $log) + public function action(string $domain, string $siteId, ?string $branch, Response $response, Document $project, Certificate $queueForCertificates, Event $queueForEvents, Database $dbForPlatform, Database $dbForProject, array $platform, Log $log) { $this->validateDomainRestrictions($domain, $platform); From e5f0c2df12f9296cb16e6466c35a9f1412035484 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 00:47:54 +0000 Subject: [PATCH 45/57] Consolidate CI test matrix with dynamic database and mode dimensions Merge 6 E2E jobs into 3 by combining dedicated/shared mode variants into a single matrix dimension. Database adapters and table modes expand dynamically based on whether utopia-php/database changed. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 328 +++++------------------------------- 1 file changed, 42 insertions(+), 286 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1b112eebbd..10cf958652 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,19 +28,21 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v6 - - - name: Fetch base branch - run: git fetch origin ${{ github.event.pull_request.base.ref }} + with: + fetch-depth: 0 - name: Check for utopia-php/database changes id: check run: | - if git diff origin/${{ github.event.pull_request.base.ref }} HEAD -- composer.lock | grep -q '"name": "utopia-php/database"'; then - echo "Database version changed, going to run all mode tests." - echo "database_changed=true" >> "$GITHUB_ENV" + BASE_REF="${{ github.event.pull_request.base.ref }}" + if [ -z "$BASE_REF" ]; then + echo "database_changed=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if git diff "origin/${BASE_REF}" HEAD -- composer.lock | grep -q '"name": "utopia-php/database"'; then echo "database_changed=true" >> "$GITHUB_OUTPUT" else - echo "database_changed=false" >> "$GITHUB_ENV" echo "database_changed=false" >> "$GITHUB_OUTPUT" fi @@ -193,20 +195,17 @@ jobs: docker compose logs e2e_service_test: - name: E2E Service Test + name: E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service }} runs-on: ubuntu-latest - needs: setup + needs: [setup, check_database_changes] permissions: contents: read pull-requests: write strategy: fail-fast: false matrix: - db_adapter: [ - MARIADB, - POSTGRESQL, - MONGODB - ] + database: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["MariaDB","PostgreSQL","MongoDB"]' || '["MongoDB"]') }} + mode: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["dedicated","shared_v1","shared_v2"]' || '["dedicated"]') }} service: [ Account, Avatars, @@ -241,23 +240,18 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true - - name: Set DB Adapter environment - id: set-db-env + - name: Set database environment run: | - DB_ADAPTER_LOWER=$(echo "${{ matrix.db_adapter }}" | tr 'A-Z' 'a-z') - echo "COMPOSE_PROFILES=${DB_ADAPTER_LOWER}" >> $GITHUB_ENV + DB_LOWER=$(echo "${{ matrix.database }}" | tr 'A-Z' 'a-z') + echo "COMPOSE_PROFILES=${DB_LOWER}" >> $GITHUB_ENV + echo "_APP_DB_ADAPTER=${DB_LOWER}" >> $GITHUB_ENV + echo "_APP_DB_HOST=${DB_LOWER}" >> $GITHUB_ENV - if [ "${{ matrix.db_adapter }}" = "MARIADB" ]; then - echo "_APP_DB_ADAPTER=mariadb" >> $GITHUB_ENV - echo "_APP_DB_HOST=mariadb" >> $GITHUB_ENV + if [ "${{ matrix.database }}" = "MariaDB" ]; then echo "_APP_DB_PORT=3306" >> $GITHUB_ENV - elif [ "${{ matrix.db_adapter }}" = "MONGODB" ]; then - echo "_APP_DB_ADAPTER=mongodb" >> $GITHUB_ENV - echo "_APP_DB_HOST=mongodb" >> $GITHUB_ENV + elif [ "${{ matrix.database }}" = "MongoDB" ]; then echo "_APP_DB_PORT=27017" >> $GITHUB_ENV - elif [ "${{ matrix.db_adapter }}" = "POSTGRESQL" ]; then - echo "_APP_DB_ADAPTER=postgresql" >> $GITHUB_ENV - echo "_APP_DB_HOST=postgresql" >> $GITHUB_ENV + elif [ "${{ matrix.database }}" = "PostgreSQL" ]; then echo "_APP_DB_PORT=5432" >> $GITHUB_ENV fi @@ -271,8 +265,8 @@ jobs: timeout-minutes: 5 env: _APP_BROWSER_HOST: http://invalid-browser/v1 - _APP_DATABASE_SHARED_TABLES: "" - _APP_DATABASE_SHARED_TABLES_V1: "" + _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -286,7 +280,7 @@ jobs: sleep 1 done - - name: Run ${{ matrix.service }} tests with Project table mode + - name: Run tests uses: itznotabug/php-retry@v3 with: max_attempts: 2 @@ -314,185 +308,19 @@ jobs: echo "=== Appwrite Logs ===" docker compose logs - e2e_shared_mode_test: - name: E2E Shared Mode Service Test - runs-on: ubuntu-latest - needs: [ setup, check_database_changes ] - if: needs.check_database_changes.outputs.database_changed == 'true' - permissions: - contents: read - pull-requests: write - strategy: - fail-fast: false - matrix: - service: - [ - Account, - Avatars, - Console, - Databases, - Functions, - FunctionsSchedule, - GraphQL, - Health, - Locale, - Projects, - Realtime, - Sites, - Proxy, - Storage, - Teams, - Users, - Webhooks, - VCS, - Messaging, - Migrations, - Tokens - ] - tables-mode: [ - 'Shared V1', - 'Shared V2', - ] - - steps: - - name: checkout - uses: actions/checkout@v6 - - - name: Load Cache - uses: actions/cache@v5 - with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true - - - name: Login to Docker Hub - uses: docker/login-action@v4 - with: - username: ${{ vars.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Load and Start Appwrite - timeout-minutes: 5 - env: - _APP_DATABASE_SHARED_TABLES: database_db_main - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} - run: | - docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet --ignore-buildable - docker compose up -d --quiet-pull --wait - - - name: Wait for Open Runtimes - timeout-minutes: 3 - run: | - while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do - echo "Waiting for Executor to come online" - sleep 1 - done - - - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 60 - timeout_minutes: 20 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/${{ matrix.service }} - command: | - SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" - - # Services that rely on sequential test method execution (shared static state) - FUNCTIONAL_FLAG="--functional" - case "${{ matrix.service }}" in - Databases|Functions|Realtime) FUNCTIONAL_FLAG="" ;; - esac - - docker compose exec -T \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite vendor/bin/paratest --processes $(nproc) $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --exclude-group ciIgnore --log-junit tests/e2e/Services/${{ matrix.service }}/junit.xml - - - name: Failure Logs - if: failure() - run: | - echo "=== Appwrite Worker Builds Logs ===" - docker compose logs appwrite-worker-builds - echo "=== OpenRuntimes Executor Logs ===" - docker compose logs openruntimes-executor - e2e_abuse_enabled: - name: E2E Service Test (Abuse) + name: E2E / Abuse (${{ matrix.mode }}) runs-on: ubuntu-latest - needs: setup - permissions: - contents: read - pull-requests: write - steps: - - name: checkout - uses: actions/checkout@v6 - - - name: Load Cache - uses: actions/cache@v5 - with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true - - - name: Login to Docker Hub - uses: docker/login-action@v4 - with: - username: ${{ vars.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Load and Start Appwrite - timeout-minutes: 5 - env: - _APP_OPTIONS_ABUSE: enabled - _APP_DATABASE_SHARED_TABLES: "" - _APP_DATABASE_SHARED_TABLES_V1: "" - run: | - docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet --ignore-buildable - docker compose up -d --quiet-pull --wait - - - name: Run abuse-enabled tests in dedicated table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 60 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e - command: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e --group=abuseEnabled - - - name: Failure Logs - if: failure() - run: | - echo "=== Appwrite Worker Builds Logs ===" - docker compose logs appwrite-worker-builds - echo "=== OpenRuntimes Executor Logs ===" - docker compose logs openruntimes-executor - - e2e_abuse_enabled_shared_mode: - name: E2E Shared Mode Service Test (Abuse) - runs-on: ubuntu-latest - needs: [ setup, check_database_changes ] - if: needs.check_database_changes.outputs.database_changed == 'true' + needs: [setup, check_database_changes] permissions: contents: read pull-requests: write strategy: fail-fast: false matrix: - tables-mode: [ - 'Shared V1', - 'Shared V2', - ] + mode: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["dedicated","shared_v1","shared_v2"]' || '["dedicated"]') }} steps: - - name: checkout + - name: Checkout repository uses: actions/checkout@v6 - name: Load Cache @@ -512,14 +340,14 @@ jobs: timeout-minutes: 5 env: _APP_OPTIONS_ABUSE: enabled - _APP_DATABASE_SHARED_TABLES: database_db_main - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} + _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable docker compose up -d --quiet-pull --wait - - name: Run abuse-enabled tests in ${{ matrix.tables-mode }} table mode + - name: Run tests uses: itznotabug/php-retry@v3 with: max_attempts: 2 @@ -536,92 +364,22 @@ jobs: - name: Failure Logs if: failure() run: | - echo "=== Appwrite Worker Builds Logs ===" - docker compose logs appwrite-worker-builds - echo "=== OpenRuntimes Executor Logs ===" - docker compose logs openruntimes-executor + echo "=== Appwrite Logs ===" + docker compose logs e2e_screenshots: - name: E2E Service Test (Site Screenshots) + name: E2E / Screenshots (${{ matrix.mode }}) runs-on: ubuntu-latest - needs: setup - permissions: - contents: read - pull-requests: write - steps: - - name: checkout - uses: actions/checkout@v6 - - - name: Load Cache - uses: actions/cache@v5 - with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true - - - name: Login to Docker Hub - uses: docker/login-action@v4 - with: - username: ${{ vars.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Load and Start Appwrite - timeout-minutes: 5 - env: - _APP_DATABASE_SHARED_TABLES: "" - _APP_DATABASE_SHARED_TABLES_V1: "" - run: | - docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet --ignore-buildable - docker compose up -d --quiet-pull --wait - - - name: Wait for Open Runtimes - timeout-minutes: 3 - run: | - while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do - echo "Waiting for Executor to come online" - sleep 1 - done - - - name: Run Site tests with browser connected in dedicated table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 60 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Sites - command: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots - - - name: Failure Logs - if: failure() - run: | - echo "=== Appwrite Worker Builds Logs ===" - docker compose logs appwrite-worker-builds - echo "=== OpenRuntimes Executor Logs ===" - docker compose logs openruntimes-executor - - e2e_screenshots_shared_mode: - name: E2E Shared Mode Service Test (Site Screenshots) - runs-on: ubuntu-latest - needs: [ setup, check_database_changes ] - if: needs.check_database_changes.outputs.database_changed == 'true' + needs: [setup, check_database_changes] permissions: contents: read pull-requests: write strategy: fail-fast: false matrix: - tables-mode: [ - 'Shared V1', - 'Shared V2', - ] + mode: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["dedicated","shared_v1","shared_v2"]' || '["dedicated"]') }} steps: - - name: checkout + - name: Checkout repository uses: actions/checkout@v6 - name: Load Cache @@ -640,8 +398,8 @@ jobs: - name: Load and Start Appwrite timeout-minutes: 5 env: - _APP_DATABASE_SHARED_TABLES: database_db_main - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} + _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -655,7 +413,7 @@ jobs: sleep 1 done - - name: Run Site tests with browser connected in ${{ matrix.tables-mode }} table mode + - name: Run tests uses: itznotabug/php-retry@v3 with: max_attempts: 2 @@ -672,7 +430,5 @@ jobs: - name: Failure Logs if: failure() run: | - echo "=== Appwrite Worker Builds Logs ===" - docker compose logs appwrite-worker-builds - echo "=== OpenRuntimes Executor Logs ===" - docker compose logs openruntimes-executor + echo "=== Appwrite Logs ===" + docker compose logs From 09317f290a2355037b92ff7dae62f979b7254924 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 08:55:47 +0000 Subject: [PATCH 46/57] Clean up database env setup and improve matrix naming Hardcode lowercase env vars per database branch instead of using tr. Use proper casing for database matrix values (MongoDB, MariaDB, PostgreSQL). Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 10cf958652..7d119f78f5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -242,16 +242,20 @@ jobs: - name: Set database environment run: | - DB_LOWER=$(echo "${{ matrix.database }}" | tr 'A-Z' 'a-z') - echo "COMPOSE_PROFILES=${DB_LOWER}" >> $GITHUB_ENV - echo "_APP_DB_ADAPTER=${DB_LOWER}" >> $GITHUB_ENV - echo "_APP_DB_HOST=${DB_LOWER}" >> $GITHUB_ENV - if [ "${{ matrix.database }}" = "MariaDB" ]; then + echo "COMPOSE_PROFILES=mariadb" >> $GITHUB_ENV + echo "_APP_DB_ADAPTER=mariadb" >> $GITHUB_ENV + echo "_APP_DB_HOST=mariadb" >> $GITHUB_ENV echo "_APP_DB_PORT=3306" >> $GITHUB_ENV elif [ "${{ matrix.database }}" = "MongoDB" ]; then + echo "COMPOSE_PROFILES=mongodb" >> $GITHUB_ENV + echo "_APP_DB_ADAPTER=mongodb" >> $GITHUB_ENV + echo "_APP_DB_HOST=mongodb" >> $GITHUB_ENV echo "_APP_DB_PORT=27017" >> $GITHUB_ENV elif [ "${{ matrix.database }}" = "PostgreSQL" ]; then + echo "COMPOSE_PROFILES=postgresql" >> $GITHUB_ENV + echo "_APP_DB_ADAPTER=postgresql" >> $GITHUB_ENV + echo "_APP_DB_HOST=postgresql" >> $GITHUB_ENV echo "_APP_DB_PORT=5432" >> $GITHUB_ENV fi From edd948557e4bc142ce722d59826b108cd8e1eb16 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 11:01:36 +0000 Subject: [PATCH 47/57] Refactor matrix job to use GitHub API and clean up test config Replace shell-based database change detection with github-script using the GitHub API, eliminating the need for a full checkout. Restructure matrix generation with guard clauses and no mutation. Remove ciIgnore exclude group from test command. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 65 ++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7d119f78f5..7b16ac4982 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,31 +20,42 @@ on: default: '' jobs: - check_database_changes: - name: Check if utopia-php/database changed + matrix: + name: Generate test matrix runs-on: ubuntu-latest outputs: - database_changed: ${{ steps.check.outputs.database_changed }} + databases: ${{ steps.generate.outputs.databases }} + modes: ${{ steps.generate.outputs.modes }} steps: - - name: Checkout repository - uses: actions/checkout@v6 + - name: Generate matrix + id: generate + uses: actions/github-script@v7 with: - fetch-depth: 0 + script: | + const allDatabases = ['MariaDB', 'PostgreSQL', 'MongoDB']; + const allModes = ['dedicated', 'shared_v1', 'shared_v2']; - - name: Check for utopia-php/database changes - id: check - run: | - BASE_REF="${{ github.event.pull_request.base.ref }}" - if [ -z "$BASE_REF" ]; then - echo "database_changed=true" >> "$GITHUB_OUTPUT" - exit 0 - fi + const defaultDatabases = ['MongoDB']; + const defaultModes = ['dedicated']; - if git diff "origin/${BASE_REF}" HEAD -- composer.lock | grep -q '"name": "utopia-php/database"'; then - echo "database_changed=true" >> "$GITHUB_OUTPUT" - else - echo "database_changed=false" >> "$GITHUB_OUTPUT" - fi + const pr = context.payload.pull_request; + if (!pr) { + core.setOutput('databases', JSON.stringify(allDatabases)); + core.setOutput('modes', JSON.stringify(allModes)); + return; + } + + const files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pr.number, + }); + + const lockFile = files.find(f => f.filename === 'composer.lock'); + const databaseChanged = lockFile?.patch?.includes('"name": "utopia-php/database"') ?? false; + + core.setOutput('databases', JSON.stringify(databaseChanged ? allDatabases : defaultDatabases)); + core.setOutput('modes', JSON.stringify(databaseChanged ? allModes : defaultModes)); setup: name: Setup & Build Appwrite Image @@ -197,15 +208,15 @@ jobs: e2e_service_test: name: E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service }} runs-on: ubuntu-latest - needs: [setup, check_database_changes] + needs: [setup, matrix] permissions: contents: read pull-requests: write strategy: fail-fast: false matrix: - database: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["MariaDB","PostgreSQL","MongoDB"]' || '["MongoDB"]') }} - mode: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["dedicated","shared_v1","shared_v2"]' || '["dedicated"]') }} + database: ${{ fromJSON(needs.matrix.outputs.databases) }} + mode: ${{ fromJSON(needs.matrix.outputs.modes) }} service: [ Account, Avatars, @@ -304,7 +315,7 @@ jobs: docker compose exec -T \ -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite vendor/bin/paratest --processes $(nproc) $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --exclude-group ciIgnore --log-junit tests/e2e/Services/${{ matrix.service }}/junit.xml + appwrite vendor/bin/paratest --processes $(nproc) $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --log-junit tests/e2e/Services/${{ matrix.service }}/junit.xml - name: Failure Logs if: failure() @@ -315,14 +326,14 @@ jobs: e2e_abuse_enabled: name: E2E / Abuse (${{ matrix.mode }}) runs-on: ubuntu-latest - needs: [setup, check_database_changes] + needs: [setup, matrix] permissions: contents: read pull-requests: write strategy: fail-fast: false matrix: - mode: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["dedicated","shared_v1","shared_v2"]' || '["dedicated"]') }} + mode: ${{ fromJSON(needs.matrix.outputs.modes) }} steps: - name: Checkout repository uses: actions/checkout@v6 @@ -374,14 +385,14 @@ jobs: e2e_screenshots: name: E2E / Screenshots (${{ matrix.mode }}) runs-on: ubuntu-latest - needs: [setup, check_database_changes] + needs: [setup, matrix] permissions: contents: read pull-requests: write strategy: fail-fast: false matrix: - mode: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["dedicated","shared_v1","shared_v2"]' || '["dedicated"]') }} + mode: ${{ fromJSON(needs.matrix.outputs.modes) }} steps: - name: Checkout repository uses: actions/checkout@v6 From aecca2f503c08edca8c179b6ec8e90259a2c2d50 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:39:14 +0000 Subject: [PATCH 48/57] Consolidate PR workflows into single CI workflow Merge linter, static-analysis, tests, and benchmark workflows into ci.yml with structured job naming (Checks / Format, Tests / E2E / ..., etc.). Shared Docker image build between tests and benchmark. Update actions to latest versions. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/benchmark.yml | 123 ------------------ .github/workflows/{tests.yml => ci.yml} | 161 +++++++++++++++++++++--- .github/workflows/linter.yml | 28 ----- .github/workflows/stale.yml | 2 +- .github/workflows/static-analysis.yml | 21 ---- 5 files changed, 147 insertions(+), 188 deletions(-) delete mode 100644 .github/workflows/benchmark.yml rename .github/workflows/{tests.yml => ci.yml} (70%) delete mode 100644 .github/workflows/linter.yml delete mode 100644 .github/workflows/static-analysis.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index b7b4fa0d2f..0000000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: Benchmark -concurrency: - group: '${{ github.workflow }}-${{ github.ref }}' - cancel-in-progress: true -env: - COMPOSE_FILE: docker-compose.yml - IMAGE: appwrite-dev - CACHE_KEY: 'appwrite-dev-${{ github.event.pull_request.head.sha }}' -'on': - - pull_request -jobs: - setup: - name: Setup & Build Appwrite Image - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - submodules: recursive - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Build Appwrite - uses: docker/build-push-action@v6 - with: - context: . - push: false - tags: '${{ env.IMAGE }}' - load: true - cache-from: type=gha - cache-to: 'type=gha,mode=max' - outputs: 'type=docker,dest=/tmp/${{ env.IMAGE }}.tar' - target: development - build-args: | - DEBUG=false - TESTING=true - VERSION=dev - - name: Cache Docker Image - uses: actions/cache@v4 - with: - key: '${{ env.CACHE_KEY }}' - path: '/tmp/${{ env.IMAGE }}.tar' - benchmarking: - name: Benchmark - runs-on: ubuntu-latest - needs: setup - permissions: - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - name: Load Cache - uses: actions/cache@v4 - with: - key: '${{ env.CACHE_KEY }}' - path: '/tmp/${{ env.IMAGE }}.tar' - fail-on-cache-miss: true - - name: Load and Start Appwrite - run: | - sed -i 's/traefik/localhost/g' .env - docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose up -d - sleep 10 - - name: Install Oha - run: | - echo "deb [signed-by=/usr/share/keyrings/azlux-archive-keyring.gpg] http://packages.azlux.fr/debian/ stable main" | sudo tee /etc/apt/sources.list.d/azlux.list - sudo wget -O /usr/share/keyrings/azlux-archive-keyring.gpg https://azlux.fr/repo.gpg - sudo apt update - sudo apt install oha - oha --version - - name: Benchmark PR - run: 'oha -z 180s http://localhost/v1/health/version --output-format json > benchmark.json' - - name: Cleaning - run: docker compose down -v - - name: Installing latest version - run: | - rm docker-compose.yml - rm .env - curl https://appwrite.io/install/compose -o docker-compose.yml - curl https://appwrite.io/install/env -o .env - sed -i 's/_APP_OPTIONS_ABUSE=enabled/_APP_OPTIONS_ABUSE=disabled/g' .env - docker compose up -d - sleep 10 - - name: Benchmark Latest - run: oha -z 180s http://localhost/v1/health/version --output-format json > benchmark-latest.json - - name: Prepare comment - run: | - echo '## :sparkles: Benchmark results' > benchmark.txt - echo ' ' >> benchmark.txt - echo "- Requests per second: $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt - echo "- Requests with 200 status code: $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt - echo "- P99 latency: $(jq -r '.latencyPercentiles.p99' benchmark.json )" >> benchmark.txt - echo " " >> benchmark.txt - echo " " >> benchmark.txt - echo "## :zap: Benchmark Comparison" >> benchmark.txt - echo " " >> benchmark.txt - echo "| Metric | This PR | Latest version | " >> benchmark.txt - echo "| --- | --- | --- | " >> benchmark.txt - echo "| RPS | $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.summary.requestsPerSec|tonumber|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt - echo "| 200 | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt - echo "| P99 | $(jq -r '.latencyPercentiles.p99' benchmark.json ) | $(jq -r '.latencyPercentiles.p99' benchmark-latest.json ) | " >> benchmark.txt - - name: Save results - uses: actions/upload-artifact@v6 - if: '${{ !cancelled() }}' - with: - name: benchmark.json - path: benchmark.json - retention-days: 7 - - name: Find Comment - if: github.event.pull_request.head.repo.full_name == github.repository - uses: peter-evans/find-comment@v3 - id: fc - with: - issue-number: '${{ github.event.pull_request.number }}' - comment-author: 'github-actions[bot]' - body-includes: Benchmark results - - name: Comment on PR - if: github.event.pull_request.head.repo.full_name == github.repository - uses: peter-evans/create-or-update-comment@v4 - with: - comment-id: '${{ steps.fc.outputs.comment-id }}' - issue-number: '${{ github.event.pull_request.number }}' - body-path: benchmark.txt - edit-mode: replace diff --git a/.github/workflows/tests.yml b/.github/workflows/ci.yml similarity index 70% rename from .github/workflows/tests.yml rename to .github/workflows/ci.yml index 7b16ac4982..233b365a9a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ -name: "Tests" +name: CI concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ci-${{ github.ref }} cancel-in-progress: true env: @@ -20,8 +20,46 @@ on: default: '' jobs: + format: + name: Checks / Format + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 2 + + - run: git checkout HEAD^2 + + - name: Validate composer.json and composer.lock + run: | + docker run --rm -v $PWD:/app composer:2.8 sh -c \ + "composer validate" + + - name: Run Linter + run: | + docker run --rm -v $PWD:/app composer:2.8 sh -c \ + "composer install --profile --ignore-platform-reqs && composer lint" + + analyze: + name: Checks / Analyze + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v6 + + - name: Run CodeQL + run: | + docker run --rm -v $PWD:/app composer:2.8 sh -c \ + "composer install --profile --ignore-platform-reqs && composer check" + + - name: Run Locale check + run: | + docker run --rm -v $PWD:/app node:24-alpine sh -c \ + "cd /app/.github/workflows/static-analysis/locale && node index.js" + matrix: - name: Generate test matrix + name: Tests / Matrix runs-on: ubuntu-latest outputs: databases: ${{ steps.generate.outputs.databases }} @@ -29,7 +67,7 @@ jobs: steps: - name: Generate matrix id: generate - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | const allDatabases = ['MariaDB', 'PostgreSQL', 'MongoDB']; @@ -58,7 +96,7 @@ jobs: core.setOutput('modes', JSON.stringify(databaseChanged ? allModes : defaultModes)); setup: - name: Setup & Build Appwrite Image + name: Setup runs-on: ubuntu-latest steps: - name: Checkout repository @@ -97,14 +135,13 @@ jobs: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar - unit_test: - name: Unit Test + unit: + name: Tests / Unit runs-on: ubuntu-latest needs: setup permissions: contents: read pull-requests: write - steps: - name: checkout uses: actions/checkout@v6 @@ -146,8 +183,8 @@ jobs: -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" appwrite test /usr/src/code/tests/unit - e2e_general_test: - name: E2E General Test + e2e_general: + name: Tests / E2E / General runs-on: ubuntu-latest needs: setup permissions: @@ -205,8 +242,8 @@ jobs: echo "=== Appwrite Logs ===" docker compose logs - e2e_service_test: - name: E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service }} + e2e_service: + name: Tests / E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service }} runs-on: ubuntu-latest needs: [setup, matrix] permissions: @@ -323,8 +360,8 @@ jobs: echo "=== Appwrite Logs ===" docker compose logs - e2e_abuse_enabled: - name: E2E / Abuse (${{ matrix.mode }}) + e2e_abuse: + name: Tests / E2E / Abuse (${{ matrix.mode }}) runs-on: ubuntu-latest needs: [setup, matrix] permissions: @@ -383,7 +420,7 @@ jobs: docker compose logs e2e_screenshots: - name: E2E / Screenshots (${{ matrix.mode }}) + name: Tests / E2E / Screenshots (${{ matrix.mode }}) runs-on: ubuntu-latest needs: [setup, matrix] permissions: @@ -447,3 +484,97 @@ jobs: run: | echo "=== Appwrite Logs ===" docker compose logs + + benchmark: + name: Benchmark + runs-on: ubuntu-latest + needs: setup + permissions: + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Load Cache + uses: actions/cache@v5 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + fail-on-cache-miss: true + + - name: Load and Start Appwrite + run: | + sed -i 's/traefik/localhost/g' .env + docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose up -d + sleep 10 + + - name: Install Oha + run: | + echo "deb [signed-by=/usr/share/keyrings/azlux-archive-keyring.gpg] http://packages.azlux.fr/debian/ stable main" | sudo tee /etc/apt/sources.list.d/azlux.list + sudo wget -O /usr/share/keyrings/azlux-archive-keyring.gpg https://azlux.fr/repo.gpg + sudo apt update + sudo apt install oha + oha --version + + - name: Benchmark PR + run: 'oha -z 180s http://localhost/v1/health/version --output-format json > benchmark.json' + + - name: Cleaning + run: docker compose down -v + + - name: Installing latest version + run: | + rm docker-compose.yml + rm .env + curl https://appwrite.io/install/compose -o docker-compose.yml + curl https://appwrite.io/install/env -o .env + sed -i 's/_APP_OPTIONS_ABUSE=enabled/_APP_OPTIONS_ABUSE=disabled/g' .env + docker compose up -d + sleep 10 + + - name: Benchmark Latest + run: oha -z 180s http://localhost/v1/health/version --output-format json > benchmark-latest.json + + - name: Prepare comment + run: | + echo '## :sparkles: Benchmark results' > benchmark.txt + echo ' ' >> benchmark.txt + echo "- Requests per second: $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt + echo "- Requests with 200 status code: $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt + echo "- P99 latency: $(jq -r '.latencyPercentiles.p99' benchmark.json )" >> benchmark.txt + echo " " >> benchmark.txt + echo " " >> benchmark.txt + echo "## :zap: Benchmark Comparison" >> benchmark.txt + echo " " >> benchmark.txt + echo "| Metric | This PR | Latest version | " >> benchmark.txt + echo "| --- | --- | --- | " >> benchmark.txt + echo "| RPS | $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.summary.requestsPerSec|tonumber|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt + echo "| 200 | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt + echo "| P99 | $(jq -r '.latencyPercentiles.p99' benchmark.json ) | $(jq -r '.latencyPercentiles.p99' benchmark-latest.json ) | " >> benchmark.txt + + - name: Save results + uses: actions/upload-artifact@v7 + if: ${{ !cancelled() }} + with: + name: benchmark.json + path: benchmark.json + retention-days: 7 + + - name: Find Comment + if: github.event.pull_request.head.repo.full_name == github.repository + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: Benchmark results + + - name: Comment on PR + if: github.event.pull_request.head.repo.full_name == github.repository + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: benchmark.txt + edit-mode: replace diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml deleted file mode 100644 index f4ae5df1ce..0000000000 --- a/.github/workflows/linter.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: "Linter" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: [pull_request] -jobs: - lint: - name: Linter - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - fetch-depth: 2 - - - run: git checkout HEAD^2 - - - name: Validate composer.json and composer.lock - run: | - docker run --rm -v $PWD:/app composer:2.8 sh -c \ - "composer validate" - - name: Run Linter - run: | - docker run --rm -v $PWD:/app composer:2.8 sh -c \ - "composer install --profile --ignore-platform-reqs && composer lint" diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 5987eeeb0c..6e4a8ba73b 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/stale@v9 + - uses: actions/stale@v10 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: "This issue has been labeled as a 'question', indicating that it requires additional information from the requestor. It has been inactive for 7 days. If no further activity occurs, this issue will be closed in 14 days." diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml deleted file mode 100644 index a0dc38b3b4..0000000000 --- a/.github/workflows/static-analysis.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: "Static code analysis" - -on: [pull_request] -jobs: - lint: - name: CodeQL - runs-on: ubuntu-latest - - steps: - - name: Check out the repo - uses: actions/checkout@v6 - - - name: Run CodeQL - run: | - docker run --rm -v $PWD:/app composer:2.8 sh -c \ - "composer install --profile --ignore-platform-reqs && composer check" - - - name: Run Locale check - run: | - docker run --rm -v $PWD:/app node:24-alpine sh -c \ - "cd /app/.github/workflows/static-analysis/locale && node index.js" From 8d0a4d7f92bd11f3f6e58ac5205585a14840c945 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:56:41 +0000 Subject: [PATCH 49/57] Consolidate remaining PR workflows and simplify Trivy scan - Move check-dependencies into ci.yml as Checks / Dependencies (upgrade to osv-scanner-reusable-pr.yml@v2.3.3, drop merge_group) - Move pr-scan into ci.yml as Checks / Image (upgrade Trivy to 0.33.1, use SARIF + upload-sarif instead of custom PR comment logic) - Rename Setup job to Build - Fix format job git checkout HEAD^2 to only run on pull_request - Rename PHPStan step correctly (was mislabeled CodeQL) - Add Docker Hub login to benchmark job - Remove no-op pull_request trigger from ai-moderator Co-Authored-By: Claude Opus 4.6 --- .github/workflows/ai-moderator.yml | 2 - .github/workflows/check-dependencies.yml | 19 ---- .github/workflows/ci.yml | 80 +++++++++++++++-- .github/workflows/pr-scan.yml | 106 ----------------------- 4 files changed, 71 insertions(+), 136 deletions(-) delete mode 100644 .github/workflows/check-dependencies.yml delete mode 100644 .github/workflows/pr-scan.yml diff --git a/.github/workflows/ai-moderator.yml b/.github/workflows/ai-moderator.yml index d0b180985f..483f3dbeee 100644 --- a/.github/workflows/ai-moderator.yml +++ b/.github/workflows/ai-moderator.yml @@ -5,8 +5,6 @@ on: types: [opened, edited] issue_comment: types: [created, edited] - pull_request: - types: [opened, edited] pull_request_review: types: [submitted, edited] pull_request_review_comment: diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml deleted file mode 100644 index 17caf3aa6b..0000000000 --- a/.github/workflows/check-dependencies.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Check dependencies - -# Adapted from https://google.github.io/osv-scanner/github-action/#scan-on-pull-request - -on: - pull_request: - branches: [main, 1.*.x] - merge_group: - branches: [main, 1.*.x] - -permissions: - # Require writing security events to upload SARIF file to security tab - security-events: write - # Only need to read contents - contents: read - -jobs: - scan-pr: - uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@v1.7.1" \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 233b365a9a..859ace8c4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,61 @@ on: default: '' jobs: + dependencies: + name: Checks / Dependencies + if: github.event_name == 'pull_request' + permissions: + security-events: write + contents: read + uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@v2.3.3" + + security: + name: Checks / Image + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + steps: + - name: Check out code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: 'recursive' + + - name: Build the Docker image + uses: docker/build-push-action@v6 + with: + context: . + push: false + load: true + tags: pr_image:${{ github.sha }} + target: production + + - name: Run Trivy vulnerability scanner on image + uses: aquasecurity/trivy-action@0.33.1 + with: + image-ref: 'pr_image:${{ github.sha }}' + format: 'sarif' + output: 'trivy-image-results.sarif' + severity: 'CRITICAL,HIGH' + + - name: Run Trivy vulnerability scanner on source code + uses: aquasecurity/trivy-action@0.33.1 + with: + scan-type: 'fs' + scan-ref: '.' + format: 'sarif' + output: 'trivy-fs-results.sarif' + severity: 'CRITICAL,HIGH' + skip-setup-trivy: true + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v4 + if: always() + with: + sarif_file: '.' + format: name: Checks / Format runs-on: ubuntu-latest @@ -30,6 +85,7 @@ jobs: fetch-depth: 2 - run: git checkout HEAD^2 + if: github.event_name == 'pull_request' - name: Validate composer.json and composer.lock run: | @@ -48,7 +104,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v6 - - name: Run CodeQL + - name: Run PHPStan run: | docker run --rm -v $PWD:/app composer:2.8 sh -c \ "composer install --profile --ignore-platform-reqs && composer check" @@ -95,8 +151,8 @@ jobs: core.setOutput('databases', JSON.stringify(databaseChanged ? allDatabases : defaultDatabases)); core.setOutput('modes', JSON.stringify(databaseChanged ? allModes : defaultModes)); - setup: - name: Setup + build: + name: Build runs-on: ubuntu-latest steps: - name: Checkout repository @@ -138,7 +194,7 @@ jobs: unit: name: Tests / Unit runs-on: ubuntu-latest - needs: setup + needs: build permissions: contents: read pull-requests: write @@ -186,7 +242,7 @@ jobs: e2e_general: name: Tests / E2E / General runs-on: ubuntu-latest - needs: setup + needs: build permissions: contents: read pull-requests: write @@ -245,7 +301,7 @@ jobs: e2e_service: name: Tests / E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service }} runs-on: ubuntu-latest - needs: [setup, matrix] + needs: [build, matrix] permissions: contents: read pull-requests: write @@ -363,7 +419,7 @@ jobs: e2e_abuse: name: Tests / E2E / Abuse (${{ matrix.mode }}) runs-on: ubuntu-latest - needs: [setup, matrix] + needs: [build, matrix] permissions: contents: read pull-requests: write @@ -422,7 +478,7 @@ jobs: e2e_screenshots: name: Tests / E2E / Screenshots (${{ matrix.mode }}) runs-on: ubuntu-latest - needs: [setup, matrix] + needs: [build, matrix] permissions: contents: read pull-requests: write @@ -488,7 +544,7 @@ jobs: benchmark: name: Benchmark runs-on: ubuntu-latest - needs: setup + needs: build permissions: pull-requests: write steps: @@ -502,6 +558,12 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Load and Start Appwrite run: | sed -i 's/traefik/localhost/g' .env diff --git a/.github/workflows/pr-scan.yml b/.github/workflows/pr-scan.yml deleted file mode 100644 index 51f3460d03..0000000000 --- a/.github/workflows/pr-scan.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: PR Security Scan -on: - pull_request_target: - types: [opened, synchronize, reopened] - -jobs: - scan: - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - steps: - - name: Check out code - uses: actions/checkout@v6 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - submodules: 'recursive' - - - name: Build the Docker image - uses: docker/build-push-action@v6 - with: - context: . - push: false - load: true - tags: pr_image:${{ github.sha }} - target: production - - - name: Run Trivy vulnerability scanner on image - uses: aquasecurity/trivy-action@0.20.0 - with: - image-ref: 'pr_image:${{ github.sha }}' - format: 'json' - output: 'trivy-image-results.json' - severity: 'CRITICAL,HIGH' - - - name: Run Trivy vulnerability scanner on source code - uses: aquasecurity/trivy-action@0.20.0 - with: - scan-type: 'fs' - scan-ref: '.' - format: 'json' - output: 'trivy-fs-results.json' - severity: 'CRITICAL,HIGH' - - - name: Process Trivy scan results - id: process-results - uses: actions/github-script@v8 - with: - script: | - const fs = require('fs'); - let commentBody = '## Security Scan Results for PR\n\n'; - - function processResults(results, title) { - let sectionBody = `### ${title}\n\n`; - if (results.Results && results.Results.some(result => result.Vulnerabilities && result.Vulnerabilities.length > 0)) { - sectionBody += '| Package | Version | Vulnerability | Severity |\n'; - sectionBody += '|---------|---------|----------------|----------|\n'; - - const uniqueVulns = new Set(); - results.Results.forEach(result => { - if (result.Vulnerabilities) { - result.Vulnerabilities.forEach(vuln => { - const vulnKey = `${vuln.PkgName}-${vuln.InstalledVersion}-${vuln.VulnerabilityID}`; - if (!uniqueVulns.has(vulnKey)) { - uniqueVulns.add(vulnKey); - sectionBody += `| ${vuln.PkgName} | ${vuln.InstalledVersion} | [${vuln.VulnerabilityID}](https://nvd.nist.gov/vuln/detail/${vuln.VulnerabilityID}) | ${vuln.Severity} |\n`; - } - }); - } - }); - } else { - sectionBody += '🎉 No vulnerabilities found!\n'; - } - return sectionBody; - } - - try { - const imageResults = JSON.parse(fs.readFileSync('trivy-image-results.json', 'utf8')); - const fsResults = JSON.parse(fs.readFileSync('trivy-fs-results.json', 'utf8')); - - commentBody += processResults(imageResults, "Docker Image Scan Results"); - commentBody += '\n'; - commentBody += processResults(fsResults, "Source Code Scan Results"); - - } catch (error) { - commentBody += `There was an error while running the security scan: ${error.message}\n`; - commentBody += 'Please contact the core team for assistance.'; - } - - core.setOutput('comment-body', commentBody); - - name: Find Comment - uses: peter-evans/find-comment@v3 - id: fc - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: Security Scan Results for PR - - - name: Create or update comment - uses: peter-evans/create-or-update-comment@v3 - with: - issue-number: ${{ github.event.pull_request.number }} - comment-id: ${{ steps.fc.outputs.comment-id }} - body: ${{ steps.process-results.outputs.comment-body }} - edit-mode: replace From e67ed2660a89bcb92a15726983b80f110fe91802 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:00:44 +0000 Subject: [PATCH 50/57] Add actions: read permission for osv-scanner reusable workflow Co-Authored-By: Claude Opus 4.6 --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 859ace8c4f..f99ee24513 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ jobs: name: Checks / Dependencies if: github.event_name == 'pull_request' permissions: + actions: read security-events: write contents: read uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@v2.3.3" From 26326d05e93dbc643852def0adad296eb407c316 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:04:33 +0000 Subject: [PATCH 51/57] Guard SARIF upload against missing files from failed Trivy scans Co-Authored-By: Claude Opus 4.6 --- .github/workflows/ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f99ee24513..4aa874a286 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,9 +70,17 @@ jobs: severity: 'CRITICAL,HIGH' skip-setup-trivy: true + - name: Check for SARIF files + id: sarif-check + if: always() + run: | + if ls *.sarif 1>/dev/null 2>&1; then + echo "exists=true" >> $GITHUB_OUTPUT + fi + - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v4 - if: always() + if: always() && steps.sarif-check.outputs.exists == 'true' with: sarif_file: '.' From e99f682cd655db92256cb3d3ff37faeebb6b12c0 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:27:38 +0000 Subject: [PATCH 52/57] Update trivy-action to v0.35.0 Co-Authored-By: Claude Opus 4.6 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4aa874a286..8c33a0ff08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,7 +53,7 @@ jobs: target: production - name: Run Trivy vulnerability scanner on image - uses: aquasecurity/trivy-action@0.33.1 + uses: aquasecurity/trivy-action@0.35.0 with: image-ref: 'pr_image:${{ github.sha }}' format: 'sarif' @@ -61,7 +61,7 @@ jobs: severity: 'CRITICAL,HIGH' - name: Run Trivy vulnerability scanner on source code - uses: aquasecurity/trivy-action@0.33.1 + uses: aquasecurity/trivy-action@0.35.0 with: scan-type: 'fs' scan-ref: '.' From 1abbca9318b63bf52252581755d319658626a144 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:42:28 +0000 Subject: [PATCH 53/57] Split SARIF uploads with unique categories to fix codeql-action error Co-Authored-By: Claude Opus 4.6 --- .github/workflows/ci.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c33a0ff08..e59b14e550 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,19 +70,19 @@ jobs: severity: 'CRITICAL,HIGH' skip-setup-trivy: true - - name: Check for SARIF files - id: sarif-check - if: always() - run: | - if ls *.sarif 1>/dev/null 2>&1; then - echo "exists=true" >> $GITHUB_OUTPUT - fi - - - name: Upload Trivy scan results to GitHub Security tab + - name: Upload image scan results uses: github/codeql-action/upload-sarif@v4 - if: always() && steps.sarif-check.outputs.exists == 'true' + if: always() && hashFiles('trivy-image-results.sarif') != '' with: - sarif_file: '.' + sarif_file: 'trivy-image-results.sarif' + category: 'trivy-image' + + - name: Upload source code scan results + uses: github/codeql-action/upload-sarif@v4 + if: always() && hashFiles('trivy-fs-results.sarif') != '' + with: + sarif_file: 'trivy-fs-results.sarif' + category: 'trivy-source' format: name: Checks / Format From a804cba99957beb491d57fa4ddcf0d4690dbaebc Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:01:26 +0000 Subject: [PATCH 54/57] Refactor usage metrics to stateless publisher pattern Co-Authored-By: Claude Opus 4.6 --- CONTRIBUTING.md | 45 ++-- app/cli.php | 42 +-- app/controllers/api/account.php | 26 +- app/controllers/shared/api.php | 196 +++++++------- app/init/resources.php | 231 +++++++++-------- app/worker.php | 58 +++-- src/Appwrite/Bus/Listeners/Usage.php | 43 ++-- src/Appwrite/Event/Message/Base.php | 21 ++ src/Appwrite/Event/Message/Usage.php | 49 ++++ src/Appwrite/Event/Publisher/Base.php | 33 +++ src/Appwrite/Event/Publisher/Usage.php | 39 +++ src/Appwrite/Event/StatsUsage.php | 96 ------- .../Http/Account/MFA/Challenges/Create.php | 14 +- .../Modules/Avatars/Http/Screenshots/Get.php | 8 +- .../Documents/Attribute/Decrement.php | 8 +- .../Documents/Attribute/Increment.php | 8 +- .../Collections/Documents/Bulk/Delete.php | 10 +- .../Collections/Documents/Bulk/Update.php | 10 +- .../Collections/Documents/Bulk/Upsert.php | 12 +- .../Collections/Documents/Create.php | 10 +- .../Collections/Documents/Delete.php | 8 +- .../Databases/Collections/Documents/Get.php | 8 +- .../Collections/Documents/Update.php | 8 +- .../Collections/Documents/Upsert.php | 8 +- .../Databases/Collections/Documents/XList.php | 8 +- .../Databases/Http/Databases/Delete.php | 1 - .../Http/Databases/Transactions/Update.php | 15 +- .../Databases/Http/TablesDB/Delete.php | 1 - .../Http/TablesDB/Tables/Rows/Bulk/Delete.php | 2 +- .../Http/TablesDB/Tables/Rows/Bulk/Update.php | 2 +- .../Http/TablesDB/Tables/Rows/Bulk/Upsert.php | 2 +- .../TablesDB/Tables/Rows/Column/Decrement.php | 2 +- .../TablesDB/Tables/Rows/Column/Increment.php | 2 +- .../Http/TablesDB/Tables/Rows/Create.php | 2 +- .../Http/TablesDB/Tables/Rows/Delete.php | 2 +- .../Http/TablesDB/Tables/Rows/Get.php | 2 +- .../Http/TablesDB/Tables/Rows/Update.php | 2 +- .../Http/TablesDB/Tables/Rows/Upsert.php | 2 +- .../Http/TablesDB/Tables/Rows/XList.php | 2 +- .../Http/TablesDB/Transactions/Update.php | 2 +- .../Functions/Http/Executions/Create.php | 8 +- .../Modules/Functions/Workers/Builds.php | 241 ++++++++---------- .../Health/Http/Health/Queue/Failed/Get.php | 8 +- .../Http/Health/Queue/StatsUsage/Get.php | 8 +- .../Modules/Teams/Http/Memberships/Create.php | 87 +++---- src/Appwrite/Platform/Workers/Messaging.php | 34 ++- src/Appwrite/Platform/Workers/Migrations.php | 42 +-- src/Appwrite/Platform/Workers/Webhooks.php | 33 ++- src/Appwrite/Usage/Context.php | 74 ++++++ .../Services/Migrations/MigrationsBase.php | 6 +- 50 files changed, 852 insertions(+), 729 deletions(-) create mode 100644 src/Appwrite/Event/Message/Base.php create mode 100644 src/Appwrite/Event/Message/Usage.php create mode 100644 src/Appwrite/Event/Publisher/Base.php create mode 100644 src/Appwrite/Event/Publisher/Usage.php delete mode 100644 src/Appwrite/Event/StatsUsage.php create mode 100644 src/Appwrite/Usage/Context.php diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ccc8e8372..5d7ab96a4f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -409,14 +409,16 @@ Next follow the appropriate steps below depending on whether you're adding the m **API** -In file `app/controllers/shared/api.php` On the database listener, add to an existing or create a new switch case. Add a call to the usage worker with your new metric const like so: +In file `app/controllers/shared/api.php` On the database listener, add to an existing or create a new switch case. Accumulate metrics in the usage context like so: ```php case $document->getCollection() === 'teams': - $queueForStatsUsage - ->addMetric(METRIC_TEAMS, $value); // per project + $usage->addMetric(METRIC_TEAMS, $value); // per project break; ``` + +The metrics will be automatically published by the shutdown hook at the end of the request. There is no need to manually trigger or publish. + There are cases when you need to handle metric that has a parent entity, like buckets. Files are linked to a parent bucket, you should verify you remove the files stats when you delete a bucket. @@ -425,14 +427,13 @@ In that case you need also to handle children removal using addReduce() method c ```php case $document->getCollection() === 'buckets': //buckets - $queueForStatsUsage - ->addMetric(METRIC_BUCKETS, $value); // per project + $usage->addMetric(METRIC_BUCKETS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage + $usage ->addReduce($document); } break; - + ``` In addition, you will also need to add some logic to the `reduce()` method of the Usage worker located in `/src/Appwrite/Platform/Workers/Usage.php`, like so: @@ -460,8 +461,12 @@ case $document->getCollection() === 'buckets': **Background worker** -You need to inject the usage queue in the desired worker on the constructor method +You need to inject the usage context and publisher in the desired worker on the constructor method ```php +use Appwrite\Usage\Context; +use Appwrite\Event\Publisher\Usage as UsagePublisher; +use Appwrite\Event\Message\Usage as UsageMessage; + /** * @throws Exception */ @@ -474,24 +479,32 @@ public function __construct() ->inject('dbForProject') ->inject('queueForFunctions') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') + ->inject('publisherForUsage') ->inject('log') - ->callback(fn (Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $log)); + ->callback(fn (Message $message, Database $dbForProject, Func $queueForFunctions, Event $queueForEvents, Context $usage, UsagePublisher $publisherForUsage, Log $log) => $this->action($message, $dbForProject, $queueForFunctions, $queueForEvents, $usage, $publisherForUsage, $log)); } ``` -and then trigger the queue with the new metric like so: +and then accumulate metrics, create a message, and publish like so: ```php -$queueForStatsUsage +$usage ->addMetric(METRIC_BUILDS, 1) ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace('{functionInternalId}', $function->getSequence(), METRIC_FUNCTION_ID_BUILDS), 1) + ->addMetric(str_replace('{functionInternalId}', $function->getSequence(), METRIC_FUNCTION_ID_BUILDS), 1) ->addMetric(str_replace('{functionInternalId}', $function->getSequence(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0)) - ->addMetric(str_replace('{functionInternalId}', $function->getSequence(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000) - ->setProject($project) - ->trigger(); + ->addMetric(str_replace('{functionInternalId}', $function->getSequence(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000); + +// Publish the accumulated metrics (workers don't have shutdown hooks) +$message = new UsageMessage( + project: $project, + metrics: $usage->getMetrics(), + reduce: $usage->getReduce() +); +$publisherForUsage->enqueue($message); +$usage->reset(); ``` diff --git a/app/cli.php b/app/cli.php index 052643f004..ee134b9487 100644 --- a/app/cli.php +++ b/app/cli.php @@ -4,11 +4,13 @@ require_once __DIR__ . '/init.php'; use Appwrite\Event\Certificate; use Appwrite\Event\Delete; +use Appwrite\Event\Event; use Appwrite\Event\Func; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\StatsResources; -use Appwrite\Event\StatsUsage; use Appwrite\Platform\Appwrite; use Appwrite\Runtimes\Runtimes; +use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Database\Documents\User; use Executor\Executor; use Swoole\Runtime; @@ -29,6 +31,7 @@ use Utopia\Platform\Service; use Utopia\Pools\Group; use Utopia\Queue\Broker\Pool as BrokerPool; use Utopia\Queue\Publisher; +use Utopia\Queue\Queue; use Utopia\Registry\Registry; use Utopia\System\System; use Utopia\Telemetry\Adapter\None as NoTelemetry; @@ -47,7 +50,7 @@ $platform = new Appwrite(); $args = $platform->getEnv('argv'); \array_shift($args); -if (!isset($args[0])) { +if (! isset($args[0])) { Console::error('Missing task name'); Console::exit(1); } @@ -85,6 +88,7 @@ $setResource('pools', function (Registry $register) { $setResource('authorization', function () { $authorization = new Authorization(); $authorization->disable(); + return $authorization; }, []); @@ -113,7 +117,7 @@ $setResource('dbForPlatform', function ($pools, $cache, $authorization) { $collections = Config::getParam('collections', [])['console']; $last = \array_key_last($collections); - if (!($dbForPlatform->exists($dbForPlatform->getDatabase(), $last))) { /** TODO cache ready variable using registry */ + if (! ($dbForPlatform->exists($dbForPlatform->getDatabase(), $last))) { /** TODO cache ready variable using registry */ throw new Exception('Tables not ready yet.'); } @@ -122,10 +126,10 @@ $setResource('dbForPlatform', function ($pools, $cache, $authorization) { Console::warning($err->getMessage()); sleep($sleep); } - } while ($attempts < $maxAttempts && !$ready); + } while ($attempts < $maxAttempts && ! $ready); - if (!$ready) { - throw new Exception("Console is not ready yet. Please try again later."); + if (! $ready) { + throw new Exception('Console is not ready yet. Please try again later.'); } return $dbForPlatform; @@ -163,7 +167,7 @@ $setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $c if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant((int) $project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database @@ -184,7 +188,7 @@ $setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $c if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant((int) $project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database @@ -207,8 +211,9 @@ $setResource('getLogsDB', function (Group $pools, Cache $cache, Authorization $a $database = null; return function (?Document $project = null) use ($pools, $cache, $database, $authorization) { - if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant((int)$project->getSequence()); + if ($database !== null && $project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant((int) $project->getSequence()); + return $database; } @@ -224,8 +229,8 @@ $setResource('getLogsDB', function (Group $pools, Cache $cache, Authorization $a ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); // set tenant - if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant((int)$project->getSequence()); + if ($project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant((int) $project->getSequence()); } return $database; @@ -243,15 +248,16 @@ $setResource('publisherFunctions', function (BrokerPool $publisher) { $setResource('publisherMigrations', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); -$setResource('publisherStatsUsage', function (BrokerPool $publisher) { - return $publisher; -}, ['publisher']); $setResource('publisherMessaging', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); -$setResource('queueForStatsUsage', function (Publisher $publisher) { - return new StatsUsage($publisher); -}, ['publisher']); +$setResource('usage', function () { + return new UsageContext(); +}, []); +$setResource('publisherForUsage', fn (Publisher $publisher) => new UsagePublisher( + $publisher, + new Queue(System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME)) +), ['publisher']); $setResource('queueForStatsResources', function (Publisher $publisher) { return new StatsResources($publisher); }, ['publisher']); diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index b58a9b4185..a780bfdac3 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -14,7 +14,6 @@ use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Hooks\Hooks; use Appwrite\Network\Validator\Email as EmailValidator; @@ -28,6 +27,7 @@ use Appwrite\SDK\MethodType; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Template\Template; use Appwrite\URL\URL as URLParser; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Database\Validator\Queries\Identities; @@ -2801,12 +2801,12 @@ Http::post('/v1/account/tokens/phone') ->inject('queueForMessaging') ->inject('locale') ->inject('timelimit') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('store') ->inject('proofForCode') ->inject('authorization') - ->action(function (string $userId, string $phone, Request $request, Response $response, User $user, Document $project, array $platform, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, Store $store, ProofsCode $proofForCode, Authorization $authorization) { + ->action(function (string $userId, string $phone, Request $request, Response $response, User $user, Document $project, array $platform, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale, callable $timelimit, Context $usage, array $plan, Store $store, ProofsCode $proofForCode, Authorization $authorization) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -2955,16 +2955,12 @@ Http::post('/v1/account/tokens/phone') $countryCode = $helper->parse($phone)->getCountryCode(); if (!empty($countryCode)) { - $queueForStatsUsage - ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + $usage->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } catch (NumberParseException $e) { // Ignore invalid phone number for country code stats } - $queueForStatsUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); + $usage->addMetric(METRIC_AUTH_METHOD_PHONE, 1); } $token->setAttribute('secret', $secret); @@ -4199,11 +4195,11 @@ Http::post('/v1/account/verifications/phone') ->inject('project') ->inject('locale') ->inject('timelimit') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('proofForCode') ->inject('authorization') - ->action(function (Request $request, Response $response, User $user, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Document $project, Locale $locale, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, ProofsCode $proofForCode, Authorization $authorization) { + ->action(function (Request $request, Response $response, User $user, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Document $project, Locale $locale, callable $timelimit, Context $usage, array $plan, ProofsCode $proofForCode, Authorization $authorization) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -4288,16 +4284,12 @@ Http::post('/v1/account/verifications/phone') $countryCode = $helper->parse($phone)->getCountryCode(); if (!empty($countryCode)) { - $queueForStatsUsage - ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + $usage->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } catch (NumberParseException $e) { // Ignore invalid phone number for country code stats } - $queueForStatsUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); + $usage->addMetric(METRIC_AUTH_METHOD_PHONE, 1); } $verification->setAttribute('secret', $secret); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 29ccc90179..c8824d3708 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -10,14 +10,16 @@ use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; +use Appwrite\Event\Message\Usage as UsageMessage; use Appwrite\Event\Messaging; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\EventProcessor; use Appwrite\SDK\Method; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; @@ -53,7 +55,7 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar $replace = $parts[1] ?? ''; $params = match ($namespace) { - 'user' => (array)$user, + 'user' => (array) $user, 'request' => $requestParams, default => $responsePayload, }; @@ -61,13 +63,13 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar if (array_key_exists($replace, $params)) { $replacement = $params[$replace]; // Convert to string if it's not already a string - if (!is_string($replacement)) { + if (! is_string($replacement)) { if (is_array($replacement)) { $replacement = json_encode($replacement); } elseif (is_object($replacement) && method_exists($replacement, '__toString')) { - $replacement = (string)$replacement; + $replacement = (string) $replacement; } elseif (is_scalar($replacement)) { - $replacement = (string)$replacement; + $replacement = (string) $replacement; } else { throw new Exception(Exception::GENERAL_SERVER_ERROR, "The server encountered an error while parsing the label: $label. Please create an issue on GitHub to allow us to investigate further https://github.com/appwrite/appwrite/issues/new/choose"); } @@ -75,6 +77,7 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar $label = \str_replace($find, $replacement, $label); } } + return $label; }; @@ -160,7 +163,7 @@ Http::init() $scopes = $roles[$role]['scopes']; // Step 5: API Key Authentication - if (!empty($apiKey)) { + if (! empty($apiKey)) { // Check if key is expired if ($apiKey->isExpired()) { throw new Exception(Exception::PROJECT_KEY_EXPIRED); @@ -170,7 +173,6 @@ Http::init() $role = $apiKey->getRole(); $scopes = $apiKey->getScopes(); - // Handle special app role case if ($apiKey->getRole() === User::ROLE_APPS) { // Disable authorization checks for project API keys @@ -193,19 +195,19 @@ Http::init() // For standard keys, update last accessed time if (\in_array($apiKey->getType(), [API_KEY_STANDARD, API_KEY_ORGANIZATION, API_KEY_ACCOUNT])) { $dbKey = null; - if (!empty($apiKey->getProjectId())) { + if (! empty($apiKey->getProjectId())) { $dbKey = $project->find( key: 'secret', find: $request->getHeader('x-appwrite-key', ''), subject: 'keys' ); - } elseif (!empty($apiKey->getUserId())) { + } elseif (! empty($apiKey->getUserId())) { $dbKey = $user->find( key: 'secret', find: $request->getHeader('x-appwrite-key', ''), subject: 'keys' ); - } elseif (!empty($apiKey->getTeamId())) { + } elseif (! empty($apiKey->getTeamId())) { $dbKey = $team->find( key: 'secret', find: $request->getHeader('x-appwrite-key', ''), @@ -213,9 +215,7 @@ Http::init() ); } - if (!$dbKey) { - \var_dump($apiKey); - \var_dump($request->getHeader('x-appwrite-key', '')); + if (! $dbKey) { throw new Exception(Exception::USER_UNAUTHORIZED); } @@ -233,7 +233,7 @@ Http::init() if ($sdk !== 'UNKNOWN' && $sdkValidator->isValid($sdk)) { $sdks = $dbKey->getAttribute('sdks', []); - if (!in_array($sdk, $sdks)) { + if (! in_array($sdk, $sdks)) { $sdks[] = $sdk; $updates->setAttribute('sdks', $sdks); @@ -241,14 +241,14 @@ Http::init() } } - if (!$updates->isEmpty()) { + if (! $updates->isEmpty()) { $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->updateDocument('keys', $dbKey->getId(), $updates)); - if (!empty($apiKey->getProjectId())) { + if (! empty($apiKey->getProjectId())) { $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('projects', $project->getId())); - } elseif (!empty($apiKey->getUserId())) { + } elseif (! empty($apiKey->getUserId())) { $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('users', $user->getId())); - } elseif (!empty($apiKey->getTeamId())) { + } elseif (! empty($apiKey->getTeamId())) { $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->purgeCachedDocument('teams', $team->getId())); } } @@ -285,7 +285,7 @@ Http::init() } } } // Admin User Authentication - elseif (($project->getId() === 'console' && !$team->isEmpty() && !$user->isEmpty()) || ($project->getId() !== 'console' && !$user->isEmpty() && $mode === APP_MODE_ADMIN)) { + elseif (($project->getId() === 'console' && ! $team->isEmpty() && ! $user->isEmpty()) || ($project->getId() !== 'console' && ! $user->isEmpty() && $mode === APP_MODE_ADMIN)) { $teamId = $team->getId(); $adminRoles = []; $memberships = $user->getAttribute('memberships', []); @@ -310,7 +310,7 @@ Http::init() // Useful for those who have project-specific roles but don't have team-wide role. $scopes = ['teams.read', 'projects.read']; foreach ($adminRoles as $adminRole) { - $isTeamWideRole = !str_starts_with($adminRole, 'project-'); + $isTeamWideRole = ! str_starts_with($adminRole, 'project-'); $isProjectSpecificRole = $projectId !== 'console' && str_starts_with($adminRole, 'project-' . $projectId); if ($isTeamWideRole || $isProjectSpecificRole) { @@ -348,18 +348,18 @@ Http::init() * But, for actions on resources (sites, functions, etc.) in a non-console project, we explicitly check * whether the admin user has necessary permission on the project (sites, functions, etc. don't have permissions associated to them). */ - if (empty($apiKey) && !$user->isEmpty() && $project->getId() !== 'console' && $mode === APP_MODE_ADMIN) { + if (empty($apiKey) && ! $user->isEmpty() && $project->getId() !== 'console' && $mode === APP_MODE_ADMIN) { $input = new Input(Database::PERMISSION_READ, $project->getPermissionsByType(Database::PERMISSION_READ)); $initialStatus = $authorization->getStatus(); $authorization->enable(); - if (!$authorization->isValid($input)) { + if (! $authorization->isValid($input)) { throw new Exception(Exception::PROJECT_NOT_FOUND); } $authorization->setStatus($initialStatus); } // Step 6: Update project and user last activity - if (!$project->isEmpty() && $project->getId() !== 'console') { + if (! $project->isEmpty() && $project->getId() !== 'console') { $accessedAt = $project->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $accessedAt) { $authorization->skip(fn () => $dbForPlatform->updateDocument('projects', $project->getId(), new Document([ @@ -368,12 +368,12 @@ Http::init() } } - if (!empty($user->getId())) { + if (! empty($user->getId())) { $accessedAt = $user->getAttribute('accessedAt', 0); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCESS)) > $accessedAt) { $user->setAttribute('accessedAt', DateTime::now()); - if ($project->getId() !== 'console' && APP_MODE_ADMIN !== $mode) { + if ($project->getId() !== 'console' && $mode !== APP_MODE_ADMIN) { $dbForProject->updateDocument('users', $user->getId(), new Document([ 'accessedAt' => $user->getAttribute('accessedAt') ])); @@ -397,26 +397,26 @@ Http::init() $method = $method[0]; } - if (!empty($method)) { + if (! empty($method)) { $namespace = $method->getNamespace(); if ( array_key_exists($namespace, $project->getAttribute('services', [])) - && !$project->getAttribute('services', [])[$namespace] - && !(User::isPrivileged($authorization->getRoles()) || User::isApp($authorization->getRoles())) + && ! $project->getAttribute('services', [])[$namespace] + && ! (User::isPrivileged($authorization->getRoles()) || User::isApp($authorization->getRoles())) ) { throw new Exception(Exception::GENERAL_SERVICE_DISABLED); } } // Step 9: Validate scope permissions - $allowed = (array)$route->getLabel('scope', 'none'); + $allowed = (array) $route->getLabel('scope', 'none'); if (empty(\array_intersect($allowed, $scopes))) { throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, $user->getAttribute('email', 'User') . ' (role: ' . \strtolower($roles[$role]['label']) . ') missing scopes (' . \json_encode($allowed) . ')'); } // Step 10: Check if user is blocked - if (false === $user->getAttribute('status')) { // Account is blocked + if ($user->getAttribute('status') === false) { // Account is blocked throw new Exception(Exception::USER_BLOCKED); } @@ -434,7 +434,7 @@ Http::init() $minimumFactors = ($mfaEnabled && $hasMoreFactors) ? 2 : 1; // Step 13: Handle Multi-Factor Authentication - if (!in_array('mfa', $route->getGroups())) { + if (! in_array('mfa', $route->getGroups())) { if ($session && \count($session->getAttribute('factors', [])) < $minimumFactors) { throw new Exception(Exception::USER_MORE_FACTORS_REQUIRED); } @@ -454,7 +454,7 @@ Http::init() ->inject('queueForDeletes') ->inject('queueForDatabase') ->inject('queueForBuilds') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForFunctions') ->inject('queueForMails') ->inject('dbForProject') @@ -467,14 +467,14 @@ Http::init() ->inject('telemetry') ->inject('platform') ->inject('authorization') - ->action(function (Http $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, StatsUsage $queueForStatsUsage, Func $queueForFunctions, Mail $queueForMails, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Telemetry $telemetry, array $platform, Authorization $authorization) { + ->action(function (Http $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Messaging $queueForMessaging, Audit $queueForAudits, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Context $usage, Func $queueForFunctions, Mail $queueForMails, Database $dbForProject, callable $timelimit, Document $resourceToken, string $mode, ?Key $apiKey, array $plan, Document $devKey, Telemetry $telemetry, array $platform, Authorization $authorization) { $route = $utopia->getRoute(); if ( array_key_exists('rest', $project->getAttribute('apis', [])) - && !$project->getAttribute('apis', [])['rest'] - && !(User::isPrivileged($authorization->getRoles()) || User::isApp($authorization->getRoles())) + && ! $project->getAttribute('apis', [])['rest'] + && ! (User::isPrivileged($authorization->getRoles()) || User::isApp($authorization->getRoles())) ) { throw new AppwriteException(AppwriteException::GENERAL_API_DISABLED); } @@ -486,7 +486,7 @@ Http::init() $abuseKeyLabel = $route->getLabel('abuse-key', 'url:{url},ip:{ip}'); $timeLimitArray = []; - $abuseKeyLabel = (!is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; + $abuseKeyLabel = (! is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; foreach ($abuseKeyLabel as $abuseKey) { $start = $request->getContentRangeStart(); @@ -499,7 +499,7 @@ Http::init() ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getHostname() . $route->getPath()) ->setParam('{method}', $request->getMethod()) - ->setParam('{chunkId}', (int)($start / ($end + 1 - $start))); + ->setParam('{chunkId}', (int) ($start / ($end + 1 - $start))); $timeLimitArray[] = $timeLimit; } @@ -511,7 +511,7 @@ Http::init() foreach ($timeLimitArray as $timeLimit) { foreach ($request->getParams() as $key => $value) { // Set request params as potential abuse keys - if (!empty($value)) { + if (! empty($value)) { $timeLimit->setParam('{param-' . $key . '}', (\is_array($value)) ? \json_encode($value) : $value); } } @@ -534,8 +534,8 @@ Http::init() if ( $enabled // Abuse is enabled - && !$isAppUser // User is not API key - && !$isPrivilegedUser // User is not an admin + && ! $isAppUser // User is not API key + && ! $isPrivilegedUser // User is not an admin && $devKey->isEmpty() // request doesn't not contain development key && $abuse->check() // Route is rate-limited ) { @@ -564,19 +564,13 @@ Http::init() ->setProject($project); /* If a session exists, use the user associated with the session */ - if (!$user->isEmpty()) { + if (! $user->isEmpty()) { $userClone = clone $user; // $user doesn't support `type` and can cause unintended effects. $userClone->setAttribute('type', ACTIVITY_TYPE_USER); $queueForAudits->setUser($userClone); } - if (!empty($apiKey) && !empty($apiKey->getDisabledMetrics())) { - foreach ($apiKey->getDisabledMetrics() as $key) { - $queueForStatsUsage->disableMetric($key); - } - } - /* Auto-set projects */ $queueForDeletes->setProject($project); $queueForDatabase->setProject($project); @@ -590,69 +584,64 @@ Http::init() $queueForBuilds->setPlatform($platform); $queueForMails->setPlatform($platform); - $useCache = $route->getLabel('cache', false); $storageCacheOperationsCounter = $telemetry->createCounter('storage.cache.operations.load'); if ($useCache) { $route = $utopia->match($request); $isImageTransformation = $route->getPath() === '/v1/storage/buckets/:bucketId/files/:fileId/preview'; - $isDisabled = isset($plan['imageTransformations']) && $plan['imageTransformations'] === -1 && !User::isPrivileged($authorization->getRoles()); + $isDisabled = isset($plan['imageTransformations']) && $plan['imageTransformations'] === -1 && ! User::isPrivileged($authorization->getRoles()); $key = $request->cacheIdentifier(); - $cacheLog = $authorization->skip(fn () => $dbForProject->getDocument('cache', $key)); + $cacheLog = $authorization->skip(fn () => $dbForProject->getDocument('cache', $key)); $cache = new Cache( new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-' . $project->getId()) ); $timestamp = 60 * 60 * 24 * 180; // Temporarily increase the TTL to 180 day to ensure files in the cache are still fetched. $data = $cache->load($key, $timestamp); - if (!empty($data) && !$cacheLog->isEmpty()) { - $usageMetric = $route->getLabel('usage.metric', null); - if ($usageMetric === METRIC_AVATARS_SCREENSHOTS_GENERATED) { - $queueForStatsUsage->disableMetric(METRIC_AVATARS_SCREENSHOTS_GENERATED); - } + if (! empty($data) && ! $cacheLog->isEmpty()) { $parts = explode('/', $cacheLog->getAttribute('resourceType', '')); $type = $parts[0] ?? null; - if ($type === 'bucket' && (!$isImageTransformation || !$isDisabled)) { + if ($type === 'bucket' && (! $isImageTransformation || ! $isDisabled)) { $bucketId = $parts[1] ?? null; $bucket = $authorization->skip(fn () => $dbForProject->getDocument('buckets', $bucketId)); - $isToken = !$resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); + $isToken = ! $resourceToken->isEmpty() && $resourceToken->getAttribute('bucketInternalId') === $bucket->getSequence(); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); - if ($bucket->isEmpty() || (!$bucket->getAttribute('enabled') && !$isAppUser && !$isPrivilegedUser)) { + if ($bucket->isEmpty() || (! $bucket->getAttribute('enabled') && ! $isAppUser && ! $isPrivilegedUser)) { throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND); } - if (!$bucket->getAttribute('transformations', true) && !$isAppUser && !$isPrivilegedUser) { + if (! $bucket->getAttribute('transformations', true) && ! $isAppUser && ! $isPrivilegedUser) { throw new Exception(Exception::STORAGE_BUCKET_TRANSFORMATIONS_DISABLED); } $fileSecurity = $bucket->getAttribute('fileSecurity', false); $valid = $authorization->isValid(new Input(Database::PERMISSION_READ, $bucket->getRead())); - if (!$fileSecurity && !$valid && !$isToken) { + if (! $fileSecurity && ! $valid && ! $isToken) { throw new Exception(Exception::USER_UNAUTHORIZED); } $parts = explode('/', $cacheLog->getAttribute('resource')); $fileId = $parts[1] ?? null; - if ($fileSecurity && !$valid && !$isToken) { + if ($fileSecurity && ! $valid && ! $isToken) { $file = $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId); } else { $file = $authorization->skip(fn () => $dbForProject->getDocument('bucket_' . $bucket->getSequence(), $fileId)); } - if (!$resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { + if (! $resourceToken->isEmpty() && $resourceToken->getAttribute('fileInternalId') !== $file->getSequence()) { throw new Exception(Exception::USER_UNAUTHORIZED); } if ($file->isEmpty()) { throw new Exception(Exception::STORAGE_FILE_NOT_FOUND); } - //Do not update transformedAt if it's a console user - if (!User::isPrivileged($authorization->getRoles())) { + // Do not update transformedAt if it's a console user + if (! User::isPrivileged($authorization->getRoles())) { $transformedAt = $file->getAttribute('transformedAt', ''); if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_PROJECT_ACCESS)) > $transformedAt) { $file->setAttribute('transformedAt', DateTime::now()); @@ -668,7 +657,7 @@ Http::init() ->addHeader('X-Appwrite-Cache', 'hit') ->setContentType($cacheLog->getAttribute('mimeType')); $storageCacheOperationsCounter->add(1, ['result' => 'hit']); - if (!$isImageTransformation || !$isDisabled) { + if (! $isImageTransformation || ! $isDisabled) { $response->send($data); } } else { @@ -691,7 +680,7 @@ Http::init() return; } - if (!$user->isEmpty()) { + if (! $user->isEmpty()) { throw new Exception(Exception::USER_SESSION_ALREADY_EXISTS); } }); @@ -745,7 +734,8 @@ Http::shutdown() ->inject('user') ->inject('queueForEvents') ->inject('queueForAudits') - ->inject('queueForStatsUsage') + ->inject('usage') + ->inject('publisherForUsage') ->inject('queueForDeletes') ->inject('queueForDatabase') ->inject('queueForBuilds') @@ -758,11 +748,12 @@ Http::shutdown() ->inject('timelimit') ->inject('eventProcessor') ->inject('bus') - ->action(function (Http $utopia, Request $request, Response $response, Document $project, User $user, Event $queueForEvents, Audit $queueForAudits, StatsUsage $queueForStatsUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, Authorization $authorization, callable $timelimit, EventProcessor $eventProcessor, Bus $bus) use ($parseLabel) { + ->inject('apiKey') + ->action(function (Http $utopia, Request $request, Response $response, Document $project, User $user, Event $queueForEvents, Audit $queueForAudits, Context $usage, UsagePublisher $publisherForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Func $queueForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, Authorization $authorization, callable $timelimit, EventProcessor $eventProcessor, Bus $bus, ?Key $apiKey) use ($parseLabel) { $responsePayload = $response->getPayload(); - if (!empty($queueForEvents->getEvent())) { + if (! empty($queueForEvents->getEvent())) { if (empty($queueForEvents->getPayload())) { $queueForEvents->setPayload($responsePayload); } @@ -784,7 +775,7 @@ Http::shutdown() } // Only trigger functions if there are matching function events - if (!empty($functionsEvents)) { + if (! empty($functionsEvents)) { foreach ($generatedEvents as $event) { if (isset($functionsEvents[$event])) { $queueForFunctions @@ -796,7 +787,7 @@ Http::shutdown() } // Only trigger webhooks if there are matching webhook events - if (!empty($webhooksEvents)) { + if (! empty($webhooksEvents)) { foreach ($generatedEvents as $event) { if (isset($webhooksEvents[$event])) { $queueForWebhooks @@ -820,7 +811,7 @@ Http::shutdown() if ($abuseEnabled && \count($abuseResetCode) > 0 && \in_array($response->getStatusCode(), $abuseResetCode)) { $abuseKeyLabel = $route->getLabel('abuse-key', 'url:{url},ip:{ip}'); - $abuseKeyLabel = (!is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; + $abuseKeyLabel = (! is_array($abuseKeyLabel)) ? [$abuseKeyLabel] : $abuseKeyLabel; foreach ($abuseKeyLabel as $abuseKey) { $start = $request->getContentRangeStart(); @@ -833,10 +824,10 @@ Http::shutdown() ->setParam('{ip}', $request->getIP()) ->setParam('{url}', $request->getHostname() . $route->getPath()) ->setParam('{method}', $request->getMethod()) - ->setParam('{chunkId}', (int)($start / ($end + 1 - $start))); + ->setParam('{chunkId}', (int) ($start / ($end + 1 - $start))); foreach ($request->getParams() as $key => $value) { // Set request params as potential abuse keys - if (!empty($value)) { + if (! empty($value)) { $timeLimit->setParam('{param-' . $key . '}', (\is_array($value)) ? \json_encode($value) : $value); } } @@ -850,14 +841,14 @@ Http::shutdown() * Audit labels */ $pattern = $route->getLabel('audits.resource', null); - if (!empty($pattern)) { + if (! empty($pattern)) { $resource = $parseLabel($pattern, $responsePayload, $requestParams, $user); - if (!empty($resource) && $resource !== $pattern) { + if (! empty($resource) && $resource !== $pattern) { $queueForAudits->setResource($resource); } } - if (!$user->isEmpty()) { + if (! $user->isEmpty()) { $userClone = clone $user; // $user doesn't support `type` and can cause unintended effects. $userClone->setAttribute('type', ACTIVITY_TYPE_USER); @@ -883,13 +874,13 @@ Http::shutdown() $queueForAudits->setUser($user); } - if (!empty($queueForAudits->getResource()) && !$queueForAudits->getUser()->isEmpty()) { + if (! empty($queueForAudits->getResource()) && ! $queueForAudits->getUser()->isEmpty()) { /** * audits.payload is switched to default true * in order to auto audit payload for all endpoints */ $pattern = $route->getLabel('audits.payload', true); - if (!empty($pattern)) { + if (! empty($pattern)) { $queueForAudits->setPayload($responsePayload); } @@ -900,19 +891,19 @@ Http::shutdown() $queueForAudits->trigger(); } - if (!empty($queueForDeletes->getType())) { + if (! empty($queueForDeletes->getType())) { $queueForDeletes->trigger(); } - if (!empty($queueForDatabase->getType())) { + if (! empty($queueForDatabase->getType())) { $queueForDatabase->trigger(); } - if (!empty($queueForBuilds->getType())) { + if (! empty($queueForBuilds->getType())) { $queueForBuilds->trigger(); } - if (!empty($queueForMessaging->getType())) { + if (! empty($queueForMessaging->getType())) { $queueForMessaging->trigger(); } @@ -921,14 +912,14 @@ Http::shutdown() if ($useCache) { $resource = $resourceType = null; $data = $response->getPayload(); - if (!empty($data['payload'])) { + if (! empty($data['payload'])) { $pattern = $route->getLabel('cache.resource', null); - if (!empty($pattern)) { + if (! empty($pattern)) { $resource = $parseLabel($pattern, $responsePayload, $requestParams, $user); } $pattern = $route->getLabel('cache.resourceType', null); - if (!empty($pattern)) { + if (! empty($pattern)) { $resourceType = $parseLabel($pattern, $responsePayload, $requestParams, $user); } @@ -938,7 +929,7 @@ Http::shutdown() $key = $request->cacheIdentifier(); $signature = md5($data['payload']); - $cacheLog = $authorization->skip(fn () => $dbForProject->getDocument('cache', $key)); + $cacheLog = $authorization->skip(fn () => $dbForProject->getDocument('cache', $key)); $accessedAt = $cacheLog->getAttribute('accessedAt', 0); $now = DateTime::now(); if ($cacheLog->isEmpty()) { @@ -971,7 +962,7 @@ Http::shutdown() } if ($project->getId() !== 'console') { - if (!User::isPrivileged($authorization->getRoles())) { + if (! User::isPrivileged($authorization->getRoles())) { $bus->dispatch(new RequestCompleted( project: $project->getArrayCopy(), request: $request, @@ -979,9 +970,32 @@ Http::shutdown() )); } - $queueForStatsUsage - ->setProject($project) - ->trigger(); + // Publish usage metrics if context has data + if (! $usage->isEmpty()) { + $metrics = $usage->getMetrics(); + + // Filter out API key disabled metrics using suffix pattern matching + $disabledMetrics = $apiKey?->getDisabledMetrics() ?? []; + if (! empty($disabledMetrics)) { + $metrics = array_values(array_filter($metrics, function ($metric) use ($disabledMetrics) { + foreach ($disabledMetrics as $pattern) { + if (str_ends_with($metric['key'], $pattern)) { + return false; + } + } + + return true; + })); + } + + $message = new UsageMessage( + project: $project, + metrics: $metrics, + reduce: $usage->getReduce() + ); + + $publisherForUsage->enqueue($message); + } } }); diff --git a/app/init/resources.php b/app/init/resources.php index d5486c2a49..1bab4491a4 100644 --- a/app/init/resources.php +++ b/app/init/resources.php @@ -14,10 +14,10 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Screenshot; use Appwrite\Event\StatsResources; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; @@ -26,6 +26,7 @@ use Appwrite\Network\Cors; use Appwrite\Network\Platform; use Appwrite\Network\Validator\Origin; use Appwrite\Network\Validator\Redirect; +use Appwrite\Usage\Context as UsageContext; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; @@ -57,6 +58,7 @@ use Utopia\Logger\Log; use Utopia\Pools\Group; use Utopia\Queue\Broker\Pool as BrokerPool; use Utopia\Queue\Publisher; +use Utopia\Queue\Queue; use Utopia\Storage\Device; use Utopia\Storage\Device\AWS; use Utopia\Storage\Device\Backblaze; @@ -88,6 +90,7 @@ Http::setResource('register', fn () => $register); Http::setResource('locale', function () { $locale = new Locale(System::getEnv('_APP_LOCALE', 'en')); $locale->setFallback(System::getEnv('_APP_LOCALE', 'en')); + return $locale; }); @@ -108,9 +111,6 @@ Http::setResource('publisherFunctions', function (Publisher $publisher) { Http::setResource('publisherMigrations', function (Publisher $publisher) { return $publisher; }, ['publisher']); -Http::setResource('publisherStatsUsage', function (Publisher $publisher) { - return $publisher; -}, ['publisher']); Http::setResource('publisherMails', function (Publisher $publisher) { return $publisher; }, ['publisher']); @@ -150,9 +150,13 @@ Http::setResource('queueForWebhooks', function (Publisher $publisher) { Http::setResource('queueForRealtime', function () { return new Realtime(); }, []); -Http::setResource('queueForStatsUsage', function (Publisher $publisher) { - return new StatsUsage($publisher); -}, ['publisher']); +Http::setResource('usage', function () { + return new UsageContext(); +}, []); +Http::setResource('publisherForUsage', fn (Publisher $publisher) => new UsagePublisher( + $publisher, + new Queue(System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME)) +), ['publisher']); Http::setResource('queueForAudits', function (Publisher $publisher) { return new AuditEvent($publisher); }, ['publisher']); @@ -186,14 +190,14 @@ Http::setResource('allowedHostnames', function (array $platform, Document $proje $allowed = [...($platform['hostnames'] ?? [])]; /* Add platform configured hostnames */ - if (!$project->isEmpty() && $project->getId() !== 'console') { + if (! $project->isEmpty() && $project->getId() !== 'console') { $platforms = $project->getAttribute('platforms', []); $hostnames = Platform::getHostnames($platforms); $allowed = [...$allowed, ...$hostnames]; } /* Add the request hostname if a dev key is found */ - if (!$devKey->isEmpty()) { + if (! $devKey->isEmpty()) { $allowed[] = $request->getHostname(); } @@ -211,12 +215,12 @@ Http::setResource('allowedHostnames', function (array $platform, Document $proje } /* Allow the request origin of rule */ - if (!$rule->isEmpty() && !empty($rule->getAttribute('domain', ''))) { + if (! $rule->isEmpty() && ! empty($rule->getAttribute('domain', ''))) { $allowed[] = $rule->getAttribute('domain', ''); } /* Allow the request origin if a dev key is found */ - if (!$devKey->isEmpty() && !empty($hostname)) { + if (! $devKey->isEmpty() && ! empty($hostname)) { $allowed[] = $hostname; } @@ -229,7 +233,7 @@ Http::setResource('allowedHostnames', function (array $platform, Document $proje Http::setResource('allowedSchemes', function (array $platform, Document $project) { $allowed = [...($platform['schemas'] ?? [])]; - if (!$project->isEmpty() && $project->getId() !== 'console') { + if (! $project->isEmpty() && $project->getId() !== 'console') { /* Add hardcoded schemes */ $allowed[] = 'exp'; $allowed[] = 'appwrite-callback-' . $project->getId(); @@ -273,7 +277,7 @@ Http::setResource('rule', function (Request $request, Database $dbForPlatform, D // Temporary implementation until custom wildcard domains are an official feature // Allow trusted projects; Used for Console (website) previews - if (!$permitsCurrentProject && !$rule->isEmpty() && !empty($rule->getAttribute('projectId', ''))) { + if (! $permitsCurrentProject && ! $rule->isEmpty() && ! empty($rule->getAttribute('projectId', ''))) { $trustedProjects = []; foreach (\explode(',', System::getEnv('_APP_CONSOLE_TRUSTED_PROJECTS', '')) as $trustedProject) { if (empty($trustedProject)) { @@ -286,7 +290,7 @@ Http::setResource('rule', function (Request $request, Database $dbForPlatform, D } } - if (!$permitsCurrentProject) { + if (! $permitsCurrentProject) { return new Document(); } @@ -309,16 +313,18 @@ Http::setResource('cors', function (array $allowedHostnames) { }, ['allowedHostnames']); Http::setResource('originValidator', function (Document $devKey, array $allowedHostnames, array $allowedSchemes) { - if (!$devKey->isEmpty()) { + if (! $devKey->isEmpty()) { return new URL(); } + return new Origin($allowedHostnames, $allowedSchemes); }, ['devKey', 'allowedHostnames', 'allowedSchemes']); Http::setResource('redirectValidator', function (Document $devKey, array $allowedHostnames, array $allowedSchemes) { - if (!$devKey->isEmpty()) { + if (! $devKey->isEmpty()) { return new URL(); } + return new Redirect($allowedHostnames, $allowedSchemes); }, ['devKey', 'allowedHostnames', 'allowedSchemes']); @@ -342,12 +348,11 @@ Http::setResource('user', function (string $mode, Document $project, Document $c * overwriting the previous value. * 7. If account API key is passed, use user of the account API key as long as user ID header matches too */ - $authorization->setDefaultStatus(true); $store->setKey('a_session_' . $project->getId()); - if (APP_MODE_ADMIN === $mode) { + if ($mode === APP_MODE_ADMIN) { $store->setKey('a_session_' . $console->getId()); } @@ -362,7 +367,7 @@ Http::setResource('user', function (string $mode, Document $project, Document $c if (empty($store->getProperty('id', '')) && empty($store->getProperty('secret', ''))) { $sessionHeader = $request->getHeader('x-appwrite-session', ''); - if (!empty($sessionHeader)) { + if (! empty($sessionHeader)) { $store->decode($sessionHeader); } } @@ -382,14 +387,14 @@ Http::setResource('user', function (string $mode, Document $project, Document $c } $user = null; - if (APP_MODE_ADMIN === $mode) { + if ($mode === APP_MODE_ADMIN) { /** @var User $user */ $user = $dbForPlatform->getDocument('users', $store->getProperty('id', '')); } else { if ($project->isEmpty()) { $user = new User([]); } else { - if (!empty($store->getProperty('id', ''))) { + if (! empty($store->getProperty('id', ''))) { if ($project->getId() === 'console') { /** @var User $user */ $user = $dbForPlatform->getDocument('users', $store->getProperty('id', '')); @@ -402,16 +407,16 @@ Http::setResource('user', function (string $mode, Document $project, Document $c } if ( - !$user || + ! $user || $user->isEmpty() // Check a document has been found in the DB - || !$user->sessionVerify($store->getProperty('secret', ''), $proofForToken) + || ! $user->sessionVerify($store->getProperty('secret', ''), $proofForToken) ) { // Validate user has valid login token $user = new User([]); } $authJWT = $request->getHeader('x-appwrite-jwt', ''); - if (!empty($authJWT) && !$project->isEmpty()) { // JWT authentication - if (!$user->isEmpty()) { + if (! empty($authJWT) && ! $project->isEmpty()) { // JWT authentication + if (! $user->isEmpty()) { throw new Exception(Exception::USER_JWT_AND_COOKIE_SET); } @@ -423,7 +428,7 @@ Http::setResource('user', function (string $mode, Document $project, Document $c } $jwtUserId = $payload['userId'] ?? ''; - if (!empty($jwtUserId)) { + if (! empty($jwtUserId)) { if ($mode === APP_MODE_ADMIN) { $user = $dbForPlatform->getDocument('users', $jwtUserId); } else { @@ -431,7 +436,7 @@ Http::setResource('user', function (string $mode, Document $project, Document $c } } $jwtSessionId = $payload['sessionId'] ?? ''; - if (!empty($jwtSessionId)) { + if (! empty($jwtSessionId)) { if (empty($user->find('$id', $jwtSessionId, 'sessions'))) { // Match JWT to active token $user = new User([]); } @@ -441,22 +446,22 @@ Http::setResource('user', function (string $mode, Document $project, Document $c // Account based on account API key $accountKey = $request->getHeader('x-appwrite-key', ''); $accountKeyUserId = $request->getHeader('x-appwrite-user', ''); - if (!empty($accountKeyUserId) && !empty($accountKey)) { - if (!$user->isEmpty()) { + if (! empty($accountKeyUserId) && ! empty($accountKey)) { + if (! $user->isEmpty()) { throw new Exception(Exception::USER_API_KEY_AND_SESSION_SET); } $accountKeyUser = $dbForPlatform->getAuthorization()->skip(fn () => $dbForPlatform->getDocument('users', $accountKeyUserId)); - if (!$accountKeyUser->isEmpty()) { + if (! $accountKeyUser->isEmpty()) { $key = $accountKeyUser->find( key: 'secret', find: $accountKey, subject: 'keys' ); - if (!empty($key)) { + if (! empty($key)) { $expire = $key->getAttribute('expire'); - if (!empty($expire) && $expire < DatabaseDateTime::formatTz(DatabaseDateTime::now())) { + if (! empty($expire) && $expire < DatabaseDateTime::formatTz(DatabaseDateTime::now())) { throw new Exception(Exception::ACCOUNT_KEY_EXPIRED); } @@ -475,10 +480,9 @@ Http::setResource('project', function ($dbForPlatform, $request, $console, $auth /** @var Appwrite\Utopia\Request $request */ /** @var Utopia\Database\Database $dbForPlatform */ /** @var Utopia\Database\Document $console */ - $projectId = $request->getParam('project', $request->getHeader('x-appwrite-project', '')); // Realtime channel "project" can send project=Query array - if (!\is_string($projectId)) { + if (! \is_string($projectId)) { $projectId = $request->getHeader('x-appwrite-project', ''); } @@ -499,7 +503,7 @@ Http::setResource('session', function (User $user, Store $store, Token $proofFor $sessions = $user->getAttribute('sessions', []); $sessionId = $user->sessionVerify($store->getProperty('secret', ''), $proofForToken); - if (!$sessionId) { + if (! $sessionId) { return; } foreach ($sessions as $session) { @@ -509,7 +513,6 @@ Http::setResource('session', function (User $user, Store $store, Token $proofFor } } - return; }, ['user', 'store', 'proofForToken']); Http::setResource('store', function (): Store { @@ -533,12 +536,14 @@ Http::setResource('proofForPassword', function (): Password { Http::setResource('proofForToken', function (): Token { $token = new Token(); $token->setHash(new Sha()); + return $token; }); Http::setResource('proofForCode', function (): Code { $code = new Code(); $code->setHash(new Sha()); + return $code; }); @@ -550,7 +555,7 @@ Http::setResource('authorization', function () { return new Authorization(); }, []); -Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project, Response $response, Publisher $publisher, Publisher $publisherFunctions, Publisher $publisherWebhooks, Event $queueForEvents, Func $queueForFunctions, Webhook $queueForWebhooks, Realtime $queueForRealtime, StatsUsage $queueForStatsUsage, Authorization $authorization) { +Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project, Response $response, Publisher $publisher, Publisher $publisherFunctions, Publisher $publisherWebhooks, Event $queueForEvents, Func $queueForFunctions, Webhook $queueForWebhooks, Realtime $queueForRealtime, UsageContext $usage, Authorization $authorization) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -615,9 +620,8 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor ->from($queueForEvents) ->trigger(); - /** Trigger webhooks events only if a project has them enabled */ - if (!empty($project->getAttribute('webhooks'))) { + if (! empty($project->getAttribute('webhooks'))) { $queueForWebhooks ->from($queueForEvents) ->trigger(); @@ -636,7 +640,6 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor */ $functionsEventsCacheListener = function (string $event, Document $document, Document $project, Database $dbForProject) { - if ($document->getCollection() !== 'functions') { return; } @@ -658,7 +661,7 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor $dbForProject->getCache()->purge($cacheKey); }; - $usageDatabaseListener = function (string $event, Document $document, StatsUsage $queueForStatsUsage) { + $usageDatabaseListener = function (string $event, Document $document, UsageContext $usage) { $value = 1; switch ($event) { @@ -678,81 +681,78 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor switch (true) { case $document->getCollection() === 'teams': - $queueForStatsUsage->addMetric(METRIC_TEAMS, $value); // per project + $usage->addMetric(METRIC_TEAMS, $value); // per project break; case $document->getCollection() === 'users': - $queueForStatsUsage->addMetric(METRIC_USERS, $value); // per project + $usage->addMetric(METRIC_USERS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage->addReduce($document); + $usage->addReduce($document); } break; case $document->getCollection() === 'sessions': // sessions - $queueForStatsUsage->addMetric(METRIC_SESSIONS, $value); //per project + $usage->addMetric(METRIC_SESSIONS, $value); // per project break; case $document->getCollection() === 'databases': // databases - $queueForStatsUsage->addMetric(METRIC_DATABASES, $value); // per project + $usage->addMetric(METRIC_DATABASES, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage->addReduce($document); + $usage->addReduce($document); } break; - case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections + case str_starts_with($document->getCollection(), 'database_') && ! str_contains($document->getCollection(), 'collection'): // collections $parts = explode('_', $document->getCollection()); $databaseInternalId = $parts[1] ?? 0; - $queueForStatsUsage + $usage ->addMetric(METRIC_COLLECTIONS, $value) // per project ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value); if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage->addReduce($document); + $usage->addReduce($document); } break; - case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents + case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): // documents $parts = explode('_', $document->getCollection()); - $databaseInternalId = $parts[1] ?? 0; + $databaseInternalId = $parts[1] ?? 0; $collectionInternalId = $parts[3] ?? 0; - $queueForStatsUsage + $usage ->addMetric(METRIC_DOCUMENTS, $value) // per project ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value); // per collection break; - case $document->getCollection() === 'buckets': //buckets - $queueForStatsUsage - ->addMetric(METRIC_BUCKETS, $value); // per project + case $document->getCollection() === 'buckets': // buckets + $usage->addMetric(METRIC_BUCKETS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage + $usage ->addReduce($document); } break; case str_starts_with($document->getCollection(), 'bucket_'): // files $parts = explode('_', $document->getCollection()); - $bucketInternalId = $parts[1]; - $queueForStatsUsage + $bucketInternalId = $parts[1]; + $usage ->addMetric(METRIC_FILES, $value) // per project ->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES), $value) // per bucket ->addMetric(str_replace('{bucketInternalId}', $bucketInternalId, METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket break; case $document->getCollection() === 'functions': - $queueForStatsUsage - ->addMetric(METRIC_FUNCTIONS, $value); // per project + $usage->addMetric(METRIC_FUNCTIONS, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage + $usage ->addReduce($document); } break; case $document->getCollection() === 'sites': - $queueForStatsUsage - ->addMetric(METRIC_SITES, $value); // per project + $usage->addMetric(METRIC_SITES, $value); // per project if ($event === Database::EVENT_DOCUMENT_DELETE) { - $queueForStatsUsage + $usage ->addReduce($document); } break; case $document->getCollection() === 'deployments': - $queueForStatsUsage + $usage ->addMetric(METRIC_DEPLOYMENTS, $value) // per project ->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project ->addMetric(str_replace(['{resourceType}'], [$document->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_DEPLOYMENTS), $value) // per function @@ -772,30 +772,27 @@ Http::setResource('dbForProject', function (Group $pools, Database $dbForPlatfor $queueForWebhooks = new Webhook($publisherWebhooks); $queueForRealtime = new Realtime(); - $database - ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENTS_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENTS_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENTS_UPSERT, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage)) - ->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener( - $project, - $document, - $response, - $queueForEventsClone->from($queueForEvents), - $queueForFunctions->from($queueForEvents), - $queueForWebhooks->from($queueForEvents), - $queueForRealtime->from($queueForEvents) - )) - ->on(Database::EVENT_DOCUMENT_CREATE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)) - ->on(Database::EVENT_DOCUMENT_UPDATE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)) - ->on(Database::EVENT_DOCUMENT_DELETE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)) - ; - + ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $usage)) + ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $usage)) + ->on(Database::EVENT_DOCUMENTS_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $usage)) + ->on(Database::EVENT_DOCUMENTS_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $usage)) + ->on(Database::EVENT_DOCUMENTS_UPSERT, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $usage)) + ->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener( + $project, + $document, + $response, + $queueForEventsClone->from($queueForEvents), + $queueForFunctions->from($queueForEvents), + $queueForWebhooks->from($queueForEvents), + $queueForRealtime->from($queueForEvents) + )) + ->on(Database::EVENT_DOCUMENT_CREATE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)) + ->on(Database::EVENT_DOCUMENT_UPDATE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)) + ->on(Database::EVENT_DOCUMENT_DELETE, 'purge-function-events-cache', fn ($event, $document) => $functionsEventsCacheListener($event, $document, $project, $database)); return $database; -}, ['pools', 'dbForPlatform', 'cache', 'project', 'response', 'publisher', 'publisherFunctions', 'publisherWebhooks', 'queueForEvents', 'queueForFunctions', 'queueForWebhooks', 'queueForRealtime', 'queueForStatsUsage', 'authorization']); +}, ['pools', 'dbForPlatform', 'cache', 'project', 'response', 'publisher', 'publisherFunctions', 'publisherWebhooks', 'queueForEvents', 'queueForFunctions', 'queueForWebhooks', 'queueForRealtime', 'usage', 'authorization']); Http::setResource('dbForPlatform', function (Group $pools, Cache $cache, Authorization $authorization) { @@ -844,8 +841,7 @@ Http::setResource('getProjectDB', function (Group $pools, Database $dbForPlatfor ->setMetadata('project', $project->getId()) ->setTimeout(APP_DATABASE_TIMEOUT_MILLISECONDS_API) ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES) - ->setDocumentType('users', User::class) - ; + ->setDocumentType('users', User::class); $sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', '')); @@ -865,6 +861,7 @@ Http::setResource('getProjectDB', function (Group $pools, Database $dbForPlatfor if (isset($databases[$dsn->getHost()])) { $database = $databases[$dsn->getHost()]; $configure($database); + return $database; } @@ -881,8 +878,9 @@ Http::setResource('getLogsDB', function (Group $pools, Cache $cache, Authorizati $database = null; return function (?Document $project = null) use ($pools, $cache, $authorization, &$database) { - if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + if ($database !== null && $project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { $database->setTenant((int) $project->getSequence()); + return $database; } @@ -898,7 +896,7 @@ Http::setResource('getLogsDB', function (Group $pools, Cache $cache, Authorizati ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES); // set tenant - if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { + if ($project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { $database->setTenant((int) $project->getSequence()); } @@ -908,6 +906,7 @@ Http::setResource('getLogsDB', function (Group $pools, Cache $cache, Authorizati Http::setResource('audit', function ($dbForProject) { $adapter = new AdapterDatabase($dbForProject); + return new Audit($adapter); }, ['dbForProject']); @@ -923,6 +922,7 @@ Http::setResource('cache', function (Group $pools, Telemetry $telemetry) { $cache = new Cache(new Sharding($adapters)); $cache->setTelemetry($telemetry); + return $cache; }, ['pools', 'telemetry']); @@ -968,9 +968,9 @@ Http::setResource('deviceForBuilds', function ($project, Telemetry $telemetry) { function getDevice(string $root, string $connection = ''): Device { - $connection = !empty($connection) ? $connection : System::getEnv('_APP_CONNECTIONS_STORAGE', ''); + $connection = ! empty($connection) ? $connection : System::getEnv('_APP_CONNECTIONS_STORAGE', ''); - if (!empty($connection)) { + if (! empty($connection)) { $acl = 'private'; $device = Storage::DEVICE_LOCAL; $accessKey = ''; @@ -992,8 +992,9 @@ function getDevice(string $root, string $connection = ''): Device switch ($device) { case Storage::DEVICE_S3: - if (!empty($url)) { - $bucketRoot = (!empty($bucket) ? $bucket . '/' : '') . \ltrim($root, '/'); + if (! empty($url)) { + $bucketRoot = (! empty($bucket) ? $bucket . '/' : '') . \ltrim($root, '/'); + return new S3($bucketRoot, $accessKey, $accessSecret, $url, $region, $acl); } else { return new AWS($root, $accessKey, $accessSecret, $bucket, $region, $acl); @@ -1002,6 +1003,7 @@ function getDevice(string $root, string $connection = ''): Device case STORAGE::DEVICE_DO_SPACES: $device = new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl); $device->setHttpVersion(S3::HTTP_VERSION_1_1); + return $device; case Storage::DEVICE_BACKBLAZE: return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl); @@ -1025,8 +1027,9 @@ function getDevice(string $root, string $connection = ''): Device $s3Bucket = System::getEnv('_APP_STORAGE_S3_BUCKET', ''); $s3Acl = 'private'; $s3EndpointUrl = System::getEnv('_APP_STORAGE_S3_ENDPOINT', ''); - if (!empty($s3EndpointUrl)) { - $bucketRoot = (!empty($s3Bucket) ? $s3Bucket . '/' : '') . \ltrim($root, '/'); + if (! empty($s3EndpointUrl)) { + $bucketRoot = (! empty($s3Bucket) ? $s3Bucket . '/' : '') . \ltrim($root, '/'); + return new S3($bucketRoot, $s3AccessKey, $s3SecretKey, $s3EndpointUrl, $s3Region, $s3Acl); } else { return new AWS($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl); @@ -1040,6 +1043,7 @@ function getDevice(string $root, string $connection = ''): Device $doSpacesAcl = 'private'; $device = new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl); $device->setHttpVersion(S3::HTTP_VERSION_1_1); + return $device; case Storage::DEVICE_BACKBLAZE: $backblazeAccessKey = System::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', ''); @@ -1047,6 +1051,7 @@ function getDevice(string $root, string $connection = ''): Device $backblazeRegion = System::getEnv('_APP_STORAGE_BACKBLAZE_REGION', ''); $backblazeBucket = System::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', ''); $backblazeAcl = 'private'; + return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl); case Storage::DEVICE_LINODE: $linodeAccessKey = System::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', ''); @@ -1054,6 +1059,7 @@ function getDevice(string $root, string $connection = ''): Device $linodeRegion = System::getEnv('_APP_STORAGE_LINODE_REGION', ''); $linodeBucket = System::getEnv('_APP_STORAGE_LINODE_BUCKET', ''); $linodeAcl = 'private'; + return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl); case Storage::DEVICE_WASABI: $wasabiAccessKey = System::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', ''); @@ -1061,6 +1067,7 @@ function getDevice(string $root, string $connection = ''): Device $wasabiRegion = System::getEnv('_APP_STORAGE_WASABI_REGION', ''); $wasabiBucket = System::getEnv('_APP_STORAGE_WASABI_BUCKET', ''); $wasabiAcl = 'private'; + return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl); } } @@ -1087,7 +1094,6 @@ Http::setResource('passwordsDictionary', function ($register) { return $register->get('passwordsDictionary'); }, ['register']); - Http::setResource('servers', function () { $platforms = Config::getParam('sdks'); $server = $platforms[APP_SDK_PLATFORM_SERVER]; @@ -1195,16 +1201,17 @@ Http::setResource('gitHub', function (Cache $cache) { }, ['cache']); Http::setResource('requestTimestamp', function ($request) { - //TODO: Move this to the Request class itself + // TODO: Move this to the Request class itself $timestampHeader = $request->getHeader('x-appwrite-timestamp'); $requestTimestamp = null; - if (!empty($timestampHeader)) { + if (! empty($timestampHeader)) { try { $requestTimestamp = new \DateTime($timestampHeader); } catch (\Throwable $e) { throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Invalid X-Appwrite-Timestamp header value'); } } + return $requestTimestamp; }, ['request']); @@ -1221,13 +1228,13 @@ Http::setResource('devKey', function (Request $request, Document $project, array // Check if given key match project's development keys $key = $project->find('secret', $devKey, 'devKeys'); - if (!$key) { + if (! $key) { return new Document([]); } // check expiration $expire = $key->getAttribute('expire'); - if (!empty($expire) && $expire < DatabaseDateTime::formatTz(DatabaseDateTime::now())) { + if (! empty($expire) && $expire < DatabaseDateTime::formatTz(DatabaseDateTime::now())) { return new Document([]); } @@ -1248,7 +1255,7 @@ Http::setResource('devKey', function (Request $request, Document $project, array if ($sdk !== 'UNKNOWN' && $sdkValidator->isValid($sdk)) { $sdks = $key->getAttribute('sdks', []); - if (!in_array($sdk, $sdks)) { + if (! in_array($sdk, $sdks)) { $sdks[] = $sdk; $key->setAttribute('sdks', $sdks); @@ -1271,7 +1278,7 @@ Http::setResource('team', function (Document $project, Database $dbForPlatform, $teamInternalId = $project->getAttribute('teamInternalId', ''); } else { $route = $utopia->match($request); - $path = !empty($route) ? $route->getPath() : $request->getURI(); + $path = ! empty($route) ? $route->getPath() : $request->getURI(); $orgHeader = $request->getHeader('x-appwrite-organization', ''); if (str_starts_with($path, '/v1/projects/:projectId')) { $uri = $request->getURI(); @@ -1286,8 +1293,9 @@ Http::setResource('team', function (Document $project, Database $dbForPlatform, } $team = $authorization->skip(fn () => $dbForPlatform->getDocument('teams', $teamId)); + return $team; - } elseif (!empty($orgHeader)) { + } elseif (! empty($orgHeader)) { return $authorization->skip(fn () => $dbForPlatform->getDocument('teams', $orgHeader)); } } @@ -1317,13 +1325,13 @@ Http::setResource('previewHostname', function (Request $request, ?Key $apiKey) { if (Http::isDevelopment()) { $allowed = true; - } elseif (!\is_null($apiKey) && $apiKey->getHostnameOverride() === true) { + } elseif (! \is_null($apiKey) && $apiKey->getHostnameOverride() === true) { $allowed = true; } if ($allowed) { $host = $request->getQuery('appwrite-hostname', $request->getHeader('x-appwrite-hostname', '')) ?? ''; - if (!empty($host)) { + if (! empty($host)) { return $host; } } @@ -1344,19 +1352,19 @@ Http::setResource('apiKey', function (Request $request, Document $project, Docum $organizationHeader = $request->getHeader('x-appwrite-organization'); $projectHeader = $request->getHeader('x-appwrite-project'); - if (!empty($key->getProjectId())) { + if (! empty($key->getProjectId())) { if (empty($projectHeader) || $projectHeader !== $key->getProjectId()) { throw new Exception(Exception::PROJECT_ID_MISSING); } } - if (!empty($key->getUserId())) { + if (! empty($key->getUserId())) { if (empty($userHeader) || $userHeader !== $key->getUserId()) { throw new Exception(Exception::USER_ID_MISSING); } } - if (!empty($key->getTeamId())) { + if (! empty($key->getTeamId())) { if (empty($organizationHeader) || $organizationHeader !== $key->getTeamId()) { throw new Exception(Exception::ORGANIZATION_ID_MISSING); } @@ -1370,7 +1378,7 @@ Http::setResource('executor', fn () => new Executor()); Http::setResource('resourceToken', function ($project, $dbForProject, $request, Authorization $authorization) { $tokenJWT = $request->getParam('token'); - if (!empty($tokenJWT) && !$project->isEmpty()) { // JWT authentication + if (! empty($tokenJWT) && ! $project->isEmpty()) { // JWT authentication // Use a large but reasonable maxAge to avoid auto-exp when token has no expiry $jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), RESOURCE_TOKEN_ALGORITHM, RESOURCE_TOKEN_MAX_AGE, RESOURCE_TOKEN_LEEWAY); // Instantiate with key, algo, maxAge and leeway. @@ -1430,6 +1438,7 @@ Http::setResource('resourceToken', function ($project, $dbForProject, $request, default => throw new Exception(Exception::TOKEN_RESOURCE_TYPE_INVALID), }; } + return new Document([]); }, ['project', 'dbForProject', 'request', 'authorization']); diff --git a/app/worker.php b/app/worker.php index 2ee1803ddc..db036b6a99 100644 --- a/app/worker.php +++ b/app/worker.php @@ -13,11 +13,12 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Screenshot; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Platform\Appwrite; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Executor\Executor; use Swoole\Runtime; @@ -42,6 +43,7 @@ use Utopia\Pools\Group; use Utopia\Queue\Broker\Pool as BrokerPool; use Utopia\Queue\Message; use Utopia\Queue\Publisher; +use Utopia\Queue\Queue; use Utopia\Queue\Server; use Utopia\Registry\Registry; use Utopia\Storage\Device\Telemetry as TelemetryDevice; @@ -58,7 +60,8 @@ Server::setResource('register', fn () => $register); Server::setResource('authorization', function () { $authorization = new Authorization(); $authorization->disable(); - return $authorization; + + return $authorization; }, []); Server::setResource('dbForPlatform', function (Cache $cache, Registry $register, Authorization $authorization) { @@ -70,9 +73,7 @@ Server::setResource('dbForPlatform', function (Cache $cache, Registry $register, ->setDatabase(APP_DATABASE) ->setAuthorization($authorization) ->setNamespace('_console') - ->setDocumentType('users', User::class) - ; - + ->setDocumentType('users', User::class); return $dbForPlatform; }, ['cache', 'register', 'authorization']); @@ -111,7 +112,7 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant((int) $project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database @@ -151,7 +152,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant((int) $project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database @@ -173,7 +174,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf if (\in_array($dsn->getHost(), $sharedTables)) { $database ->setSharedTables(true) - ->setTenant((int)$project->getSequence()) + ->setTenant((int) $project->getSequence()) ->setNamespace($dsn->getParam('namespace')); } else { $database @@ -193,9 +194,11 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf Server::setResource('getLogsDB', function (Group $pools, Cache $cache, Authorization $authorization) { $database = null; + return function (?Document $project = null) use ($pools, $cache, $database, $authorization) { - if ($database !== null && $project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant((int)$project->getSequence()); + if ($database !== null && $project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant((int) $project->getSequence()); + return $database; } @@ -211,8 +214,8 @@ Server::setResource('getLogsDB', function (Group $pools, Cache $cache, Authoriza ->setMaxQueryValues(APP_DATABASE_QUERY_MAX_VALUES_WORKER); // set tenant - if ($project !== null && !$project->isEmpty() && $project->getId() !== 'console') { - $database->setTenant((int)$project->getSequence()); + if ($project !== null && ! $project->isEmpty() && $project->getId() !== 'console') { + $database->setTenant((int) $project->getSequence()); } return $database; @@ -227,6 +230,7 @@ Server::setResource('auditRetention', function (Document $project) { if ($project->getId() === 'console') { return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE', 15778800)); // 6 months } + return DateTime::addSeconds(new \DateTime(), -1 * System::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', 1209600)); // 14 days }, ['project']); @@ -252,7 +256,7 @@ Server::setResource('redis', function () { $pass = System::getEnv('_APP_REDIS_PASS', ''); $redis = new \Redis(); - @$redis->pconnect($host, (int)$port); + @$redis->pconnect($host, (int) $port); if ($pass) { $redis->auth($pass); } @@ -269,7 +273,6 @@ Server::setResource('timelimit', function (\Redis $redis) { Server::setResource('log', fn () => new Log()); - Server::setResource('publisher', function (Group $pools) { return new BrokerPool(publisher: $pools->get('publisher')); }, ['pools']); @@ -286,10 +289,6 @@ Server::setResource('publisherMigrations', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); -Server::setResource('publisherStatsUsage', function (BrokerPool $publisher) { - return $publisher; -}, ['publisher']); - Server::setResource('publisherMessaging', function (BrokerPool $publisher) { return $publisher; }, ['publisher']); @@ -310,9 +309,13 @@ Server::setResource('consumerStatsUsage', function (BrokerPool $consumer) { return $consumer; }, ['consumer']); -Server::setResource('queueForStatsUsage', function (Publisher $publisher) { - return new StatsUsage($publisher); -}, ['publisher']); +Server::setResource('usage', function () { + return new Context(); +}, []); +Server::setResource('publisherForUsage', fn (Publisher $publisher) => new UsagePublisher( + $publisher, + new Queue(System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME)) +), ['publisher']); Server::setResource('queueForDatabase', function (Publisher $publisher) { return new EventDatabase($publisher); @@ -354,7 +357,6 @@ Server::setResource('queueForFunctions', function (Publisher $publisher) { return new Func($publisher); }, ['publisher']); - Server::setResource('queueForRealtime', function () { return new Realtime(); }, []); @@ -484,11 +486,13 @@ Server::setResource('getAudit', function (Database $dbForPlatform, callable $get return function (Document $project) use ($dbForPlatform, $getProjectDB) { if ($project->isEmpty() || $project->getId() === 'console') { $adapter = new AdapterDatabase($dbForPlatform); + return new UtopiaAudit($adapter); } $dbForProject = $getProjectDB($project); $adapter = new AdapterDatabase($dbForProject); + return new UtopiaAudit($adapter); }; }, ['dbForPlatform', 'getProjectDB']); @@ -505,7 +509,7 @@ $pools = $register->get('pools'); $platform = new Appwrite(); $args = $platform->getEnv('argv'); -if (!isset($args[1])) { +if (! isset($args[1])) { Console::error('Missing worker name'); Console::exit(1); } @@ -530,10 +534,10 @@ try { 'workersNum' => System::getEnv('_APP_WORKERS_NUM', 1), 'connection' => $pools->get('consumer')->pop()->getResource(), 'workerName' => strtolower($workerName) ?? null, - 'queueName' => $queueName + 'queueName' => $queueName, ]); } catch (\Throwable $e) { - Console::error($e->getMessage() . ', File: ' . $e->getFile() . ', Line: ' . $e->getLine()); + Console::error($e->getMessage() . ', File: ' . $e->getFile() . ', Line: ' . $e->getLine()); } $worker = $platform->getWorker(); @@ -550,11 +554,11 @@ $worker ->inject('pools') ->inject('project') ->inject('authorization') - ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project, Authorization $authorization) use ($worker, $queueName) { + ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project, Authorization $authorization) use ($queueName) { $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); if ($logger) { - $log->setNamespace("appwrite-worker"); + $log->setNamespace('appwrite-worker'); $log->setServer(System::getEnv('_APP_LOGGING_SERVICE_IDENTIFIER', \gethostname())); $log->setVersion($version); $log->setType(Log::TYPE_ERROR); diff --git a/src/Appwrite/Bus/Listeners/Usage.php b/src/Appwrite/Bus/Listeners/Usage.php index 219287033d..48178f1dee 100644 --- a/src/Appwrite/Bus/Listeners/Usage.php +++ b/src/Appwrite/Bus/Listeners/Usage.php @@ -4,11 +4,12 @@ namespace Appwrite\Bus\Listeners; use Appwrite\Bus\Events\ExecutionCompleted; use Appwrite\Bus\Events\RequestCompleted; -use Appwrite\Event\StatsUsage; +use Appwrite\Event\Message\Usage as UsageMessage; +use Appwrite\Event\Publisher\Usage as Publisher; +use Appwrite\Usage\Context; use Utopia\Bus\Event; use Utopia\Bus\Listener; use Utopia\Database\Document; -use Utopia\Queue\Publisher; class Usage extends Listener { @@ -29,20 +30,21 @@ class Usage extends Listener { $this ->desc('Records usage metrics') - ->inject('publisherStatsUsage') + ->inject('publisherForUsage') + ->inject('usage') ->callback($this->handle(...)); } - public function handle(Event $event, Publisher $publisher): void + public function handle(Event $event, Publisher $publisherForUsage, Context $usage): void { match (true) { - $event instanceof ExecutionCompleted => $this->handleExecutionCompleted($event, $publisher), - $event instanceof RequestCompleted => $this->handleRequestCompleted($event, $publisher), + $event instanceof ExecutionCompleted => $this->handleExecutionCompleted($event, $publisherForUsage), + $event instanceof RequestCompleted => $this->handleRequestCompleted($event, $usage), default => null, }; } - private function handleExecutionCompleted(ExecutionCompleted $event, Publisher $publisher): void + private function handleExecutionCompleted(ExecutionCompleted $event, Publisher $publisherForUsage): void { $execution = new Document($event->execution); $resource = new Document($event->resource); @@ -61,9 +63,7 @@ class Usage extends Listener $compute = (int)($duration * 1000); $mbSeconds = (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $duration * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT)); - $queueForStatsUsage = new StatsUsage($publisher); - $queueForStatsUsage - ->setProject($project) + $context = (new Context()) ->addMetric(METRIC_EXECUTIONS, 1) ->addMetric(str_replace(['{resourceType}'], [$resourceType], METRIC_RESOURCE_TYPE_EXECUTIONS), 1) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$resourceType, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_EXECUTIONS), 1) @@ -72,11 +72,18 @@ class Usage extends Listener ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$resourceType, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_COMPUTE), $compute) ->addMetric(METRIC_EXECUTIONS_MB_SECONDS, $mbSeconds) ->addMetric(str_replace(['{resourceType}'], [$resourceType], METRIC_RESOURCE_TYPE_EXECUTIONS_MB_SECONDS), $mbSeconds) - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$resourceType, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_MB_SECONDS), $mbSeconds) - ->trigger(); + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$resourceType, $resourceInternalId], METRIC_RESOURCE_TYPE_ID_EXECUTIONS_MB_SECONDS), $mbSeconds); + + $message = new UsageMessage( + project: $project, + metrics: $context->getMetrics(), + reduce: $context->getReduce() + ); + + $publisherForUsage->enqueue($message); } - private function handleRequestCompleted(RequestCompleted $event, Publisher $publisher): void + private function handleRequestCompleted(RequestCompleted $event, Context $usage): void { $fileSize = 0; $file = $event->request->getFiles('file'); @@ -84,18 +91,14 @@ class Usage extends Listener $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; } - $project = new Document($event->project); $deployment = new Document($event->deployment); - $queueForStatsUsage = new StatsUsage($publisher); $inbound = $event->request->getSize() + $fileSize; $outbound = $event->response->getSize(); - $queueForStatsUsage->setProject($project); - if ($deployment->getAttribute('resourceType') === 'sites') { $siteInternalId = $deployment->getAttribute('resourceInternalId', ''); - $queueForStatsUsage + $usage ->addMetric(METRIC_SITES_REQUESTS, 1) ->addMetric(METRIC_SITES_INBOUND, $inbound) ->addMetric(METRIC_SITES_OUTBOUND, $outbound) @@ -103,12 +106,10 @@ class Usage extends Listener ->addMetric(str_replace('{siteInternalId}', $siteInternalId, METRIC_SITES_ID_INBOUND), $inbound) ->addMetric(str_replace('{siteInternalId}', $siteInternalId, METRIC_SITES_ID_OUTBOUND), $outbound); } else { - $queueForStatsUsage + $usage ->addMetric(METRIC_NETWORK_REQUESTS, 1) ->addMetric(METRIC_NETWORK_INBOUND, $inbound) ->addMetric(METRIC_NETWORK_OUTBOUND, $outbound); } - - $queueForStatsUsage->trigger(); } } diff --git a/src/Appwrite/Event/Message/Base.php b/src/Appwrite/Event/Message/Base.php new file mode 100644 index 0000000000..38b6d5edee --- /dev/null +++ b/src/Appwrite/Event/Message/Base.php @@ -0,0 +1,21 @@ + $metrics + * @param array $reduce + */ + public function __construct( + public readonly Document $project, + public readonly array $metrics, + public readonly array $reduce = [], + ) { + } + + /** + * @return array + */ + public function toArray(): array + { + return [ + 'project' => [ + '$id' => $this->project->getId(), + '$sequence' => $this->project->getSequence(), + 'database' => $this->project->getAttribute('database', ''), + ], + 'metrics' => $this->metrics, + 'reduce' => array_map(fn (Document $doc) => $doc->getArrayCopy(), $this->reduce), + ]; + } + + /** + * @param array $data + * @return static + */ + public static function fromArray(array $data): static + { + return new self( + project: new Document($data['project'] ?? []), + metrics: $data['metrics'] ?? [], + reduce: array_map(fn (array $doc) => new Document($doc), $data['reduce'] ?? []), + ); + } +} diff --git a/src/Appwrite/Event/Publisher/Base.php b/src/Appwrite/Event/Publisher/Base.php new file mode 100644 index 0000000000..2063864723 --- /dev/null +++ b/src/Appwrite/Event/Publisher/Base.php @@ -0,0 +1,33 @@ +toArray(); + + return $this->publisher->enqueue($queue, $payload); + } + + /** + * Get the size of a queue + */ + public function getQueueSize(Queue $queue, bool $failed = false): int + { + return $this->publisher->getQueueSize($queue, $failed); + } +} diff --git a/src/Appwrite/Event/Publisher/Usage.php b/src/Appwrite/Event/Publisher/Usage.php new file mode 100644 index 0000000000..104690671b --- /dev/null +++ b/src/Appwrite/Event/Publisher/Usage.php @@ -0,0 +1,39 @@ +publish($this->queue, $message); + } catch (\Throwable $th) { + Console::error('[Usage] Failed to publish usage message: ' . $th->getMessage()); + return false; + } + } + + /** + * Get the size of the usage queue + */ + public function getSize(bool $failed = false): int + { + return $this->getQueueSize($this->queue, $failed); + } +} diff --git a/src/Appwrite/Event/StatsUsage.php b/src/Appwrite/Event/StatsUsage.php deleted file mode 100644 index a944d70c94..0000000000 --- a/src/Appwrite/Event/StatsUsage.php +++ /dev/null @@ -1,96 +0,0 @@ -setQueue(System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME)) - ->setClass(System::getEnv('_APP_STATS_USAGE_CLASS_NAME', Event::STATS_USAGE_CLASS_NAME)); - } - - /** - * Add reduce. - * - * @param Document $document - * @return self - */ - public function addReduce(Document $document): self - { - $this->reduce[] = $document; - - return $this; - } - - /** - * Add metric. - * - * @param string $key - * @param int $value - * @return self - */ - public function addMetric(string $key, int $value): self - { - $this->metrics[] = [ - 'key' => $key, - 'value' => $value, - ]; - - return $this; - } - - /** - * Set disabled metrics. - * - * @param string $key - * @return self - */ - public function disableMetric(string $key): self - { - $this->disabled[] = $key; - - return $this; - } - - /** - * Prepare the payload for the event - * - * @return array - */ - protected function preparePayload(): array - { - return [ - 'project' => $this->getProject(), - 'reduce' => $this->reduce, - 'metrics' => \array_filter($this->metrics, function ($metric) { - foreach ($this->disabled as $disabledMetric) { - if (\str_ends_with($metric['key'], $disabledMetric)) { - return false; - } - } - return true; - }), - ]; - } - - public function reset(): Event - { - $this->metrics = []; - parent::reset(); - return $this; - } -} diff --git a/src/Appwrite/Platform/Modules/Account/Http/Account/MFA/Challenges/Create.php b/src/Appwrite/Platform/Modules/Account/Http/Account/MFA/Challenges/Create.php index cde6b90fd6..20a6afed2e 100644 --- a/src/Appwrite/Platform/Modules/Account/Http/Account/MFA/Challenges/Create.php +++ b/src/Appwrite/Platform/Modules/Account/Http/Account/MFA/Challenges/Create.php @@ -7,7 +7,6 @@ use Appwrite\Detector\Detector; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -15,6 +14,7 @@ use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Template\Template; +use Appwrite\Usage\Context; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use libphonenumber\NumberParseException; @@ -104,7 +104,7 @@ class Create extends Action ->inject('queueForMessaging') ->inject('queueForMails') ->inject('timelimit') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('proofForToken') ->inject('proofForCode') @@ -124,7 +124,7 @@ class Create extends Action Messaging $queueForMessaging, Mail $queueForMails, callable $timelimit, - StatsUsage $queueForStatsUsage, + Context $usage, array $plan, ProofsToken $proofForToken, ProofsCode $proofForCode @@ -201,16 +201,12 @@ class Create extends Action $countryCode = $helper->parse($phone)->getCountryCode(); if (!empty($countryCode)) { - $queueForStatsUsage - ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + $usage->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } catch (NumberParseException $e) { // Ignore invalid phone number for country code stats } - $queueForStatsUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); + $usage->addMetric(METRIC_AUTH_METHOD_PHONE, 1); break; case Type::EMAIL: if (empty(System::getEnv('_APP_SMTP_HOST'))) { diff --git a/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php b/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php index b6fd354ee3..2df12b17d1 100644 --- a/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php +++ b/src/Appwrite/Platform/Modules/Avatars/Http/Screenshots/Get.php @@ -2,7 +2,6 @@ namespace Appwrite\Platform\Modules\Avatars\Http\Screenshots; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Platform\Modules\Avatars\Http\Action; use Appwrite\SDK\AuthType; @@ -10,6 +9,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\MethodType; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Response; use Utopia\Config\Config; use Utopia\Domains\Domain; @@ -84,11 +84,11 @@ class Get extends Action ->param('quality', -1, new Range(-1, 100), 'Screenshot quality. Pass an integer between 0 to 100. Defaults to keep existing image quality.', true, example: '85') ->param('output', '', new WhiteList(\array_keys(Config::getParam('storage-outputs')), true), 'Output format type (jpeg, jpg, png, gif and webp).', true, example: 'jpeg') ->inject('response') - ->inject('queueForStatsUsage') + ->inject('usage') ->callback($this->action(...)); } - public function action(string $url, array $headers, int $viewportWidth, int $viewportHeight, float $scale, string $theme, string $userAgent, bool $fullpage, string $locale, string $timezone, float $latitude, float $longitude, float $accuracy, bool $touch, array $permissions, int $sleep, int $width, int $height, int $quality, string $output, Response $response, StatsUsage $queueForStatsUsage) + public function action(string $url, array $headers, int $viewportWidth, int $viewportHeight, float $scale, string $theme, string $userAgent, bool $fullpage, string $locale, string $timezone, float $latitude, float $longitude, float $accuracy, bool $touch, array $permissions, int $sleep, int $width, int $height, int $quality, string $output, Response $response, Context $usage) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -210,7 +210,7 @@ class Get extends Action $outputs = Config::getParam('storage-outputs'); $contentType = $outputs[$output] ?? $outputs['png']; - $queueForStatsUsage->addMetric(METRIC_AVATARS_SCREENSHOTS_GENERATED, 1); + $usage->addMetric(METRIC_AVATARS_SCREENSHOTS_GENERATED, 1); $response ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php index 85813b2354..54557eaac0 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Decrement.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Attribute; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action; use Appwrite\SDK\AuthType; @@ -11,6 +10,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use InvalidArgumentException; @@ -83,13 +83,13 @@ class Decrement extends Action ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $documentId, string $attribute, int|float $value, int|float|null $min, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage, array $plan, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, string $documentId, string $attribute, int|float $value, int|float|null $min, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, Context $usage, array $plan, Authorization $authorization): void { $isAPIKey = User::isApp($authorization->getRoles()); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); @@ -200,7 +200,7 @@ class Decrement extends Action ) ); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php index 031b5abcc6..b9c19b2d06 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Attribute/Increment.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Attribute; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action; use Appwrite\SDK\AuthType; @@ -11,6 +10,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use InvalidArgumentException; @@ -83,13 +83,13 @@ class Increment extends Action ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $documentId, string $attribute, int|float $value, int|float|null $max, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage, array $plan, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, string $documentId, string $attribute, int|float $value, int|float|null $max, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, Context $usage, array $plan, Authorization $authorization): void { $isAPIKey = User::isApp($authorization->getRoles()); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); @@ -200,7 +200,7 @@ class Increment extends Action ) ); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php index 6ab67318c7..f45b126f16 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Delete.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Bulk; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action; @@ -12,6 +11,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; use Utopia\Database\Document; @@ -76,7 +76,7 @@ class Delete extends Action ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') @@ -86,7 +86,7 @@ class Delete extends Action ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, StatsUsage $queueForStatsUsage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void + public function action(string $databaseId, string $collectionId, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Context $usage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void { $database = $dbForProject->getDocument('databases', $databaseId); if ($database->isEmpty()) { @@ -185,10 +185,10 @@ class Delete extends Action foreach ($documents as $document) { $document->setAttribute('$databaseId', $database->getId()); - $document->setAttribute('$'.$this->getCollectionsEventsContext().'Id', $collection->getId()); + $document->setAttribute('$' . $this->getCollectionsEventsContext() . 'Id', $collection->getId()); } - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $modified)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $modified)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php index d306414a89..000b59ff07 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Update.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Bulk; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action; @@ -12,6 +11,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; use Utopia\Database\Document; @@ -80,7 +80,7 @@ class Update extends Action ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') @@ -90,7 +90,7 @@ class Update extends Action ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string|array $data, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, StatsUsage $queueForStatsUsage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void + public function action(string $databaseId, string $collectionId, string|array $data, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Context $usage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void { $data = \is_string($data) ? \json_decode($data, true) @@ -216,10 +216,10 @@ class Update extends Action foreach ($documents as $document) { $document->setAttribute('$databaseId', $database->getId()); - $document->setAttribute('$'.$this->getCollectionsEventsContext().'Id', $collection->getId()); + $document->setAttribute('$' . $this->getCollectionsEventsContext() . 'Id', $collection->getId()); } - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $modified)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $modified)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php index cf30fee173..564b5ee7b6 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Bulk/Upsert.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Bulk; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents\Action; @@ -12,6 +11,7 @@ use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; use Utopia\Database\Document; @@ -78,7 +78,7 @@ class Upsert extends Action ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') @@ -88,7 +88,7 @@ class Upsert extends Action ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, array $documents, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, StatsUsage $queueForStatsUsage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void + public function action(string $databaseId, string $collectionId, array $documents, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Context $usage, Event $queueForEvents, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, EventProcessor $eventProcessor): void { $database = $dbForProject->getDocument('databases', $databaseId); if ($database->isEmpty()) { @@ -106,7 +106,7 @@ class Upsert extends Action ); if ($hasRelationships) { - throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Bulk upsert is not supported for ' . $this->getSDKNamespace() . ' with relationship attributes'); + throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Bulk upsert is not supported for ' . $this->getSDKNamespace() . ' with relationship attributes'); } foreach ($documents as $key => $document) { @@ -191,10 +191,10 @@ class Upsert extends Action foreach ($upserted as $document) { $document->setAttribute('$databaseId', $database->getId()); - $document->setAttribute('$'.$this->getCollectionsEventsContext().'Id', $collection->getId()); + $document->setAttribute('$' . $this->getCollectionsEventsContext() . 'Id', $collection->getId()); } - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $modified)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $modified)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php index 9b14122abf..0bbe7c75cf 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Create.php @@ -3,7 +3,6 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\SDK\AuthType; @@ -12,6 +11,7 @@ use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Parameter; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Response as UtopiaResponse; @@ -129,7 +129,7 @@ class Create extends Action ->inject('dbForProject') ->inject('user') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForRealtime') ->inject('queueForFunctions') ->inject('queueForWebhooks') @@ -138,7 +138,7 @@ class Create extends Action ->inject('eventProcessor') ->callback($this->action(...)); } - public function action(string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, ?array $documents, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Document $user, Event $queueForEvents, StatsUsage $queueForStatsUsage, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, Authorization $authorization, EventProcessor $eventProcessor): void + public function action(string $databaseId, string $documentId, string $collectionId, string|array $data, ?array $permissions, ?array $documents, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Document $user, Event $queueForEvents, Context $usage, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, array $plan, Authorization $authorization, EventProcessor $eventProcessor): void { $data = \is_string($data) ? \json_decode($data, true) @@ -205,7 +205,7 @@ class Create extends Action ); if ($isBulk && $hasRelationships) { - throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Bulk create is not supported for ' . $this->getSDKNamespace() .' with relationship ' . $this->getStructureContext()); + throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Bulk create is not supported for ' . $this->getSDKNamespace() . ' with relationship ' . $this->getStructureContext()); } $setPermissions = function (Document $document, ?array $permissions) use ($user, $isAPIKey, $isPrivilegedUser, $isBulk, $dbForProject, $authorization) { @@ -489,7 +489,7 @@ class Create extends Action ); } - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $operations)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $operations)); // per collection diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php index 3171fe7aaf..0996fa24ab 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php @@ -4,13 +4,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documen use Appwrite\Databases\TransactionState; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; @@ -80,7 +80,7 @@ class Delete extends Action ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') @@ -96,7 +96,7 @@ class Delete extends Action UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, - StatsUsage $queueForStatsUsage, + Context $usage, TransactionState $transactionState, array $plan, Authorization $authorization @@ -210,7 +210,7 @@ class Delete extends Action authorization: $authorization ); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1); // per collection diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Get.php index 515b7029e6..8784d30667 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Get.php @@ -3,13 +3,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents; use Appwrite\Databases\TransactionState; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; @@ -68,13 +68,13 @@ class Get extends Action ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID to read uncommitted changes within the transaction.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $documentId, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, StatsUsage $queueForStatsUsage, TransactionState $transactionState, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, string $documentId, array $queries, ?string $transactionId, UtopiaResponse $response, Database $dbForProject, Context $usage, TransactionState $transactionState, Authorization $authorization): void { $isAPIKey = User::isApp($authorization->getRoles()); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); @@ -130,7 +130,7 @@ class Get extends Action operations: $operations ); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php index 06eca79dad..ca7935dfbd 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Update.php @@ -4,13 +4,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documen use Appwrite\Databases\TransactionState; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; @@ -84,14 +84,14 @@ class Update extends Action ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?string $transactionId, ?\DateTime $requestTimestamp, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage, TransactionState $transactionState, array $plan, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?string $transactionId, ?\DateTime $requestTimestamp, UtopiaResponse $response, Database $dbForProject, Event $queueForEvents, Context $usage, TransactionState $transactionState, array $plan, Authorization $authorization): void { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -246,7 +246,7 @@ class Update extends Action $setCollection($collection, $newDocument); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php index ef76ebe7cd..dc6655dfd3 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Upsert.php @@ -4,13 +4,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documen use Appwrite\Databases\TransactionState; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Response as UtopiaResponse; @@ -88,14 +88,14 @@ class Upsert extends Action ->inject('user') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?string $transactionId, ?\DateTime $requestTimestamp, UtopiaResponse $response, Document $user, Database $dbForProject, Event $queueForEvents, StatsUsage $queueForStatsUsage, TransactionState $transactionState, array $plan, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?string $transactionId, ?\DateTime $requestTimestamp, UtopiaResponse $response, Document $user, Database $dbForProject, Event $queueForEvents, Context $usage, TransactionState $transactionState, array $plan, Authorization $authorization): void { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -256,7 +256,7 @@ class Upsert extends Action $setCollection($collection, $newDocument); - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $operations)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $operations)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php index 0c93eaf105..3384153971 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/XList.php @@ -3,13 +3,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Collections\Documents; use Appwrite\Databases\TransactionState; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Deprecated; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; @@ -75,13 +75,13 @@ class XList extends Action ->inject('response') ->inject('dbForProject') ->inject('user') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('authorization') ->callback($this->action(...)); } - public function action(string $databaseId, string $collectionId, array $queries, ?string $transactionId, bool $includeTotal, int $ttl, UtopiaResponse $response, Database $dbForProject, Document $user, StatsUsage $queueForStatsUsage, TransactionState $transactionState, Authorization $authorization): void + public function action(string $databaseId, string $collectionId, array $queries, ?string $transactionId, bool $includeTotal, int $ttl, UtopiaResponse $response, Database $dbForProject, Document $user, Context $usage, TransactionState $transactionState, Authorization $authorization): void { $isAPIKey = User::isApp($authorization->getRoles()); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); @@ -228,7 +228,7 @@ class XList extends Action ); } - $queueForStatsUsage + $usage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, max($operations, 1)) ->addMetric(str_replace('{databaseInternalId}', $database->getSequence(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Delete.php index f849de94c1..1046d7e566 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Delete.php @@ -60,7 +60,6 @@ class Delete extends Action ->inject('dbForProject') ->inject('queueForDatabase') ->inject('queueForEvents') - ->inject('queueForStatsUsage') ->callback($this->action(...)); } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php index a73ad70786..9a5a63ea91 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Transactions/Update.php @@ -5,13 +5,13 @@ namespace Appwrite\Platform\Modules\Databases\Http\Databases\Transactions; use Appwrite\Databases\TransactionState; use Appwrite\Event\Delete; use Appwrite\Event\Event; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Functions\EventProcessor; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; use Utopia\Database\Database; @@ -73,7 +73,7 @@ class Update extends Action ->inject('transactionState') ->inject('queueForDeletes') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForRealtime') ->inject('queueForFunctions') ->inject('queueForWebhooks') @@ -92,7 +92,7 @@ class Update extends Action * @param TransactionState $transactionState * @param Delete $queueForDeletes * @param Event $queueForEvents - * @param StatsUsage $queueForStatsUsage + * @param Context $usage * @param Event $queueForRealtime * @param Event $queueForFunctions * @param Event $queueForWebhooks @@ -106,7 +106,7 @@ class Update extends Action * @throws Structure * @throws \Utopia\Http\Exception */ - public function action(string $transactionId, bool $commit, bool $rollback, UtopiaResponse $response, Database $dbForProject, Document $user, TransactionState $transactionState, Delete $queueForDeletes, Event $queueForEvents, StatsUsage $queueForStatsUsage, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, Authorization $authorization, EventProcessor $eventProcessor): void + public function action(string $transactionId, bool $commit, bool $rollback, UtopiaResponse $response, Database $dbForProject, Document $user, TransactionState $transactionState, Delete $queueForDeletes, Event $queueForEvents, Context $usage, Event $queueForRealtime, Event $queueForFunctions, Event $queueForWebhooks, Authorization $authorization, EventProcessor $eventProcessor): void { if (!$commit && !$rollback) { throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Either commit or rollback must be true'); @@ -142,7 +142,7 @@ class Update extends Action $currentDocumentId = null; try { - $dbForProject->withTransaction(function () use ($dbForProject, $transactionState, $queueForDeletes, $transactionId, &$transaction, &$operations, &$totalOperations, &$databaseOperations, &$currentDocumentId, $queueForEvents, $queueForStatsUsage, $queueForRealtime, $queueForFunctions, $queueForWebhooks, $authorization) { + $dbForProject->withTransaction(function () use ($dbForProject, $transactionState, $queueForDeletes, $transactionId, &$transaction, &$operations, &$totalOperations, &$databaseOperations, &$currentDocumentId, $queueForEvents, $usage, $queueForRealtime, $queueForFunctions, $queueForWebhooks, $authorization) { $authorization->skip(fn () => $dbForProject->updateDocument('transactions', $transactionId, new Document([ 'status' => 'committing', ]))); @@ -279,11 +279,10 @@ class Update extends Action throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); } - $queueForStatsUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $totalOperations); + $usage->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $totalOperations); foreach ($databaseOperations as $sequence => $count) { - $queueForStatsUsage->addMetric( + $usage->addMetric( str_replace('{databaseInternalId}', $sequence, METRIC_DATABASE_ID_OPERATIONS_WRITES), $count ); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Delete.php index c6cd0c6999..7873d369e6 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Delete.php @@ -50,7 +50,6 @@ class Delete extends DatabaseDelete ->inject('dbForProject') ->inject('queueForDatabase') ->inject('queueForEvents') - ->inject('queueForStatsUsage') ->callback($this->action(...)); } } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Delete.php index a1695bdbc6..adaf83ccf1 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Delete.php @@ -61,7 +61,7 @@ class Delete extends DocumentsDelete ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Update.php index a6bc78b3e9..d706d1f28b 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Update.php @@ -63,7 +63,7 @@ class Update extends DocumentsUpdate ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php index 6c0815312d..58da5064f9 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Bulk/Upsert.php @@ -63,7 +63,7 @@ class Upsert extends DocumentsUpsert ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID for staging the operation.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForEvents') ->inject('queueForRealtime') ->inject('queueForFunctions') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Decrement.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Decrement.php index 63d70b40e2..e1e717e9b1 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Decrement.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Decrement.php @@ -66,7 +66,7 @@ class Decrement extends DecrementDocumentAttribute ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Increment.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Increment.php index 5beb8468d9..0b20450254 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Increment.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Column/Increment.php @@ -66,7 +66,7 @@ class Increment extends IncrementDocumentAttribute ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Create.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Create.php index 4385303ffa..fde8005d2b 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Create.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Create.php @@ -106,7 +106,7 @@ class Create extends DocumentCreate ->inject('dbForProject') ->inject('user') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForRealtime') ->inject('queueForFunctions') ->inject('queueForWebhooks') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php index bee4dc1093..1845edc307 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php @@ -68,7 +68,7 @@ class Delete extends DocumentDelete ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Get.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Get.php index f0a7fcbbc2..43b799e5b1 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Get.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Get.php @@ -57,7 +57,7 @@ class Get extends DocumentGet ->param('transactionId', null, fn (Database $dbForProject) => new Nullable(new UID($dbForProject->getAdapter()->getMaxUIDLength())), 'Transaction ID to read uncommitted changes within the transaction.', true, ['dbForProject']) ->inject('response') ->inject('dbForProject') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('authorization') ->callback($this->action(...)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Update.php index 71abb5d167..c0d90f9531 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Update.php @@ -66,7 +66,7 @@ class Update extends DocumentUpdate ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Upsert.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Upsert.php index 0bcf9f9a63..7f0aa0ad7d 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Upsert.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Upsert.php @@ -69,7 +69,7 @@ class Upsert extends DocumentUpsert ->inject('user') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('plan') ->inject('authorization') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php index 3b8ac0a70e..6e5dcd9370 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/XList.php @@ -61,7 +61,7 @@ class XList extends DocumentXList ->inject('response') ->inject('dbForProject') ->inject('user') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('transactionState') ->inject('authorization') ->callback($this->action(...)); diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php index 7da389b265..68ea2b8901 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Transactions/Update.php @@ -57,7 +57,7 @@ class Update extends TransactionsUpdate ->inject('transactionState') ->inject('queueForDeletes') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForRealtime') ->inject('queueForFunctions') ->inject('queueForWebhooks') diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php index bc506c654a..ee33abe9e1 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php @@ -6,7 +6,6 @@ use Ahc\Jwt\JWT; use Appwrite\Event\Delete as DeleteEvent; use Appwrite\Event\Event; use Appwrite\Event\Func; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Extend\Exception as AppwriteException; use Appwrite\Functions\Validator\Headers; @@ -15,6 +14,7 @@ use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response; use Executor\Executor; @@ -93,7 +93,7 @@ class Create extends Base ->inject('dbForPlatform') ->inject('user') ->inject('queueForEvents') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('queueForFunctions') ->inject('geodb') ->inject('store') @@ -121,7 +121,7 @@ class Create extends Base Database $dbForPlatform, Document $user, Event $queueForEvents, - StatsUsage $queueForStatsUsage, + Context $usage, Func $queueForFunctions, Reader $geodb, Store $store, @@ -499,7 +499,7 @@ class Create extends Base throw $th; } } finally { - $queueForStatsUsage + $usage ->addMetric(METRIC_EXECUTIONS, 1) ->addMetric(str_replace(['{resourceType}'], [RESOURCE_TYPE_FUNCTIONS], METRIC_RESOURCE_TYPE_EXECUTIONS), 1) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [RESOURCE_TYPE_FUNCTIONS, $function->getSequence()], METRIC_RESOURCE_TYPE_ID_EXECUTIONS), 1) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index c080f5d3dd..f8d8b464aa 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -5,11 +5,13 @@ namespace Appwrite\Platform\Modules\Functions\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Event; use Appwrite\Event\Func; +use Appwrite\Event\Message\Usage as UsageMessage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; use Appwrite\Event\Screenshot; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Filter\BranchDomain as BranchDomainFilter; +use Appwrite\Usage\Context; use Appwrite\Utopia\Response\Model\Deployment; use Appwrite\Vcs\Comment; use Exception; @@ -60,7 +62,8 @@ class Builds extends Action ->inject('queueForWebhooks') ->inject('queueForFunctions') ->inject('queueForRealtime') - ->inject('queueForStatsUsage') + ->inject('usage') + ->inject('publisherForUsage') ->inject('cache') ->inject('dbForProject') ->inject('deviceForFunctions') @@ -74,24 +77,6 @@ class Builds extends Action } /** - * @param Message $message - * @param Document $project - * @param Database $dbForPlatform - * @param Event $queueForEvents - * @param Screenshot $queueForScreenshots - * @param Webhook $queueForWebhooks - * @param Func $queueForFunctions - * @param Realtime $queueForRealtime - * @param StatsUsage $queueForStatsUsage - * @param Cache $cache - * @param Database $dbForProject - * @param Device $deviceForFunctions - * @param Device $deviceForSites - * @param Device $deviceForFiles - * @param Log $log - * @param Executor $executor - * @param array $plan - * @return void * @throws \Utopia\Database\Exception */ public function action( @@ -103,7 +88,8 @@ class Builds extends Action Webhook $queueForWebhooks, Func $queueForFunctions, Realtime $queueForRealtime, - StatsUsage $queueForStatsUsage, + Context $usage, + UsagePublisher $publisherForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, @@ -145,7 +131,8 @@ class Builds extends Action $queueForFunctions, $queueForRealtime, $queueForEvents, - $queueForStatsUsage, + $usage, + $publisherForUsage, $dbForPlatform, $dbForProject, $github, @@ -167,28 +154,7 @@ class Builds extends Action } /** - * @param Device $deviceForFunctions - * @param Device $deviceForSites - * @param Device $deviceForFiles - * @param Screenshot $queueForScreenshots - * @param Webhook $queueForWebhooks - * @param Func $queueForFunctions - * @param Realtime $queueForRealtime - * @param Event $queueForEvents - * @param StatsUsage $queueForStatsUsage - * @param Database $dbForPlatform - * @param Database $dbForProject - * @param GitHub $github - * @param Document $project - * @param Document $resource - * @param Document $deployment - * @param Document $template - * @param Log $log - * @param Executor $executor - * @param array $plan - * @return void * @throws \Utopia\Database\Exception - * * @throws Exception */ protected function buildDeployment( @@ -200,7 +166,8 @@ class Builds extends Action Func $queueForFunctions, Realtime $queueForRealtime, Event $queueForEvents, - StatsUsage $queueForStatsUsage, + Context $usage, + UsagePublisher $publisherForUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, @@ -272,6 +239,7 @@ class Builds extends Action if ($deployment->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } @@ -298,7 +266,7 @@ class Builds extends Action $installationId = $deployment->getAttribute('installationId', ''); $providerRepositoryId = $deployment->getAttribute('providerRepositoryId', ''); $providerCommitHash = $deployment->getAttribute('providerCommitHash', ''); - $isVcsEnabled = !empty($providerRepositoryId); + $isVcsEnabled = ! empty($providerRepositoryId); $owner = ''; $repositoryName = ''; @@ -312,7 +280,7 @@ class Builds extends Action } try { - if (!$isVcsEnabled) { + if (! $isVcsEnabled) { // Non-VCS + Template $templateRepositoryName = $template->getAttribute('repositoryName', ''); $templateOwnerName = $template->getAttribute('ownerName', ''); @@ -324,7 +292,7 @@ class Builds extends Action $templateRootDirectory = \ltrim($templateRootDirectory, '.'); $templateRootDirectory = \ltrim($templateRootDirectory, '/'); - if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateReferenceType) && !empty($templateReferenceValue)) { + if (! empty($templateRepositoryName) && ! empty($templateOwnerName) && ! empty($templateReferenceType) && ! empty($templateReferenceValue)) { $stdout = ''; $stderr = ''; @@ -358,8 +326,8 @@ class Builds extends Action $source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); $result = $localDevice->transfer($tmpPathFile, $source, $device); - if (!$result) { - throw new \Exception("Unable to move file"); + if (! $result) { + throw new \Exception('Unable to move file'); } Console::execute('rm -rf ' . \escapeshellarg($tmpTemplateDirectory), '', $stdout, $stderr); @@ -400,7 +368,7 @@ class Builds extends Action $cloneVersion = $branchName; $cloneType = GitHub::CLONE_TYPE_BRANCH; - if (!empty($commitHash)) { + if (! empty($commitHash)) { $cloneVersion = $commitHash; $cloneType = GitHub::CLONE_TYPE_COMMIT; } @@ -413,6 +381,7 @@ class Builds extends Action if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } @@ -437,7 +406,6 @@ class Builds extends Action $rootDirectory = $rootDirectoryWithoutSpaces; } - // Build from template $templateRepositoryName = $template->getAttribute('repositoryName', ''); $templateOwnerName = $template->getAttribute('ownerName', ''); @@ -449,7 +417,7 @@ class Builds extends Action $templateRootDirectory = \ltrim($templateRootDirectory, '.'); $templateRootDirectory = \ltrim($templateRootDirectory, '/'); - if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateReferenceType) && !empty($templateReferenceValue)) { + if (! empty($templateRepositoryName) && ! empty($templateOwnerName) && ! empty($templateReferenceType) && ! empty($templateReferenceValue)) { // Clone template repo $tmpTemplateDirectory = '/tmp/builds/' . $deploymentId . '/template'; @@ -468,7 +436,7 @@ class Builds extends Action Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr); // Commit and push - $exit = Console::execute('git config --global user.email '. \escapeshellarg(APP_VCS_GITHUB_EMAIL) .' && git config --global user.name '. \escapeshellarg(APP_VCS_GITHUB_USERNAME) .' && cd ' . \escapeshellarg($tmpDirectory) . ' && git checkout -b ' . \escapeshellarg($branchName) . ' && git add . && git commit -m "Create ' . \escapeshellarg($resource->getAttribute('name', '')) . ' function" && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr); + $exit = Console::execute('git config --global user.email ' . \escapeshellarg(APP_VCS_GITHUB_EMAIL) . ' && git config --global user.name ' . \escapeshellarg(APP_VCS_GITHUB_USERNAME) . ' && cd ' . \escapeshellarg($tmpDirectory) . ' && git checkout -b ' . \escapeshellarg($branchName) . ' && git add . && git commit -m "Create ' . \escapeshellarg($resource->getAttribute('name', '')) . ' function" && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr); if ($exit !== 0) { throw new \Exception('Unable to push code repository: ' . $stderr); @@ -511,7 +479,7 @@ class Builds extends Action } $directorySize = $localDevice->getDirectorySize($tmpDirectory); - $sizeLimit = (int)System::getEnv('_APP_COMPUTE_SIZE_LIMIT', '30000000'); + $sizeLimit = (int) System::getEnv('_APP_COMPUTE_SIZE_LIMIT', '30000000'); if (isset($plan['deploymentSize'])) { $sizeLimit = (int) $plan['deploymentSize'] * 1000 * 1000; @@ -529,8 +497,8 @@ class Builds extends Action $source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); $result = $localDevice->transfer($tmpPathFile, $source, $device); - if (!$result) { - throw new \Exception("Unable to move file"); + if (! $result) { + throw new \Exception('Unable to move file'); } Console::execute('rm -rf ' . \escapeshellarg($tmpPath), '', $stdout, $stderr); @@ -623,16 +591,15 @@ class Builds extends Action } $cpus = $spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT; - $memory = max($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT, $minMemory); + $memory = max($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT, $minMemory); $timeout = (int) System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900); - - $jwtExpiry = (int)System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900); + $jwtExpiry = (int) System::getEnv('_APP_COMPUTE_BUILD_TIMEOUT', 900); $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0); $apiKey = $jwtObj->encode([ 'projectId' => $project->getId(), - 'scopes' => $resource->getAttribute('scopes', []) + 'scopes' => $resource->getAttribute('scopes', []), ]); // Appwrite vars @@ -700,6 +667,7 @@ class Builds extends Action if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } @@ -721,7 +689,7 @@ class Builds extends Action $listFilesCommand .= 'echo "{APPWRITE_DETECTION_SEPARATOR_START}" && cd /usr/local/build'; // Enter output directory, if set - if (!empty($outputDirectory)) { + if (! empty($outputDirectory)) { $listFilesCommand .= ' && cd ' . \escapeshellarg($outputDirectory); } @@ -748,7 +716,7 @@ class Builds extends Action cpus: $cpus, memory: $memory, timeout: $timeout, - remove: true, + remove: true, entrypoint: $deployment->getAttribute('entrypoint', ''), destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", variables: $vars, @@ -782,6 +750,7 @@ class Builds extends Action if ($deployment->getAttribute('status') === 'canceled') { $isCanceled = true; Console::info('Ignoring realtime logs because build has been canceled'); + return; } @@ -789,7 +758,7 @@ class Builds extends Action $logs = \mb_substr($logs, 0, null, 'UTF-8'); // Do not stream logs added for SSR detection - if (!$insideSeparation) { + if (! $insideSeparation) { $separator = \strpos($logs, '{APPWRITE_DETECTION_SEPARATOR_START}'); if ($separator !== false) { $logs = \substr($logs, 0, $separator); @@ -819,19 +788,19 @@ class Builds extends Action $currentLogs = $deployment->getAttribute('buildLogs', ''); $affected = false; - $streamLogs = \str_replace("\\n", "{APPWRITE_LINEBREAK_PLACEHOLDER}", $logs); + $streamLogs = \str_replace('\\n', '{APPWRITE_LINEBREAK_PLACEHOLDER}', $logs); foreach (\explode("\n", $streamLogs) as $streamLog) { if (empty($streamLog)) { continue; } - $streamLog = \str_replace("{APPWRITE_LINEBREAK_PLACEHOLDER}", "\n", $streamLog); - $streamParts = \explode(" ", $streamLog, 2); + $streamLog = \str_replace('{APPWRITE_LINEBREAK_PLACEHOLDER}', "\n", $streamLog); + $streamParts = \explode(' ', $streamLog, 2); // TODO: use part[0] as timestamp when switching to dbForLogs for build logs $currentLogs .= $streamParts[1]; - if (!empty($streamParts[1])) { + if (! empty($streamParts[1])) { $affected = true; } } @@ -863,6 +832,7 @@ class Builds extends Action if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } @@ -870,7 +840,7 @@ class Builds extends Action throw $err; } - $buildSizeLimit = (int)System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000'); + $buildSizeLimit = (int) System::getEnv('_APP_COMPUTE_BUILD_SIZE_LIMIT', '2000000000'); if (isset($plan['buildSize'])) { $buildSizeLimit = $plan['buildSize'] * 1000 * 1000; } @@ -898,7 +868,7 @@ class Builds extends Action $deployment->setAttribute('buildLogs', $logs); $adapter = null; - if ($resource->getCollection() === 'sites' && !empty($detectionLogs)) { + if ($resource->getCollection() === 'sites' && ! empty($detectionLogs)) { $files = \explode("\n", $detectionLogs); // Parse output $files = \array_filter($files); // Remove empty $files = \array_map(fn ($file) => \trim($file), $files); // Remove whitepsaces @@ -970,9 +940,9 @@ class Builds extends Action // Check if current active deployment started later than this deployment $resource = $dbForProject->getDocument($resource->getCollection(), $resource->getId()); $currentActiveDeploymentId = $resource->getAttribute('deploymentId', ''); - if (!empty($currentActiveDeploymentId)) { + if (! empty($currentActiveDeploymentId)) { $currentActiveDeployment = $dbForProject->getDocument('deployments', $currentActiveDeploymentId); - if (!$currentActiveDeployment->isEmpty()) { + if (! $currentActiveDeployment->isEmpty()) { $currentActiveStartTime = $currentActiveDeployment->getCreatedAt(); $deploymentStartTime = $deployment->getCreatedAt(); @@ -1058,7 +1028,7 @@ class Builds extends Action if ($resource->getCollection() === 'sites') { // VCS branch $branchName = $deployment->getAttribute('providerBranch'); - if (!empty($branchName)) { + if (! empty($branchName)) { $domain = (new BranchDomainFilter())->apply([ 'branch' => $branchName, 'resourceId' => $resource->getId(), @@ -1085,7 +1055,7 @@ class Builds extends Action 'certificateId' => '', 'search' => implode(' ', [$ruleId, $domain]), 'owner' => 'Appwrite', - 'region' => $project->getAttribute('region') + 'region' => $project->getAttribute('region'), ])); } catch (Duplicate $err) { $rule = $dbForPlatform->updateDocument('rules', $ruleId, new Document([ @@ -1126,6 +1096,7 @@ class Builds extends Action if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } @@ -1139,7 +1110,7 @@ class Builds extends Action $schedule ->setAttribute('resourceUpdatedAt', DateTime::now()) ->setAttribute('schedule', $resource->getAttribute('schedule')) - ->setAttribute('active', !empty($resource->getAttribute('schedule')) && !empty($resource->getAttribute('deploymentId'))); + ->setAttribute('active', ! empty($resource->getAttribute('schedule')) && ! empty($resource->getAttribute('deploymentId'))); $dbForPlatform->updateDocument('schedules', $schedule->getId(), new Document([ 'resourceUpdatedAt' => $schedule->getAttribute('resourceUpdatedAt'), 'schedule' => $schedule->getAttribute('schedule'), @@ -1167,13 +1138,14 @@ class Builds extends Action if ($dbForProject->getDocument('deployments', $deploymentId)->getAttribute('status') === 'canceled') { $this->cancelDeployment($deployment->getId(), $dbForProject, $queueForRealtime); + return; } // Color message red $message = $th->getMessage(); - if (!\str_contains($message, '')) { - $message = "" . $message; + if (! \str_contains($message, '')) { + $message = '' . $message; } $message = \str_replace('{APPWRITE_DETECTION_SEPARATOR_START}', '', $message); @@ -1181,9 +1153,9 @@ class Builds extends Action // Combine with previous logs if deployment got past build process $previousLogs = ''; - if (!is_null($deployment->getAttribute('buildSize', null))) { + if (! is_null($deployment->getAttribute('buildSize', null))) { $previousLogs = $deployment->getAttribute('buildLogs', ''); - if (!empty($previousLogs)) { + if (! empty($previousLogs)) { $message = $previousLogs . "\n" . $message; } } @@ -1219,102 +1191,102 @@ class Builds extends Action ->trigger(); $this->sendUsage( - resource:$resource, + resource: $resource, deployment: $deployment, project: $project, - queue: $queueForStatsUsage + usage: $usage, + publisherForUsage: $publisherForUsage ); } } - protected function sendUsage(Document $resource, Document $deployment, Document $project, StatsUsage $queue): void + protected function sendUsage(Document $resource, Document $deployment, Document $project, Context $usage, UsagePublisher $publisherForUsage): void { $spec = Config::getParam('specifications')[$resource->getAttribute('buildSpecification', APP_COMPUTE_SPECIFICATION_DEFAULT)]; switch ($deployment->getAttribute('status')) { case 'ready': - $queue + $usage ->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int) $deployment->getAttribute('buildDuration', 0) * 1000) ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_SUCCESS), 1) // per function - ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_SUCCESS), (int)$deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_SUCCESS), 1) // per function - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_SUCCESS), (int)$deployment->getAttribute('buildDuration', 0) * 1000); + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_SUCCESS), (int) $deployment->getAttribute('buildDuration', 0) * 1000); break; case 'failed': - $queue + $usage ->addMetric(METRIC_BUILDS_FAILED, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int) $deployment->getAttribute('buildDuration', 0) * 1000) ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_FAILED), 1) // per function - ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_FAILED), (int)$deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_FAILED), 1) // per function - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_FAILED), (int)$deployment->getAttribute('buildDuration', 0) * 1000); + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE_FAILED), (int) $deployment->getAttribute('buildDuration', 0) * 1000); break; } - $queue + $usage ->addMetric(METRIC_BUILDS, 1) // per project ->addMetric(METRIC_BUILDS_STORAGE, $deployment->getAttribute('buildSize', 0)) - ->addMetric(METRIC_BUILDS_COMPUTE, (int)$deployment->getAttribute('buildDuration', 0) * 1000) - ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(METRIC_BUILDS_COMPUTE, (int) $deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(METRIC_BUILDS_MB_SECONDS, (int) (($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS), 1) // per function ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0)) - ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE), (int)$deployment->getAttribute('buildDuration', 0) * 1000) - ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS), (int) (($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS), 1) // per function ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0)) - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE), (int)$deployment->getAttribute('buildDuration', 0) * 1000) - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) - ->setProject($project) - ->trigger(); + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000) + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS), (int) (($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))); + + // Publish usage metrics + if (! $usage->isEmpty()) { + $message = new UsageMessage( + project: $project, + metrics: $usage->getMetrics(), + reduce: $usage->getReduce() + ); + $publisherForUsage->enqueue($message); + $usage->reset(); + } } /** * Hook to run after build success * - * @param Realtime $queueForRealtime - * @param Database $dbForProject - * @param Document $deployment - * @param array $runtime - * @param string|null $adapter - * @return void * @throws Exception */ protected function afterBuildSuccess(Realtime $queueForRealtime, Database $dbForProject, Document &$deployment, array $runtime, ?string $adapter): void { - if (!($queueForRealtime instanceof Realtime)) { + if (! ($queueForRealtime instanceof Realtime)) { throw new Exception('queueForRealtime must be an instance of Realtime'); } - if (!($dbForProject instanceof Database)) { + if (! ($dbForProject instanceof Database)) { throw new Exception('dbForProject must be an instance of Database'); } - if (!($deployment instanceof Document)) { + if (! ($deployment instanceof Document)) { throw new Exception('deployment must be an instance of Document'); } - if (!is_array($runtime)) { + if (! is_array($runtime)) { throw new Exception('runtime must be an array'); } - if (!is_string($adapter) && !is_null($adapter)) { + if (! is_string($adapter) && ! is_null($adapter)) { throw new Exception('adapter must be a string or null'); } } /** * Hook to run after deployment is activated - * - * @param Document $project - * @param Document $deployment - * @return void */ protected function afterDeploymentSuccess( Document $project, Document $deployment, ): void { - if (!($project instanceof Document)) { + if (! ($project instanceof Document)) { throw new Exception('project must be an instance of Document'); } - if (!($deployment instanceof Document)) { + if (! ($deployment instanceof Document)) { throw new Exception('deployment must be an instance of Document'); } } @@ -1322,7 +1294,7 @@ class Builds extends Action protected function getRuntime(Document $resource, string $version): array { $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); - $key = $resource->getAttribute('runtime'); + $key = $resource->getAttribute('runtime'); $runtime = match ($resource->getCollection()) { 'functions' => $runtimes[$resource->getAttribute('runtime')] ?? null, 'sites' => $runtimes[$resource->getAttribute('buildRuntime')] ?? null, @@ -1355,7 +1327,7 @@ class Builds extends Action $envCommand = ''; $bundleCommand = ''; - if (!is_null($framework)) { + if (! is_null($framework)) { $envCommand = $framework['envCommand'] ?? ''; $bundleCommand = $framework['bundleCommand'] ?? ''; } @@ -1364,7 +1336,7 @@ class Builds extends Action $commands[] = $deployment->getAttribute('buildCommands', ''); $commands[] = $bundleCommand; - $commands = array_filter($commands, fn ($command) => !empty($command)); + $commands = array_filter($commands, fn ($command) => ! empty($command)); return implode(' && ', $commands); } @@ -1373,19 +1345,6 @@ class Builds extends Action } /** - * @param string $status - * @param GitHub $github - * @param string $providerCommitHash - * @param string $owner - * @param string $repositoryName - * @param Document $project - * @param Document $resource - * @param string $deploymentId - * @param Database $dbForProject - * @param Database $dbForPlatform - * @param Realtime $queueForRealtime - * @param array $platform - * @return void * @throws Structure * @throws \Utopia\Database\Exception * @throws Conflict @@ -1413,7 +1372,7 @@ class Builds extends Action $deployment = $dbForProject->getDocument('deployments', $deploymentId); $commentId = $deployment->getAttribute('providerCommentId', ''); - if (!empty($providerCommitHash)) { + if (! empty($providerCommitHash)) { $message = match ($status) { 'ready' => 'Build succeeded.', 'failed' => 'Build failed.', @@ -1448,7 +1407,7 @@ class Builds extends Action $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, $state, $message, $providerTargetUrl, $name); } - if (!empty($commentId)) { + if (! empty($commentId)) { $retries = 0; while (true) { @@ -1456,7 +1415,7 @@ class Builds extends Action try { $dbForPlatform->createDocument('vcsCommentLocks', new Document([ - '$id' => $commentId + '$id' => $commentId, ])); break; } catch (\Throwable $err) { @@ -1470,22 +1429,22 @@ class Builds extends Action // Wrap in try/finally to ensure lock file gets deleted try { - $resourceType = match($resource->getCollection()) { + $resourceType = match ($resource->getCollection()) { 'functions' => 'function', 'sites' => 'site', default => throw new \Exception('Invalid resource type') }; $rule = $dbForPlatform->findOne('rules', [ - Query::equal("projectInternalId", [$project->getSequence()]), - Query::equal("type", ["deployment"]), - Query::equal("deploymentInternalId", [$deployment->getSequence()]), + Query::equal('projectInternalId', [$project->getSequence()]), + Query::equal('type', ['deployment']), + Query::equal('deploymentInternalId', [$deployment->getSequence()]), ]); $protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https'; - $previewUrl = match($resource->getCollection()) { + $previewUrl = match ($resource->getCollection()) { 'functions' => '', - 'sites' => !empty($rule) ? ("{$protocol}://" . $rule->getAttribute('domain', '')) : '', + 'sites' => ! empty($rule) ? ("{$protocol}://" . $rule->getAttribute('domain', '')) : '', default => throw new \Exception('Invalid resource type') }; @@ -1498,7 +1457,7 @@ class Builds extends Action } } } catch (\Throwable $th) { - Console::warning("Git action failed:"); + Console::warning('Git action failed:'); Console::warning($th->getMessage()); Console::warning($th->getTraceAsString()); diff --git a/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/Failed/Get.php b/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/Failed/Get.php index dc84d0ee37..cb3640746f 100644 --- a/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/Failed/Get.php +++ b/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/Failed/Get.php @@ -12,9 +12,9 @@ use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; use Appwrite\Event\Migration; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Screenshot; use Appwrite\Event\StatsResources; -use Appwrite\Event\StatsUsage; use Appwrite\Event\Webhook; use Appwrite\Platform\Modules\Health\Http\Health\Queue\Base; use Appwrite\SDK\AuthType; @@ -79,7 +79,7 @@ class Get extends Base ->inject('queueForMails') ->inject('queueForFunctions') ->inject('queueForStatsResources') - ->inject('queueForStatsUsage') + ->inject('publisherForUsage') ->inject('queueForWebhooks') ->inject('queueForCertificates') ->inject('queueForBuilds') @@ -99,7 +99,7 @@ class Get extends Base Mail $queueForMails, Func $queueForFunctions, StatsResources $queueForStatsResources, - StatsUsage $queueForStatsUsage, + UsagePublisher $publisherForUsage, Webhook $queueForWebhooks, Certificate $queueForCertificates, Build $queueForBuilds, @@ -116,7 +116,7 @@ class Get extends Base System::getEnv('_APP_MAILS_QUEUE_NAME', Event::MAILS_QUEUE_NAME) => $queueForMails, System::getEnv('_APP_FUNCTIONS_QUEUE_NAME', Event::FUNCTIONS_QUEUE_NAME) => $queueForFunctions, System::getEnv('_APP_STATS_RESOURCES_QUEUE_NAME', Event::STATS_RESOURCES_QUEUE_NAME) => $queueForStatsResources, - System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME) => $queueForStatsUsage, + System::getEnv('_APP_STATS_USAGE_QUEUE_NAME', Event::STATS_USAGE_QUEUE_NAME) => $publisherForUsage, System::getEnv('_APP_WEBHOOK_QUEUE_NAME', Event::WEBHOOK_QUEUE_NAME) => $queueForWebhooks, System::getEnv('_APP_CERTIFICATES_QUEUE_NAME', Event::CERTIFICATES_QUEUE_NAME) => $queueForCertificates, System::getEnv('_APP_BUILDS_QUEUE_NAME', Event::BUILDS_QUEUE_NAME) => $queueForBuilds, diff --git a/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/StatsUsage/Get.php b/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/StatsUsage/Get.php index 10678efbc3..65b3d228a6 100644 --- a/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/StatsUsage/Get.php +++ b/src/Appwrite/Platform/Modules/Health/Http/Health/Queue/StatsUsage/Get.php @@ -2,7 +2,7 @@ namespace Appwrite\Platform\Modules\Health\Http\Health\Queue\StatsUsage; -use Appwrite\Event\StatsUsage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Platform\Modules\Health\Http\Health\Queue\Base; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -42,16 +42,16 @@ class Get extends Base contentType: ContentType::JSON )) ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) - ->inject('queueForStatsUsage') + ->inject('publisherForUsage') ->inject('response') ->callback($this->action(...)); } - public function action(int|string $threshold, StatsUsage $queueForStatsUsage, Response $response): void + public function action(int|string $threshold, UsagePublisher $publisherForUsage, Response $response): void { $threshold = (int) $threshold; - $size = $queueForStatsUsage->getSize(); + $size = $publisherForUsage->getSize(); $this->assertQueueThreshold($size, $threshold); diff --git a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php index 3bf597eaca..221c3aa521 100644 --- a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php +++ b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php @@ -6,7 +6,6 @@ use Appwrite\Auth\Validator\Phone; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Network\Validator\Email as EmailValidator; use Appwrite\Platform\Action; @@ -14,6 +13,7 @@ use Appwrite\SDK\AuthType; use Appwrite\SDK\Method; use Appwrite\SDK\Response as SDKResponse; use Appwrite\Template\Template; +use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response; use libphonenumber\NumberParseException; @@ -70,7 +70,7 @@ class Create extends Action new SDKResponse( code: Response::STATUS_CODE_CREATED, model: Response::MODEL_MEMBERSHIP, - ) + ), ] )) ->label('abuse-limit', 10) @@ -91,20 +91,20 @@ class Create extends Action ->inject('queueForMessaging') ->inject('queueForEvents') ->inject('timelimit') - ->inject('queueForStatsUsage') + ->inject('usage') ->inject('plan') ->inject('proofForPassword') ->inject('proofForToken') ->callback($this->action(...)); } - public function action(string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Authorization $authorization, Locale $locale, Mail $queueForMails, Messaging $queueForMessaging, Event $queueForEvents, callable $timelimit, StatsUsage $queueForStatsUsage, array $plan, Password $proofForPassword, Token $proofForToken) + public function action(string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Authorization $authorization, Locale $locale, Mail $queueForMails, Messaging $queueForMessaging, Event $queueForEvents, callable $timelimit, Context $usage, array $plan, Password $proofForPassword, Token $proofForToken) { $isAppUser = User::isApp($authorization->getRoles()); $isPrivilegedUser = User::isPrivileged($authorization->getRoles()); if (empty($url)) { - if (!$isAppUser && !$isPrivilegedUser) { + if (! $isAppUser && ! $isPrivilegedUser) { throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'URL is required'); } } @@ -113,7 +113,7 @@ class Create extends Action throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'At least one of userId, email, or phone is required'); } - if (!$isPrivilegedUser && !$isAppUser && empty(System::getEnv('_APP_SMTP_HOST'))) { + if (! $isPrivilegedUser && ! $isAppUser && empty(System::getEnv('_APP_SMTP_HOST'))) { throw new Exception(Exception::GENERAL_SMTP_DISABLED); } @@ -124,28 +124,28 @@ class Create extends Action if ($team->isEmpty()) { throw new Exception(Exception::TEAM_NOT_FOUND); } - if (!empty($userId)) { + if (! empty($userId)) { $invitee = $dbForProject->getDocument('users', $userId); if ($invitee->isEmpty()) { throw new Exception(Exception::USER_NOT_FOUND, 'User with given userId doesn\'t exist.', 404); } - if (!empty($email) && $invitee->getAttribute('email', '') !== $email) { + if (! empty($email) && $invitee->getAttribute('email', '') !== $email) { throw new Exception(Exception::USER_ALREADY_EXISTS, 'Given userId and email doesn\'t match', 409); } - if (!empty($phone) && $invitee->getAttribute('phone', '') !== $phone) { + if (! empty($phone) && $invitee->getAttribute('phone', '') !== $phone) { throw new Exception(Exception::USER_ALREADY_EXISTS, 'Given userId and phone doesn\'t match', 409); } $email = $invitee->getAttribute('email', ''); $phone = $invitee->getAttribute('phone', ''); $name = $invitee->getAttribute('name', '') ?: $name; - } elseif (!empty($email)) { + } elseif (! empty($email)) { $invitee = $dbForProject->findOne('users', [Query::equal('email', [$email])]); // Get user by email address - if (!$invitee->isEmpty() && !empty($phone) && $invitee->getAttribute('phone', '') !== $phone) { + if (! $invitee->isEmpty() && ! empty($phone) && $invitee->getAttribute('phone', '') !== $phone) { throw new Exception(Exception::USER_ALREADY_EXISTS, 'Given email and phone doesn\'t match', 409); } - } elseif (!empty($phone)) { + } elseif (! empty($phone)) { $invitee = $dbForProject->findOne('users', [Query::equal('phone', [$phone])]); - if (!$invitee->isEmpty() && !empty($email) && $invitee->getAttribute('email', '') !== $email) { + if (! $invitee->isEmpty() && ! empty($email) && $invitee->getAttribute('email', '') !== $email) { throw new Exception(Exception::USER_ALREADY_EXISTS, 'Given phone and email doesn\'t match', 409); } } @@ -153,7 +153,7 @@ class Create extends Action if ($invitee->isEmpty()) { // Create new user if no user with same email found $limit = $project->getAttribute('auths', [])['limit'] ?? 0; - if (!$isPrivilegedUser && !$isAppUser && $limit !== 0 && $project->getId() !== 'console') { // check users limit, console invites are allways allowed. + if (! $isPrivilegedUser && ! $isAppUser && $limit !== 0 && $project->getId() !== 'console') { // check users limit, console invites are allways allowed. $total = $dbForProject->count('users', [], APP_LIMIT_USERS); if ($total >= $limit) { @@ -165,7 +165,7 @@ class Create extends Action $identityWithMatchingEmail = $dbForProject->findOne('identities', [ Query::equal('providerEmail', [$email]), ]); - if (!$identityWithMatchingEmail->isEmpty()) { + if (! $identityWithMatchingEmail->isEmpty()) { throw new Exception(Exception::USER_EMAIL_ALREADY_EXISTS); } @@ -225,7 +225,7 @@ class Create extends Action $isOwner = $authorization->hasRole('team:' . $team->getId() . '/owner'); - if (!$isOwner && !$isPrivilegedUser && !$isAppUser) { // Not owner, not admin, not app (server) + if (! $isOwner && ! $isPrivilegedUser && ! $isAppUser) { // Not owner, not admin, not app (server) throw new Exception(Exception::USER_UNAUTHORIZED, 'User is not allowed to send invitations for this team'); } @@ -255,7 +255,7 @@ class Create extends Action 'joined' => ($isPrivilegedUser || $isAppUser) ? DateTime::now() : null, 'confirm' => ($isPrivilegedUser || $isAppUser), 'secret' => $proofForToken->hash($secret), - 'search' => implode(' ', [$membershipId, $invitee->getId()]) + 'search' => implode(' ', [$membershipId, $invitee->getId()]), ]); $membership = ($isPrivilegedUser || $isAppUser) ? @@ -292,22 +292,22 @@ class Create extends Action $url = Template::parseURL($url); $url['query'] = Template::mergeQuery(((isset($url['query'])) ? $url['query'] : ''), ['membershipId' => $membership->getId(), 'userId' => $invitee->getId(), 'secret' => $secret, 'teamId' => $teamId, 'teamName' => $team->getAttribute('name')]); $url = Template::unParseURL($url); - if (!empty($email)) { + if (! empty($email)) { $projectName = $project->isEmpty() ? 'Console' : $project->getAttribute('name', '[APP-NAME]'); - $body = $locale->getText("emails.invitation.body"); - $preview = $locale->getText("emails.invitation.preview"); - $subject = $locale->getText("emails.invitation.subject"); + $body = $locale->getText('emails.invitation.body'); + $preview = $locale->getText('emails.invitation.preview'); + $subject = $locale->getText('emails.invitation.subject'); $customTemplate = $project->getAttribute('templates', [])['email.invitation-' . $locale->default] ?? []; $message = Template::fromFile(APP_CE_CONFIG_DIR . '/locale/templates/email-inner-base.tpl'); $message ->setParam('{{body}}', $body, escapeHtml: false) - ->setParam('{{hello}}', $locale->getText("emails.invitation.hello")) - ->setParam('{{footer}}', $locale->getText("emails.invitation.footer")) - ->setParam('{{thanks}}', $locale->getText("emails.invitation.thanks")) - ->setParam('{{buttonText}}', $locale->getText("emails.invitation.buttonText")) - ->setParam('{{signature}}', $locale->getText("emails.invitation.signature")); + ->setParam('{{hello}}', $locale->getText('emails.invitation.hello')) + ->setParam('{{footer}}', $locale->getText('emails.invitation.footer')) + ->setParam('{{thanks}}', $locale->getText('emails.invitation.thanks')) + ->setParam('{{buttonText}}', $locale->getText('emails.invitation.buttonText')) + ->setParam('{{signature}}', $locale->getText('emails.invitation.signature')); $body = $message->render(); $smtp = $project->getAttribute('smtp', []); @@ -315,16 +315,16 @@ class Create extends Action $senderEmail = System::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM); $senderName = System::getEnv('_APP_SYSTEM_EMAIL_NAME', APP_NAME . ' Server'); - $replyTo = ""; + $replyTo = ''; if ($smtpEnabled) { - if (!empty($smtp['senderEmail'])) { + if (! empty($smtp['senderEmail'])) { $senderEmail = $smtp['senderEmail']; } - if (!empty($smtp['senderName'])) { + if (! empty($smtp['senderName'])) { $senderName = $smtp['senderName']; } - if (!empty($smtp['replyTo'])) { + if (! empty($smtp['replyTo'])) { $replyTo = $smtp['replyTo']; } @@ -335,14 +335,14 @@ class Create extends Action ->setSmtpPassword($smtp['password'] ?? '') ->setSmtpSecure($smtp['secure'] ?? ''); - if (!empty($customTemplate)) { - if (!empty($customTemplate['senderEmail'])) { + if (! empty($customTemplate)) { + if (! empty($customTemplate['senderEmail'])) { $senderEmail = $customTemplate['senderEmail']; } - if (!empty($customTemplate['senderName'])) { + if (! empty($customTemplate['senderName'])) { $senderName = $customTemplate['senderName']; } - if (!empty($customTemplate['replyTo'])) { + if (! empty($customTemplate['replyTo'])) { $replyTo = $customTemplate['replyTo']; } @@ -363,7 +363,7 @@ class Create extends Action 'user' => $name, 'team' => $team->getAttribute('name'), 'redirect' => $url, - 'project' => $projectName + 'project' => $projectName, ]; $queueForMails @@ -374,7 +374,7 @@ class Create extends Action ->setName($invitee->getAttribute('name', '')) ->appendVariables($emailVariables) ->trigger(); - } elseif (!empty($phone)) { + } elseif (! empty($phone)) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -382,7 +382,7 @@ class Create extends Action $message = Template::fromFile(APP_CE_CONFIG_DIR . '/locale/templates/sms-base.tpl'); $customTemplate = $project->getAttribute('templates', [])['sms.invitation-' . $locale->default] ?? []; - if (!empty($customTemplate)) { + if (! empty($customTemplate)) { $message = $customTemplate['message']; } @@ -406,25 +406,20 @@ class Create extends Action try { $countryCode = $helper->parse($phone)->getCountryCode(); - if (!empty($countryCode)) { - $queueForStatsUsage - ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + if (! empty($countryCode)) { + $usage->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } } catch (NumberParseException $e) { // Ignore invalid phone number for country code stats } - $queueForStatsUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); + $usage->addMetric(METRIC_AUTH_METHOD_PHONE, 1); } } $queueForEvents ->setParam('userId', $invitee->getId()) ->setParam('teamId', $team->getId()) - ->setParam('membershipId', $membership->getId()) - ; + ->setParam('membershipId', $membership->getId()); $response ->setStatusCode(Response::STATUS_CODE_CREATED) diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index d866cc2bd0..af7d2027e3 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -2,8 +2,10 @@ namespace Appwrite\Platform\Workers; -use Appwrite\Event\StatsUsage; +use Appwrite\Event\Message\Usage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Messaging\Status as MessageStatus; +use Appwrite\Usage\Context as UsageContext; use libphonenumber\NumberParseException; use libphonenumber\PhoneNumberUtil; use Swoole\Runtime; @@ -71,7 +73,7 @@ class Messaging extends Action ->inject('log') ->inject('dbForProject') ->inject('deviceForFiles') - ->inject('queueForStatsUsage') + ->inject('publisherForUsage') ->callback($this->action(...)); } @@ -81,7 +83,7 @@ class Messaging extends Action * @param Log $log * @param Database $dbForProject * @param Device $deviceForFiles - * @param StatsUsage $queueForStatsUsage + * @param UsagePublisher $publisherForUsage * @return void * @throws \Exception */ @@ -91,7 +93,7 @@ class Messaging extends Action Log $log, Database $dbForProject, Device $deviceForFiles, - StatsUsage $queueForStatsUsage + UsagePublisher $publisherForUsage ): void { Runtime::setHookFlags(SWOOLE_HOOK_ALL ^ SWOOLE_HOOK_TCP); $payload = $message->getPayload() ?? []; @@ -115,7 +117,7 @@ class Messaging extends Action case MESSAGE_SEND_TYPE_EXTERNAL: $message = $dbForProject->getDocument('messages', $payload['messageId']); - $this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project, $queueForStatsUsage); + $this->sendExternalMessage($dbForProject, $message, $deviceForFiles, $project, $publisherForUsage); break; default: throw new \Exception('Unknown message type: ' . $type); @@ -133,7 +135,7 @@ class Messaging extends Action Document $message, Device $deviceForFiles, Document $project, - StatsUsage $queueForStatsUsage + UsagePublisher $publisherForUsage ): void { $topicIds = $message->getAttribute('topics', []); $targetIds = $message->getAttribute('targets', []); @@ -239,8 +241,8 @@ class Messaging extends Action /** * @var array $results */ - $results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { - return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { + $results = batch(\array_map(function ($providerId) use ($identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { + return function () use ($providerId, $identifiers, &$providers, $default, $message, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { if (\array_key_exists($providerId, $providers)) { $provider = $providers[$providerId]; } else { @@ -267,8 +269,8 @@ class Messaging extends Action $adapter->getMaxMessagesPerRequest() ); - return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { - return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $queueForStatsUsage) { + return batch(\array_map(function ($batch) use ($message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { + return function () use ($batch, $message, $provider, $adapter, $dbForProject, $deviceForFiles, $project, $publisherForUsage) { $deliveredTotal = 0; $deliveryErrors = []; $messageData = clone $message; @@ -308,8 +310,8 @@ class Messaging extends Action $deliveryErrors[] = 'Failed sending to targets with error: ' . $e->getMessage(); } finally { $errorTotal = \count($deliveryErrors); - $queueForStatsUsage - ->setProject($project) + $usage = new UsageContext(); + $usage ->addMetric(METRIC_MESSAGES, ($deliveredTotal + $errorTotal)) ->addMetric(METRIC_MESSAGES_SENT, $deliveredTotal) ->addMetric(METRIC_MESSAGES_FAILED, $errorTotal) @@ -318,8 +320,12 @@ class Messaging extends Action ->addMetric(str_replace('{type}', $provider->getAttribute('type'), METRIC_MESSAGES_TYPE_FAILED), $errorTotal) ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER), ($deliveredTotal + $errorTotal)) ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_SENT), $deliveredTotal) - ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal) - ->trigger(); + ->addMetric(str_replace(['{type}', '{provider}'], [$provider->getAttribute('type'), $provider->getAttribute('provider')], METRIC_MESSAGES_TYPE_PROVIDER_FAILED), $errorTotal); + + $publisherForUsage->enqueue(new Usage( + project: $project, + metrics: $usage->getMetrics(), + )); return [ 'deliveredTotal' => $deliveredTotal, diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index c4cb9ce415..ce20358626 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -4,10 +4,12 @@ namespace Appwrite\Platform\Workers; use Ahc\Jwt\JWT; use Appwrite\Event\Mail; +use Appwrite\Event\Message\Usage as UsageMessage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Event\Realtime; -use Appwrite\Event\StatsUsage; use Appwrite\Extend\Exception; use Appwrite\Template\Template; +use Appwrite\Usage\Context; use Utopia\Compression\Compression; use Utopia\Config\Config; use Utopia\Console; @@ -84,7 +86,8 @@ class Migrations extends Action ->inject('deviceForMigrations') ->inject('deviceForFiles') ->inject('queueForMails') - ->inject('queueForStatsUsage') + ->inject('usage') + ->inject('publisherForUsage') ->inject('plan') ->inject('authorization') ->callback($this->action(...)); @@ -103,7 +106,8 @@ class Migrations extends Action Device $deviceForMigrations, Device $deviceForFiles, Mail $queueForMails, - StatsUsage $queueForStatsUsage, + Context $usage, + UsagePublisher $publisherForUsage, array $plan, Authorization $authorization, ): void { @@ -147,7 +151,8 @@ class Migrations extends Action $migration, $queueForRealtime, $queueForMails, - $queueForStatsUsage, + $usage, + $publisherForUsage, $platform, $authorization ); @@ -345,7 +350,8 @@ class Migrations extends Action Document $migration, Realtime $queueForRealtime, Mail $queueForMails, - StatsUsage $queueForStatsUsage, + Context $usage, + UsagePublisher $publisherForUsage, array $platform, Authorization $authorization, ): void { @@ -360,7 +366,7 @@ class Migrations extends Action throw new \Exception('_APP_MIGRATION_HOST is not set'); } - $endpoint = 'http://'.$host.'/v1'; + $endpoint = 'http://' . $host . '/v1'; try { $credentials = $migration->getAttribute('credentials', []); @@ -463,7 +469,7 @@ class Migrations extends Action $migration->setAttribute('status', 'failed'); $migration->setAttribute('stage', 'finished'); - call_user_func($this->logError, $th, 'appwrite-worker', 'appwrite-queue-'.self::getName(), [ + call_user_func($this->logError, $th, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [ 'migrationId' => $migration->getId(), 'source' => $migration->getAttribute('source') ?? '', 'destination' => $migration->getAttribute('destination') ?? '', @@ -474,7 +480,7 @@ class Migrations extends Action $this->updateMigrationDocument($migration, $project, $queueForRealtime); if ($migration->getAttribute('status', '') === 'failed') { - Console::error('Migration('.$migration->getSequence().':'.$migration->getId().') failed, Project('.$this->project->getSequence().':'.$this->project->getId().')'); + Console::error('Migration(' . $migration->getSequence() . ':' . $migration->getId() . ') failed, Project(' . $this->project->getSequence() . ':' . $this->project->getId() . ')'); $sourceErrors = $source?->getErrors() ?? []; $destinationErrors = $destination?->getErrors() ?? []; @@ -500,8 +506,9 @@ class Migrations extends Action foreach ($aggregatedResources as $resource) { $this->processMigrationResourceStats( $resource, - $queueForStatsUsage, + $usage, $project, + $publisherForUsage, $migration->getAttribute('source'), $authorization, $migration->getAttribute('resourceId') @@ -802,7 +809,7 @@ class Migrations extends Action return $errors; } - private function processMigrationResourceStats(array $resources, StatsUsage $queueForStatsUsage, Document $projectDocument, string $source, Authorization $authorization, ?string $resourceId) + private function processMigrationResourceStats(array $resources, Context $usage, Document $projectDocument, UsagePublisher $publisherForUsage, string $source, Authorization $authorization, ?string $resourceId) { $resourceName = $resources['name']; $count = $resources['count']; @@ -819,11 +826,11 @@ class Migrations extends Action switch ($resourceName) { case ResourceDatabase::getName(): - $queueForStatsUsage->addMetric(METRIC_DATABASES, $count); + $usage->addMetric(METRIC_DATABASES, $count); break; case ResourceTable::getName(): - $queueForStatsUsage + $usage ->addMetric(METRIC_COLLECTIONS, $count) ->addMetric( str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), @@ -832,7 +839,7 @@ class Migrations extends Action break; case ResourceRow::getName(): - $queueForStatsUsage + $usage ->addMetric( str_replace( ['{databaseInternalId}','{collectionInternalId}'], @@ -852,7 +859,12 @@ class Migrations extends Action break; } - $queueForStatsUsage->setProject($projectDocument)->trigger(); - $queueForStatsUsage->reset(); + $message = new UsageMessage( + project: $projectDocument, + metrics: $usage->getMetrics(), + reduce: $usage->getReduce() + ); + $publisherForUsage->enqueue($message); + $usage->reset(); } } diff --git a/src/Appwrite/Platform/Workers/Webhooks.php b/src/Appwrite/Platform/Workers/Webhooks.php index 4855a1d4d8..fce3c7b149 100644 --- a/src/Appwrite/Platform/Workers/Webhooks.php +++ b/src/Appwrite/Platform/Workers/Webhooks.php @@ -3,8 +3,10 @@ namespace Appwrite\Platform\Workers; use Appwrite\Event\Mail; -use Appwrite\Event\StatsUsage; +use Appwrite\Event\Message\Usage as UsageMessage; +use Appwrite\Event\Publisher\Usage as UsagePublisher; use Appwrite\Template\Template; +use Appwrite\Usage\Context as UsageContext; use Exception; use Utopia\Database\Database; use Utopia\Database\Document; @@ -35,7 +37,7 @@ class Webhooks extends Action ->inject('project') ->inject('dbForPlatform') ->inject('queueForMails') - ->inject('queueForStatsUsage') + ->inject('publisherForUsage') ->inject('log') ->inject('plan') ->callback($this->action(...)); @@ -46,13 +48,13 @@ class Webhooks extends Action * @param Document $project * @param Database $dbForPlatform * @param Mail $queueForMails - * @param StatsUsage $queueForStatsUsage + * @param UsagePublisher $publisherForUsage * @param Log $log * @param array $plan * @return void * @throws Exception */ - public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage, Log $log, array $plan): void + public function action(Message $message, Document $project, Database $dbForPlatform, Mail $queueForMails, UsagePublisher $publisherForUsage, Log $log, array $plan): void { $this->errors = []; $payload = $message->getPayload() ?? []; @@ -71,7 +73,7 @@ class Webhooks extends Action foreach ($project->getAttribute('webhooks', []) as $webhook) { if (array_intersect($webhook->getAttribute('events', []), $events)) { - $this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $queueForStatsUsage, $plan); + $this->execute($events, $webhookPayload, $webhook, $user, $project, $dbForPlatform, $queueForMails, $publisherForUsage, $plan); } } @@ -91,7 +93,7 @@ class Webhooks extends Action * @param array $plan * @return void */ - private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, StatsUsage $queueForStatsUsage, array $plan): void + private function execute(array $events, string $payload, Document $webhook, Document $user, Document $project, Database $dbForPlatform, Mail $queueForMails, UsagePublisher $publisherForUsage, array $plan): void { if ($webhook->getAttribute('enabled') !== true) { return; @@ -180,26 +182,23 @@ class Webhooks extends Action $dbForPlatform->purgeCachedDocument('projects', $project->getId()); $this->errors[] = $logs; - $queueForStatsUsage + $usage = (new UsageContext()) ->addMetric(METRIC_WEBHOOKS_FAILED, 1) - ->addMetric(str_replace('{webhookInternalId}', $webhook->getSequence(), METRIC_WEBHOOK_ID_FAILED), 1) - ; - - + ->addMetric(str_replace('{webhookInternalId}', $webhook->getSequence(), METRIC_WEBHOOK_ID_FAILED), 1); } else { $dbForPlatform->updateDocument('webhooks', $webhook->getId(), new Document([ 'attempts' => 0, ])); $dbForPlatform->purgeCachedDocument('projects', $project->getId()); - $queueForStatsUsage + $usage = (new UsageContext()) ->addMetric(METRIC_WEBHOOKS_SENT, 1) - ->addMetric(str_replace('{webhookInternalId}', $webhook->getSequence(), METRIC_WEBHOOK_ID_SENT), 1) - ; + ->addMetric(str_replace('{webhookInternalId}', $webhook->getSequence(), METRIC_WEBHOOK_ID_SENT), 1); } - $queueForStatsUsage - ->setProject($project) - ->trigger(); + $publisherForUsage->enqueue(new UsageMessage( + project: $project, + metrics: $usage->getMetrics(), + )); } /** diff --git a/src/Appwrite/Usage/Context.php b/src/Appwrite/Usage/Context.php new file mode 100644 index 0000000000..7283cff836 --- /dev/null +++ b/src/Appwrite/Usage/Context.php @@ -0,0 +1,74 @@ +metrics[] = [ + 'key' => $key, + 'value' => $value, + ]; + + return $this; + } + + /** + * Add a document to reduce + */ + public function addReduce(Document $document): self + { + $this->reduce[] = $document; + + return $this; + } + + /** + * Get all metrics + * + * @return array + */ + public function getMetrics(): array + { + return $this->metrics; + } + + /** + * Get all reduce documents + * + * @return array + */ + public function getReduce(): array + { + return $this->reduce; + } + + /** + * Check if context is empty + */ + public function isEmpty(): bool + { + return empty($this->metrics) && empty($this->reduce); + } + + /** + * Reset the context + */ + public function reset(): self + { + $this->metrics = []; + $this->reduce = []; + + return $this; + } +} diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index d5fe7753a4..0d992c472e 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1651,9 +1651,9 @@ trait MigrationsBase }, 30_000, 500); // Check that email was sent with download link - $lastEmail = $this->getLastEmail(); - $this->assertNotEmpty($lastEmail); - $this->assertEquals('Your CSV export is ready', $lastEmail['subject']); + $lastEmail = $this->getLastEmail(probe: function ($email) { + $this->assertEquals('Your CSV export is ready', $email['subject']); + }); $this->assertStringContainsStringIgnoringCase('Your data export has been completed successfully', $lastEmail['text']); // Extract download URL from email HTML From 0295e27b5d72cd199b3e3929fc20b8ecd5abf4bc Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 15:42:41 +0000 Subject: [PATCH 55/57] Fix build usage metrics to use actual memory/cpus and fix escapeshellarg double-quoting Use $memory (which includes minMemory floor) and $cpus instead of raw spec values in MB-seconds metrics, fixing underreporting for sites and frameworks bumped to the minimum memory. Also remove redundant double quotes around escapeshellarg() calls in mv command. Co-Authored-By: Claude Opus 4.6 --- .../Platform/Modules/Functions/Workers/Builds.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index f8d8b464aa..443db88a7a 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -398,7 +398,7 @@ class Builds extends Action $rootDirectoryWithoutSpaces = str_replace(' ', '', $rootDirectory); $from = $tmpDirectory . '/' . $rootDirectory; $to = $tmpDirectory . '/' . $rootDirectoryWithoutSpaces; - $exit = Console::execute('mv "' . \escapeshellarg($from) . '" "' . \escapeshellarg($to) . '"', '', $stdout, $stderr); + $exit = Console::execute('mv ' . \escapeshellarg($from) . ' ' . \escapeshellarg($to), '', $stdout, $stderr); if ($exit !== 0) { throw new \Exception('Unable to move function with spaces' . $stderr); @@ -1229,15 +1229,15 @@ class Builds extends Action ->addMetric(METRIC_BUILDS, 1) // per project ->addMetric(METRIC_BUILDS_STORAGE, $deployment->getAttribute('buildSize', 0)) ->addMetric(METRIC_BUILDS_COMPUTE, (int) $deployment->getAttribute('buildDuration', 0) * 1000) - ->addMetric(METRIC_BUILDS_MB_SECONDS, (int) (($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(METRIC_BUILDS_MB_SECONDS, (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus)) ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS), 1) // per function ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0)) ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000) - ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS), (int) (($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))) + ->addMetric(str_replace(['{resourceType}'], [$deployment->getAttribute('resourceType')], METRIC_RESOURCE_TYPE_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus)) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS), 1) // per function ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_STORAGE), $deployment->getAttribute('buildSize', 0)) ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_COMPUTE), (int) $deployment->getAttribute('buildDuration', 0) * 1000) - ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS), (int) (($spec['memory'] ?? APP_COMPUTE_MEMORY_DEFAULT) * $deployment->getAttribute('buildDuration', 0) * ($spec['cpus'] ?? APP_COMPUTE_CPUS_DEFAULT))); + ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$deployment->getAttribute('resourceType'), $resource->getSequence()], METRIC_RESOURCE_TYPE_ID_BUILDS_MB_SECONDS), (int) ($memory * $deployment->getAttribute('buildDuration', 0) * $cpus)); // Publish usage metrics if (! $usage->isEmpty()) { From 530d681c482388f3da608059c913e207332e7e9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 13 Mar 2026 11:25:17 +0100 Subject: [PATCH 56/57] Fix staging env --- app/config/collections/projects.php | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php index 6c417ae145..55dceb9b40 100644 --- a/app/config/collections/projects.php +++ b/app/config/collections/projects.php @@ -788,6 +788,17 @@ return [ 'default' => null, 'filters' => [], ], + [ + 'array' => false, + '$id' => ID::custom('specification'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 128, + 'signed' => false, + 'required' => false, + 'default' => APP_COMPUTE_SPECIFICATION_DEFAULT, + 'filters' => [], + ], [ 'array' => false, '$id' => ID::custom('buildSpecification'), @@ -1245,6 +1256,17 @@ return [ 'array' => false, 'filters' => [], ], + [ + 'array' => false, + '$id' => ID::custom('specification'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 128, + 'signed' => false, + 'required' => false, + 'default' => APP_COMPUTE_SPECIFICATION_DEFAULT, + 'filters' => [], + ], [ 'array' => false, '$id' => ID::custom('buildSpecification'), From 236dceae2c11d706d7525d75de256a3060a6810d Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 13 Mar 2026 13:59:06 +0000 Subject: [PATCH 57/57] Pin utopia-php/servers to 0.2.5 Co-Authored-By: Claude Opus 4.6 --- composer.json | 1 + 1 file changed, 1 insertion(+) diff --git a/composer.json b/composer.json index abe51e500b..19e1bb5c5a 100644 --- a/composer.json +++ b/composer.json @@ -76,6 +76,7 @@ "utopia-php/span": "1.1.*", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.15.*", + "utopia-php/servers": "0.2.5", "utopia-php/registry": "0.5.*", "utopia-php/storage": "1.0.*", "utopia-php/system": "0.10.*",