From 7644e0fe48eb0a2dbba50d6db71d4c41a8560d50 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 3 Mar 2026 18:48:37 +0530 Subject: [PATCH 01/51] Add realtime metrics for connections, messages, and bandwidth in project usage --- app/controllers/api/project.php | 34 ++++ app/init/constants.php | 6 + app/realtime.php | 107 ++++++++++- .../Utopia/Response/Model/UsageProject.php | 39 ++++ tests/e2e/General/UsageTest.php | 172 ++++++++++++++++++ 5 files changed, 355 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index d24519e3fb..1fd33a5db0 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -72,6 +72,10 @@ Http::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, + METRIC_REALTIME_CONNECTIONS, + METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, + METRIC_REALTIME_INBOUND, + METRIC_REALTIME_OUTBOUND, ], 'period' => [ METRIC_NETWORK_REQUESTS, @@ -85,6 +89,10 @@ Http::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, + METRIC_REALTIME_CONNECTIONS, + METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, + METRIC_REALTIME_INBOUND, + METRIC_REALTIME_OUTBOUND, ] ]; @@ -347,6 +355,26 @@ Http::get('/v1/project/usage') ]; } + // Realtime bandwidth = realtime.inbound + realtime.outbound (per bucket) + $realtimeProjectBandwidth = []; + foreach ($usage[METRIC_REALTIME_INBOUND] as $item) { + $realtimeProjectBandwidth[$item['date']] ??= 0; + $realtimeProjectBandwidth[$item['date']] += $item['value']; + } + + foreach ($usage[METRIC_REALTIME_OUTBOUND] as $item) { + $realtimeProjectBandwidth[$item['date']] ??= 0; + $realtimeProjectBandwidth[$item['date']] += $item['value']; + } + + $realtimeBandwidth = []; + foreach ($realtimeProjectBandwidth as $date => $value) { + $realtimeBandwidth[] = [ + 'date' => $date, + 'value' => $value + ]; + } + $response->dynamic(new Document([ 'requests' => ($usage[METRIC_NETWORK_REQUESTS]), 'network' => $network, @@ -367,10 +395,16 @@ Http::get('/v1/project/usage') 'deploymentsStorageTotal' => $total[METRIC_DEPLOYMENTS_STORAGE], 'databasesReadsTotal' => $total[METRIC_DATABASES_OPERATIONS_READS], 'databasesWritesTotal' => $total[METRIC_DATABASES_OPERATIONS_WRITES], + 'realtimeConnectionsTotal' => $total[METRIC_REALTIME_CONNECTIONS], + 'realtimeMessagesTotal' => $total[METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT], + 'realtimeBandwidthTotal' => ($total[METRIC_REALTIME_INBOUND] ?? 0) + ($total[METRIC_REALTIME_OUTBOUND] ?? 0), 'executionsBreakdown' => $executionsBreakdown, 'bucketsBreakdown' => $bucketsBreakdown, 'databasesReads' => $usage[METRIC_DATABASES_OPERATIONS_READS], 'databasesWrites' => $usage[METRIC_DATABASES_OPERATIONS_WRITES], + 'realtimeConnections' => $usage[METRIC_REALTIME_CONNECTIONS], + 'realtimeMessages' => $usage[METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT], + 'realtimeBandwidth' => $realtimeBandwidth, 'databasesStorageBreakdown' => $databasesStorageBreakdown, 'executionsMbSecondsBreakdown' => $executionsMbSecondsBreakdown, 'buildsMbSecondsBreakdown' => $buildsMbSecondsBreakdown, diff --git a/app/init/constants.php b/app/init/constants.php index c6a5bc853e..cd77b05746 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -361,6 +361,12 @@ const METRIC_AVATARS_SCREENSHOTS_GENERATED = 'avatars.screenshotsGenerated'; const METRIC_FUNCTIONS_RUNTIME = 'functions.runtimes.{runtime}'; const METRIC_SITES_FRAMEWORK = 'sites.frameworks.{framework}'; +// Realtime metrics +const METRIC_REALTIME_CONNECTIONS = 'realtime.connections'; +const METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT = 'realtime.messages.sent'; +const METRIC_REALTIME_INBOUND = 'realtime.inbound'; +const METRIC_REALTIME_OUTBOUND = 'realtime.outbound'; + // Resource types const RESOURCE_TYPE_PROJECTS = 'projects'; const RESOURCE_TYPE_FUNCTIONS = 'functions'; diff --git a/app/realtime.php b/app/realtime.php index 7ec24d03c8..8a066bcbab 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -1,5 +1,6 @@ getSequence()])) { + return $ctx['queueForStatsUsage'][$project->getSequence()]; + } + + global $register; + + /** @var Group $pools */ + $pools = $register->get('pools'); + + $queue = new StatsUsage(new BrokerPool(publisher: $pools->get('publisher'))); + $queue->setProject($project); + + return $ctx['queueForStatsUsage'][$project->getSequence()] = $queue; + } +} + $realtime = getRealtime(); /** @@ -545,20 +572,59 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, } $total = 0; + $outboundBytes = 0; + foreach ($groups as $group) { $data = $event['data']; $data['subscriptions'] = $group['subscriptions']; - $server->send($group['ids'], json_encode([ + $payloadJson = json_encode([ 'type' => 'event', 'data' => $data - ])); - $total += count($group['ids']); + ]); + + $server->send($group['ids'], $payloadJson); + + $count = count($group['ids']); + $total += $count; + $outboundBytes += strlen($payloadJson) * $count; } if ($total > 0) { $register->get('telemetry.messageSentCounter')->add($total); $stats->incr($event['project'], 'messages', $total); + + $projectId = $event['project'] ?? null; + + if (!empty($projectId)) { + try { + $consoleDB = getConsoleDB(); + /** @var Document $project */ + $project = $consoleDB->getAuthorization()->skip( + fn () => $consoleDB->getDocument('projects', $projectId) + ); + + if (!$project->isEmpty()) { + $queueForStatsUsage = getQueueForStatsUsageForProject($project); + + $queueForStatsUsage->addMetric( + METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, + $total + ); + + if ($outboundBytes > 0) { + $queueForStatsUsage->addMetric( + METRIC_REALTIME_OUTBOUND, + $outboundBytes + ); + } + + $queueForStatsUsage->trigger(); + } + } catch (Throwable $th) { + logError($th, 'realtimeUsageOutbound', tags: ['projectId' => $projectId]); + } + } } }); } catch (Throwable $th) { @@ -707,6 +773,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, ]); $stats->incr($project->getId(), 'connections'); $stats->incr($project->getId(), 'connectionsTotal'); + + try { + $queueForStatsUsage = getQueueForStatsUsageForProject($project); + $queueForStatsUsage + ->addMetric(METRIC_REALTIME_CONNECTIONS, 1) + ->trigger(); + } catch (\Throwable $th) { + logError($th, 'realtimeUsageConnections', project: $project); + } + } catch (Throwable $th) { logError($th, 'realtime', project: $project, user: $logUser, authorization: $authorization); @@ -748,6 +824,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $authorization = null; try { + $rawSize = \strlen($message); $response = new Response(new SwooleResponse()); $projectId = $realtime->connections[$connection]['projectId'] ?? null; @@ -786,6 +863,18 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many messages.'); } + // Record realtime inbound bytes for this project + if ($project !== null && !$project->isEmpty()) { + try { + $queueForStatsUsage = getQueueForStatsUsageForProject($project); + $queueForStatsUsage + ->addMetric(METRIC_REALTIME_INBOUND, $rawSize) + ->trigger(); + } catch (Throwable $th) { + logError($th, 'realtimeUsageInbound', project: $project); + } + } + $message = json_decode($message, true); if (is_null($message) || (!array_key_exists('type', $message) && !array_key_exists('data', $message))) { @@ -905,6 +994,18 @@ $server->onClose(function (int $connection) use ($realtime, $stats, $register) { if (array_key_exists($connection, $realtime->connections)) { $stats->decr($realtime->connections[$connection]['projectId'], 'connectionsTotal'); $register->get('telemetry.connectionCounter')->add(-1); + + $projectId = $realtime->connections[$connection]['projectId']; + + $consoleDB = getConsoleDB(); + $project = $consoleDB->getAuthorization()->skip( + fn () => $consoleDB->getDocument('projects', $projectId) + ); + + if (!$project->isEmpty()) { + $queue = getQueueForStatsUsageForProject($project); + $queue->addMetric(METRIC_REALTIME_CONNECTIONS, -1)->trigger(); + } } $realtime->unsubscribe($connection); diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index ee644aa845..c219a35f29 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -100,6 +100,24 @@ class UsageProject extends Model 'default' => 0, 'example' => 0, ]) + ->addRule('realtimeConnectionsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Current aggregated number of open Realtime connections.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('realtimeMessagesTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of Realtime messages sent to clients.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('realtimeBandwidthTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total consumed Realtime bandwidth (in bytes).', + 'default' => 0, + 'example' => 0, + ]) ->addRule('requests', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of requests per period.', @@ -114,6 +132,27 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) + ->addRule('realtimeConnections', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of open Realtime connections per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('realtimeMessages', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of Realtime messages sent to clients per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('realtimeBandwidth', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated consumed Realtime bandwidth (in bytes) per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ->addRule('users', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of users per period.', diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index e7b5b1d844..29229ae2bd 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -11,6 +11,7 @@ use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideServer; use Tests\E2E\Services\Functions\FunctionsBase; +use Tests\E2E\Services\Realtime\RealtimeBase; use Tests\E2E\Services\Sites\SitesBase; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -23,6 +24,7 @@ class UsageTest extends Scope use ProjectCustom; use SideServer; use FunctionsBase; + use RealtimeBase; use SitesBase { FunctionsBase::createDeployment insteadof SitesBase; FunctionsBase::setupDeployment insteadof SitesBase; @@ -1408,6 +1410,176 @@ class UsageTest extends Scope }); } + public function testRealtimeUsageMetrics(): void + { + $user = $this->getUser(); + $session = $user['session'] ?? ''; + $projectId = $this->getProject()['$id']; + + // Baseline realtime usage before opening a new connection + $baseline = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $connectionsBefore = $baseline['body']['realtimeConnectionsTotal'] ?? 0; + $messagesBefore = $baseline['body']['realtimeMessagesTotal'] ?? 0; + + $connectionCount = 3; + $clients = []; + + for ($i = 0; $i < $connectionCount; $i++) { + $client = $this->getWebsocket(['documents'], [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ], null); + + $connected = json_decode($client->receive(), true); + $this->assertEquals('connected', $connected['type']); + + $clients[] = $client; + } + + try { + $database = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Realtime Usage DB', + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Realtime Usage Collection', + 'permissions' => [ + Permission::create(Role::user($user['$id'])), + ], + 'documentSecurity' => true, + ]); + + $collectionId = $collection['body']['$id']; + + $attribute = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'key' => 'name', + 'size' => 256, + 'required' => true, + ]); + + $this->assertEquals(202, $attribute['headers']['status-code']); + + $this->assertEventually(function () use ($databaseId, $collectionId, $projectId) { + $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/name', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + $this->assertEquals('available', $response['body']['status']); + }, 30000, 250); + + $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders()), [ + 'documentId' => ID::unique(), + 'data' => [ + 'name' => 'Realtime Usage Doc', + ], + 'permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $this->assertEquals(201, $document['headers']['status-code']); + + $event = json_decode($clients[0]->receive(), true); + $this->assertEquals('event', $event['type']); + + // After creating a document we expect all connections to receive an event + $this->assertEventually(function () use ($connectionsBefore, $messagesBefore, $connectionCount) { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertEquals(200, $response['headers']['status-code']); + + $this->assertArrayHasKey('realtimeConnectionsTotal', $response['body']); + $this->assertArrayHasKey('realtimeMessagesTotal', $response['body']); + $this->assertArrayHasKey('realtimeBandwidthTotal', $response['body']); + $this->assertArrayHasKey('realtimeConnections', $response['body']); + $this->assertArrayHasKey('realtimeMessages', $response['body']); + $this->assertArrayHasKey('realtimeBandwidth', $response['body']); + + // We expect exactly $connectionCount additional open connections and $connectionCount additional message deliveries + $this->assertEquals($connectionsBefore + $connectionCount, $response['body']['realtimeConnectionsTotal']); + $this->assertEquals($messagesBefore + $connectionCount, $response['body']['realtimeMessagesTotal']); + $this->assertGreaterThan(0, $response['body']['realtimeBandwidthTotal']); + + $this->validateDates($response['body']['realtimeConnections']); + $this->validateDates($response['body']['realtimeMessages']); + $this->validateDates($response['body']['realtimeBandwidth']); + }, 60000, 2000); + + // Now close a single connection and ensure the counters reflect it + $clients[0]->close(); + + $this->assertEventually(function () use ($connectionsBefore, $messagesBefore, $connectionCount) { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertEquals(200, $response['headers']['status-code']); + + $this->assertArrayHasKey('realtimeConnectionsTotal', $response['body']); + $this->assertArrayHasKey('realtimeMessagesTotal', $response['body']); + $this->assertArrayHasKey('realtimeBandwidthTotal', $response['body']); + + // One of the connections is closed, so we expect one less open connection. + // Messages and bandwidth are cumulative and should not decrease. + $this->assertEquals($connectionsBefore + $connectionCount - 1, $response['body']['realtimeConnectionsTotal']); + $this->assertEquals($messagesBefore + $connectionCount, $response['body']['realtimeMessagesTotal']); + $this->assertGreaterThan(0, $response['body']['realtimeBandwidthTotal']); + }, 60000, 2000); + } finally { + foreach ($clients as $client) { + $client->close(); + } + } + } + public function tearDown(): void { $this->projectId = ''; From 82db411517641559f9ebbf52d2f3a809f74e8830 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 3 Mar 2026 19:36:49 +0530 Subject: [PATCH 02/51] updated --- app/realtime.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/realtime.php b/app/realtime.php index 8a066bcbab..33833d27ea 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -226,7 +226,7 @@ if (!function_exists('getTelemetry')) { } } -if (!function_exists('queueForStatsUsage')) { +if (!function_exists('getQueueForStatsUsageForProject')) { function getQueueForStatsUsageForProject(Document $project): StatsUsage { $ctx = Coroutine::getContext(); From 9fdd7c1c6e55181eea155a820352e870b5fe0e8d Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 5 Mar 2026 11:30:15 +0530 Subject: [PATCH 03/51] added try catch connection close metric --- app/realtime.php | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 33833d27ea..76a84e1d9d 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -997,14 +997,18 @@ $server->onClose(function (int $connection) use ($realtime, $stats, $register) { $projectId = $realtime->connections[$connection]['projectId']; - $consoleDB = getConsoleDB(); - $project = $consoleDB->getAuthorization()->skip( - fn () => $consoleDB->getDocument('projects', $projectId) - ); + try { + $consoleDB = getConsoleDB(); + $project = $consoleDB->getAuthorization()->skip( + fn () => $consoleDB->getDocument('projects', $projectId) + ); - if (!$project->isEmpty()) { - $queue = getQueueForStatsUsageForProject($project); - $queue->addMetric(METRIC_REALTIME_CONNECTIONS, -1)->trigger(); + if (!$project->isEmpty()) { + $queue = getQueueForStatsUsageForProject($project); + $queue->addMetric(METRIC_REALTIME_CONNECTIONS, -1)->trigger(); + } + } catch (Throwable $th) { + logError($th, 'realtimeUsageConnectionClose', tags: ['projectId' => $projectId]); } } $realtime->unsubscribe($connection); From b5c2cc971610df03dabe1045624eb20b2a2d2953 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 5 Mar 2026 18:10:09 +0530 Subject: [PATCH 04/51] Add triggerStats function for realtime usage metrics tracking --- app/realtime.php | 160 +++++++++++++++++++++++++++++------------------ 1 file changed, 100 insertions(+), 60 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 76a84e1d9d..8250a395a9 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -251,6 +251,43 @@ if (!function_exists('getQueueForStatsUsageForProject')) { } } +if (!function_exists('triggerStats')) { + /** + * Trigger realtime usage stats with a generic metric map. + * + * @param array $event Metrics in the form METRIC_CONSTANT => value + */ + function triggerStats(array $event, string $projectId): void + { + if (empty($projectId)) { + return; + } + + try { + $consoleDB = getConsoleDB(); + /** @var Document $project */ + $project = $consoleDB->getAuthorization()->skip( + fn () => $consoleDB->getDocument('projects', $projectId) + ); + + if ($project->isEmpty()) { + return; + } + + $queueForStatsUsage = getQueueForStatsUsageForProject($project); + + foreach ($event as $metric => $value) { + $queueForStatsUsage->addMetric($metric, $value); + } + + $queueForStatsUsage->trigger(); + $queueForStatsUsage->reset(); + } catch (Throwable $th) { + logError($th, 'realtimeStats', tags: ['projectId' => $projectId]); + } + } +} + $realtime = getRealtime(); /** @@ -597,33 +634,15 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $projectId = $event['project'] ?? null; if (!empty($projectId)) { - try { - $consoleDB = getConsoleDB(); - /** @var Document $project */ - $project = $consoleDB->getAuthorization()->skip( - fn () => $consoleDB->getDocument('projects', $projectId) - ); + $metrics = [ + METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT => $total, + ]; - if (!$project->isEmpty()) { - $queueForStatsUsage = getQueueForStatsUsageForProject($project); - - $queueForStatsUsage->addMetric( - METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, - $total - ); - - if ($outboundBytes > 0) { - $queueForStatsUsage->addMetric( - METRIC_REALTIME_OUTBOUND, - $outboundBytes - ); - } - - $queueForStatsUsage->trigger(); - } - } catch (Throwable $th) { - logError($th, 'realtimeUsageOutbound', tags: ['projectId' => $projectId]); + if ($outboundBytes > 0) { + $metrics[METRIC_REALTIME_OUTBOUND] = $outboundBytes; } + + triggerStats($metrics, $projectId); } } }); @@ -701,6 +720,19 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many requests'); } + // Record realtime inbound bytes for this project (WS handshake + query params) + try { + $rawSize = $request->getSize(); + } catch (Throwable) { + $rawSize = \strlen((string) $request->getURI()); + } + + if ($rawSize > 0) { + triggerStats([ + METRIC_REALTIME_INBOUND => $rawSize, + ], $project->getId()); + } + /* * Validate Client Domain - Check to avoid CSRF attack. * Adding Appwrite API domains to allow XDOMAIN communication. @@ -755,14 +787,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $user = empty($user->getId()) ? null : $response->output($user, Response::MODEL_ACCOUNT); - $server->send([$connection], json_encode([ + $connectedPayloadJson = json_encode([ 'type' => 'connected', 'data' => [ 'channels' => $names, 'subscriptions' => $mapping, 'user' => $user ] - ])); + ]); + + $server->send([$connection], $connectedPayloadJson); $register->get('telemetry.connectionCounter')->add(1); $register->get('telemetry.connectionCreatedCounter')->add(1); @@ -774,14 +808,10 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $stats->incr($project->getId(), 'connections'); $stats->incr($project->getId(), 'connectionsTotal'); - try { - $queueForStatsUsage = getQueueForStatsUsageForProject($project); - $queueForStatsUsage - ->addMetric(METRIC_REALTIME_CONNECTIONS, 1) - ->trigger(); - } catch (\Throwable $th) { - logError($th, 'realtimeUsageConnections', project: $project); - } + $connectedOutboundBytes = \strlen($connectedPayloadJson); + + triggerStats([METRIC_REALTIME_CONNECTIONS => 1, METRIC_REALTIME_OUTBOUND => $connectedOutboundBytes], $project->getId()); + } catch (Throwable $th) { logError($th, 'realtime', project: $project, user: $logUser, authorization: $authorization); @@ -865,14 +895,9 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re // Record realtime inbound bytes for this project if ($project !== null && !$project->isEmpty()) { - try { - $queueForStatsUsage = getQueueForStatsUsageForProject($project); - $queueForStatsUsage - ->addMetric(METRIC_REALTIME_INBOUND, $rawSize) - ->trigger(); - } catch (Throwable $th) { - logError($th, 'realtimeUsageInbound', project: $project); - } + triggerStats([ + METRIC_REALTIME_INBOUND => $rawSize, + ], $project->getId()); } $message = json_decode($message, true); @@ -883,9 +908,21 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re switch ($message['type']) { case 'ping': - $server->send([$connection], json_encode([ + $pongPayloadJson = json_encode([ 'type' => 'pong' - ])); + ]); + + $server->send([$connection], $pongPayloadJson); + + if ($project !== null && !$project->isEmpty()) { + $pongOutboundBytes = \strlen($pongPayloadJson); + + if ($pongOutboundBytes > 0) { + triggerStats([ + METRIC_REALTIME_OUTBOUND => $pongOutboundBytes, + ], $project->getId()); + } + } break; case 'authentication': @@ -946,14 +983,27 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re } $user = $response->output($user, Response::MODEL_ACCOUNT); - $server->send([$connection], json_encode([ + + $authResponsePayloadJson = json_encode([ 'type' => 'response', 'data' => [ 'to' => 'authentication', 'success' => true, 'user' => $user ] - ])); + ]); + + $server->send([$connection], $authResponsePayloadJson); + + if ($project !== null && !$project->isEmpty()) { + $authOutboundBytes = \strlen($authResponsePayloadJson); + + if ($authOutboundBytes > 0) { + triggerStats([ + METRIC_REALTIME_OUTBOUND => $authOutboundBytes, + ], $project->getId()); + } + } break; @@ -997,19 +1047,9 @@ $server->onClose(function (int $connection) use ($realtime, $stats, $register) { $projectId = $realtime->connections[$connection]['projectId']; - try { - $consoleDB = getConsoleDB(); - $project = $consoleDB->getAuthorization()->skip( - fn () => $consoleDB->getDocument('projects', $projectId) - ); - - if (!$project->isEmpty()) { - $queue = getQueueForStatsUsageForProject($project); - $queue->addMetric(METRIC_REALTIME_CONNECTIONS, -1)->trigger(); - } - } catch (Throwable $th) { - logError($th, 'realtimeUsageConnectionClose', tags: ['projectId' => $projectId]); - } + triggerStats([ + METRIC_REALTIME_CONNECTIONS => -1, + ], $projectId); } $realtime->unsubscribe($connection); From 03d7bccde2a4465d651a5575b4a3321990ced6fe Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 5 Mar 2026 18:14:16 +0530 Subject: [PATCH 05/51] updated tests --- tests/e2e/General/UsageTest.php | 55 +++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 29229ae2bd..2eb2673c8a 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1546,6 +1546,61 @@ class UsageTest extends Scope $this->validateDates($response['body']['realtimeBandwidth']); }, 60000, 2000); + // Capture a snapshot of usage after the broadcasted document event + $afterEventUsage = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertEquals(200, $afterEventUsage['headers']['status-code']); + + $connectionsAfterEvent = $afterEventUsage['body']['realtimeConnectionsTotal'] ?? 0; + $messagesAfterEvent = $afterEventUsage['body']['realtimeMessagesTotal'] ?? 0; + $bandwidthAfterEvent = $afterEventUsage['body']['realtimeBandwidthTotal'] ?? 0; + + // Send a ping over an existing connection to exercise the ping/pong + // realtime usage metrics path (inbound + outbound bytes) without + // generating additional "messages sent" usage. + $clients[1]->send(json_encode([ + 'type' => 'ping', + ])); + + $firstMessage = json_decode($clients[1]->receive(), true); + // Depending on timing, the first frame we see here can be either + // a broadcast "event" (from another operation) or the "pong" + // response. Both are valid, and in either case the ping/pong + // traffic still exercises the realtime usage metrics paths. + $this->assertContains($firstMessage['type'], ['event']); + + // We expect: + // - connections count to remain the same + // - messages count to remain the same (no new broadcast events) + // - bandwidth total to increase because of ping/pong traffic + $this->assertEventually(function () use ($connectionsAfterEvent, $messagesAfterEvent, $bandwidthAfterEvent) { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertEquals(200, $response['headers']['status-code']); + + $this->assertEquals($connectionsAfterEvent, $response['body']['realtimeConnectionsTotal']); + $this->assertEquals($messagesAfterEvent, $response['body']['realtimeMessagesTotal']); + $this->assertGreaterThan($bandwidthAfterEvent, $response['body']['realtimeBandwidthTotal']); + }, 60000, 2000); + // Now close a single connection and ensure the counters reflect it $clients[0]->close(); From ece5b8173277ff0d41cbbb0b56c798baf0a0f2fe Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 6 Mar 2026 15:27:30 +0530 Subject: [PATCH 06/51] updated test --- tests/e2e/General/UsageTest.php | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 2eb2673c8a..f51b04b9c6 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1571,12 +1571,23 @@ class UsageTest extends Scope 'type' => 'ping', ])); - $firstMessage = json_decode($clients[1]->receive(), true); - // Depending on timing, the first frame we see here can be either - // a broadcast "event" (from another operation) or the "pong" - // response. Both are valid, and in either case the ping/pong - // traffic still exercises the realtime usage metrics paths. - $this->assertContains($firstMessage['type'], ['event']); + // A broadcast document "event" can still be queued for this connection. + // Read until we see the pong response (and fail fast on unexpected frames). + $pong = null; + for ($i = 0; $i < 5; $i++) { + $frame = json_decode($clients[1]->receive(), true); + $this->assertIsArray($frame); + $this->assertArrayHasKey('type', $frame); + + if ($frame['type'] === 'pong') { + $pong = $frame; + break; + } + + $this->assertEquals('event', $frame['type']); + } + + $this->assertNotNull($pong, 'Expected to receive a pong frame after ping.'); // We expect: // - connections count to remain the same From 0ea196d21c3a76fcee5d04fc3024fba1d6d6eadd Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 6 Mar 2026 15:37:30 +0530 Subject: [PATCH 07/51] updated inbound raw size to the request size --- app/realtime.php | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 8250a395a9..9ab6601ed7 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -720,18 +720,11 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many requests'); } - // Record realtime inbound bytes for this project (WS handshake + query params) - try { - $rawSize = $request->getSize(); - } catch (Throwable) { - $rawSize = \strlen((string) $request->getURI()); - } + $rawSize = $request->getSize(); - if ($rawSize > 0) { - triggerStats([ - METRIC_REALTIME_INBOUND => $rawSize, - ], $project->getId()); - } + triggerStats([ + METRIC_REALTIME_INBOUND => $rawSize, + ], $project->getId()); /* * Validate Client Domain - Check to avoid CSRF attack. From ab0fa70bca60c26d191f4a43499f8908694e6ec4 Mon Sep 17 00:00:00 2001 From: eldadfux Date: Mon, 9 Mar 2026 19:31:44 +0100 Subject: [PATCH 08/51] Enhance project context validation in realtime message handling. Ensure that projectId is checked for emptiness before processing messages, and enforce project context requirement for non-ping messages. --- app/realtime.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/realtime.php b/app/realtime.php index e0591a2596..0239e70f22 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -763,7 +763,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $database = getConsoleDB(); $database->setAuthorization($authorization); - if ($projectId !== 'console') { + if (!empty($projectId) && $projectId !== 'console') { $project = $authorization->skip(fn () => $database->getDocument('projects', $projectId)); $database = getProjectDB($project); @@ -795,6 +795,11 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re throw new Exception(Exception::REALTIME_MESSAGE_FORMAT_INVALID, 'Message format is not valid.'); } + // Ping does not require project context; other messages do (e.g. after unsubscribe during auth) + if (empty($projectId) && ($message['type'] ?? '') !== 'ping') { + throw new Exception(Exception::REALTIME_POLICY_VIOLATION, 'Missing project context. Reconnect to the project first.'); + } + switch ($message['type']) { case 'ping': $server->send([$connection], json_encode([ From 39f3bc7b9dfc09d6c277b01a6efc61a43e30e637 Mon Sep 17 00:00:00 2001 From: eldadfux Date: Mon, 9 Mar 2026 20:08:41 +0100 Subject: [PATCH 09/51] Fix SDK namespace call --- app/controllers/general.php | 3 +++ app/http.php | 3 +++ 2 files changed, 6 insertions(+) diff --git a/app/controllers/general.php b/app/controllers/general.php index 57edd98bc4..f77aa3ec52 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -1399,6 +1399,9 @@ Http::error() $sdk = $route?->getLabel("sdk", false); $action = 'UNKNOWN_NAMESPACE.UNKNOWN.METHOD'; if (!empty($sdk)) { + if (\is_array($sdk)) { + $sdk = $sdk[0]; + } /** @var \Appwrite\SDK\Method $sdk */ $action = $sdk->getNamespace() . '.' . $sdk->getMethodName(); } elseif ($route === null) { diff --git a/app/http.php b/app/http.php index 7f771de130..1302940856 100644 --- a/app/http.php +++ b/app/http.php @@ -581,6 +581,9 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool $action = 'UNKNOWN_NAMESPACE.UNKNOWN.METHOD'; if (!empty($sdk)) { + if (\is_array($sdk)) { + $sdk = $sdk[0]; + } /** @var Appwrite\SDK\Method $sdk */ $action = $sdk->getNamespace() . '.' . $sdk->getMethodName(); } elseif ($route === null) { From a0167d6c6c18560cd8d6d4b8762437c7358916aa Mon Sep 17 00:00:00 2001 From: eldadfux Date: Mon, 9 Mar 2026 20:17:32 +0100 Subject: [PATCH 10/51] Fix for when vcs comment is empty --- src/Appwrite/Vcs/Comment.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Vcs/Comment.php b/src/Appwrite/Vcs/Comment.php index 90f9c8f95d..e6d6996748 100644 --- a/src/Appwrite/Vcs/Comment.php +++ b/src/Appwrite/Vcs/Comment.php @@ -251,7 +251,7 @@ class Comment $json = \base64_decode($state); $builds = \json_decode($json, true); - $this->builds = $builds; + $this->builds = \is_array($builds) ? $builds : []; return $this; } From eccc39a4669db5afb52e35928e071524329a2a05 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 10 Mar 2026 12:15:25 +0530 Subject: [PATCH 11/51] refactor: remove realtime metrics from project usage endpoints and related tests --- app/controllers/api/project.php | 34 --- app/realtime.php | 59 +---- .../Utopia/Response/Model/UsageProject.php | 39 --- tests/e2e/General/UsageTest.php | 236 ------------------ 4 files changed, 1 insertion(+), 367 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 1fd33a5db0..d24519e3fb 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -72,10 +72,6 @@ Http::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, - METRIC_REALTIME_CONNECTIONS, - METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, - METRIC_REALTIME_INBOUND, - METRIC_REALTIME_OUTBOUND, ], 'period' => [ METRIC_NETWORK_REQUESTS, @@ -89,10 +85,6 @@ Http::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, - METRIC_REALTIME_CONNECTIONS, - METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, - METRIC_REALTIME_INBOUND, - METRIC_REALTIME_OUTBOUND, ] ]; @@ -355,26 +347,6 @@ Http::get('/v1/project/usage') ]; } - // Realtime bandwidth = realtime.inbound + realtime.outbound (per bucket) - $realtimeProjectBandwidth = []; - foreach ($usage[METRIC_REALTIME_INBOUND] as $item) { - $realtimeProjectBandwidth[$item['date']] ??= 0; - $realtimeProjectBandwidth[$item['date']] += $item['value']; - } - - foreach ($usage[METRIC_REALTIME_OUTBOUND] as $item) { - $realtimeProjectBandwidth[$item['date']] ??= 0; - $realtimeProjectBandwidth[$item['date']] += $item['value']; - } - - $realtimeBandwidth = []; - foreach ($realtimeProjectBandwidth as $date => $value) { - $realtimeBandwidth[] = [ - 'date' => $date, - 'value' => $value - ]; - } - $response->dynamic(new Document([ 'requests' => ($usage[METRIC_NETWORK_REQUESTS]), 'network' => $network, @@ -395,16 +367,10 @@ Http::get('/v1/project/usage') 'deploymentsStorageTotal' => $total[METRIC_DEPLOYMENTS_STORAGE], 'databasesReadsTotal' => $total[METRIC_DATABASES_OPERATIONS_READS], 'databasesWritesTotal' => $total[METRIC_DATABASES_OPERATIONS_WRITES], - 'realtimeConnectionsTotal' => $total[METRIC_REALTIME_CONNECTIONS], - 'realtimeMessagesTotal' => $total[METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT], - 'realtimeBandwidthTotal' => ($total[METRIC_REALTIME_INBOUND] ?? 0) + ($total[METRIC_REALTIME_OUTBOUND] ?? 0), 'executionsBreakdown' => $executionsBreakdown, 'bucketsBreakdown' => $bucketsBreakdown, 'databasesReads' => $usage[METRIC_DATABASES_OPERATIONS_READS], 'databasesWrites' => $usage[METRIC_DATABASES_OPERATIONS_WRITES], - 'realtimeConnections' => $usage[METRIC_REALTIME_CONNECTIONS], - 'realtimeMessages' => $usage[METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT], - 'realtimeBandwidth' => $realtimeBandwidth, 'databasesStorageBreakdown' => $databasesStorageBreakdown, 'executionsMbSecondsBreakdown' => $executionsMbSecondsBreakdown, 'buildsMbSecondsBreakdown' => $buildsMbSecondsBreakdown, diff --git a/app/realtime.php b/app/realtime.php index 9ab6601ed7..1e58832203 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -1,6 +1,5 @@ getSequence()])) { - return $ctx['queueForStatsUsage'][$project->getSequence()]; - } - - global $register; - - /** @var Group $pools */ - $pools = $register->get('pools'); - - $queue = new StatsUsage(new BrokerPool(publisher: $pools->get('publisher'))); - $queue->setProject($project); - - return $ctx['queueForStatsUsage'][$project->getSequence()] = $queue; - } -} - if (!function_exists('triggerStats')) { - /** - * Trigger realtime usage stats with a generic metric map. - * - * @param array $event Metrics in the form METRIC_CONSTANT => value - */ function triggerStats(array $event, string $projectId): void { - if (empty($projectId)) { - return; - } - - try { - $consoleDB = getConsoleDB(); - /** @var Document $project */ - $project = $consoleDB->getAuthorization()->skip( - fn () => $consoleDB->getDocument('projects', $projectId) - ); - - if ($project->isEmpty()) { - return; - } - - $queueForStatsUsage = getQueueForStatsUsageForProject($project); - - foreach ($event as $metric => $value) { - $queueForStatsUsage->addMetric($metric, $value); - } - - $queueForStatsUsage->trigger(); - $queueForStatsUsage->reset(); - } catch (Throwable $th) { - logError($th, 'realtimeStats', tags: ['projectId' => $projectId]); - } + return; } } diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index c219a35f29..ee644aa845 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -100,24 +100,6 @@ class UsageProject extends Model 'default' => 0, 'example' => 0, ]) - ->addRule('realtimeConnectionsTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Current aggregated number of open Realtime connections.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('realtimeMessagesTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total number of Realtime messages sent to clients.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('realtimeBandwidthTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total consumed Realtime bandwidth (in bytes).', - 'default' => 0, - 'example' => 0, - ]) ->addRule('requests', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of requests per period.', @@ -132,27 +114,6 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) - ->addRule('realtimeConnections', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of open Realtime connections per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - ->addRule('realtimeMessages', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of Realtime messages sent to clients per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - ->addRule('realtimeBandwidth', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated consumed Realtime bandwidth (in bytes) per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) ->addRule('users', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of users per period.', diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index f51b04b9c6..cabbdf6913 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1410,242 +1410,6 @@ class UsageTest extends Scope }); } - public function testRealtimeUsageMetrics(): void - { - $user = $this->getUser(); - $session = $user['session'] ?? ''; - $projectId = $this->getProject()['$id']; - - // Baseline realtime usage before opening a new connection - $baseline = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $connectionsBefore = $baseline['body']['realtimeConnectionsTotal'] ?? 0; - $messagesBefore = $baseline['body']['realtimeMessagesTotal'] ?? 0; - - $connectionCount = 3; - $clients = []; - - for ($i = 0; $i < $connectionCount; $i++) { - $client = $this->getWebsocket(['documents'], [ - 'origin' => 'http://localhost', - 'cookie' => 'a_session_' . $projectId . '=' . $session, - ], null); - - $connected = json_decode($client->receive(), true); - $this->assertEquals('connected', $connected['type']); - - $clients[] = $client; - } - - try { - $database = $this->client->call(Client::METHOD_POST, '/databases', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - 'x-appwrite-key' => $this->getProject()['apiKey'], - ], [ - 'databaseId' => ID::unique(), - 'name' => 'Realtime Usage DB', - ]); - - $databaseId = $database['body']['$id']; - - $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - 'x-appwrite-key' => $this->getProject()['apiKey'], - ], [ - 'collectionId' => ID::unique(), - 'name' => 'Realtime Usage Collection', - 'permissions' => [ - Permission::create(Role::user($user['$id'])), - ], - 'documentSecurity' => true, - ]); - - $collectionId = $collection['body']['$id']; - - $attribute = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - 'x-appwrite-key' => $this->getProject()['apiKey'], - ], [ - 'key' => 'name', - 'size' => 256, - 'required' => true, - ]); - - $this->assertEquals(202, $attribute['headers']['status-code']); - - $this->assertEventually(function () use ($databaseId, $collectionId, $projectId) { - $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/name', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - 'x-appwrite-key' => $this->getProject()['apiKey'], - ]); - $this->assertEquals('available', $response['body']['status']); - }, 30000, 250); - - $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - ], $this->getHeaders()), [ - 'documentId' => ID::unique(), - 'data' => [ - 'name' => 'Realtime Usage Doc', - ], - 'permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - ]); - - $this->assertEquals(201, $document['headers']['status-code']); - - $event = json_decode($clients[0]->receive(), true); - $this->assertEquals('event', $event['type']); - - // After creating a document we expect all connections to receive an event - $this->assertEventually(function () use ($connectionsBefore, $messagesBefore, $connectionCount) { - $response = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $this->assertEquals(200, $response['headers']['status-code']); - - $this->assertArrayHasKey('realtimeConnectionsTotal', $response['body']); - $this->assertArrayHasKey('realtimeMessagesTotal', $response['body']); - $this->assertArrayHasKey('realtimeBandwidthTotal', $response['body']); - $this->assertArrayHasKey('realtimeConnections', $response['body']); - $this->assertArrayHasKey('realtimeMessages', $response['body']); - $this->assertArrayHasKey('realtimeBandwidth', $response['body']); - - // We expect exactly $connectionCount additional open connections and $connectionCount additional message deliveries - $this->assertEquals($connectionsBefore + $connectionCount, $response['body']['realtimeConnectionsTotal']); - $this->assertEquals($messagesBefore + $connectionCount, $response['body']['realtimeMessagesTotal']); - $this->assertGreaterThan(0, $response['body']['realtimeBandwidthTotal']); - - $this->validateDates($response['body']['realtimeConnections']); - $this->validateDates($response['body']['realtimeMessages']); - $this->validateDates($response['body']['realtimeBandwidth']); - }, 60000, 2000); - - // Capture a snapshot of usage after the broadcasted document event - $afterEventUsage = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $this->assertEquals(200, $afterEventUsage['headers']['status-code']); - - $connectionsAfterEvent = $afterEventUsage['body']['realtimeConnectionsTotal'] ?? 0; - $messagesAfterEvent = $afterEventUsage['body']['realtimeMessagesTotal'] ?? 0; - $bandwidthAfterEvent = $afterEventUsage['body']['realtimeBandwidthTotal'] ?? 0; - - // Send a ping over an existing connection to exercise the ping/pong - // realtime usage metrics path (inbound + outbound bytes) without - // generating additional "messages sent" usage. - $clients[1]->send(json_encode([ - 'type' => 'ping', - ])); - - // A broadcast document "event" can still be queued for this connection. - // Read until we see the pong response (and fail fast on unexpected frames). - $pong = null; - for ($i = 0; $i < 5; $i++) { - $frame = json_decode($clients[1]->receive(), true); - $this->assertIsArray($frame); - $this->assertArrayHasKey('type', $frame); - - if ($frame['type'] === 'pong') { - $pong = $frame; - break; - } - - $this->assertEquals('event', $frame['type']); - } - - $this->assertNotNull($pong, 'Expected to receive a pong frame after ping.'); - - // We expect: - // - connections count to remain the same - // - messages count to remain the same (no new broadcast events) - // - bandwidth total to increase because of ping/pong traffic - $this->assertEventually(function () use ($connectionsAfterEvent, $messagesAfterEvent, $bandwidthAfterEvent) { - $response = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $this->assertEquals(200, $response['headers']['status-code']); - - $this->assertEquals($connectionsAfterEvent, $response['body']['realtimeConnectionsTotal']); - $this->assertEquals($messagesAfterEvent, $response['body']['realtimeMessagesTotal']); - $this->assertGreaterThan($bandwidthAfterEvent, $response['body']['realtimeBandwidthTotal']); - }, 60000, 2000); - - // Now close a single connection and ensure the counters reflect it - $clients[0]->close(); - - $this->assertEventually(function () use ($connectionsBefore, $messagesBefore, $connectionCount) { - $response = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $this->assertEquals(200, $response['headers']['status-code']); - - $this->assertArrayHasKey('realtimeConnectionsTotal', $response['body']); - $this->assertArrayHasKey('realtimeMessagesTotal', $response['body']); - $this->assertArrayHasKey('realtimeBandwidthTotal', $response['body']); - - // One of the connections is closed, so we expect one less open connection. - // Messages and bandwidth are cumulative and should not decrease. - $this->assertEquals($connectionsBefore + $connectionCount - 1, $response['body']['realtimeConnectionsTotal']); - $this->assertEquals($messagesBefore + $connectionCount, $response['body']['realtimeMessagesTotal']); - $this->assertGreaterThan(0, $response['body']['realtimeBandwidthTotal']); - }, 60000, 2000); - } finally { - foreach ($clients as $client) { - $client->close(); - } - } - } - public function tearDown(): void { $this->projectId = ''; From 7f4cba276e63700c75052bade5e41a62d11e6414 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 10 Mar 2026 12:19:46 +0530 Subject: [PATCH 12/51] updated tests --- tests/e2e/General/UsageTest.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index cabbdf6913..e7b5b1d844 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -11,7 +11,6 @@ use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideServer; use Tests\E2E\Services\Functions\FunctionsBase; -use Tests\E2E\Services\Realtime\RealtimeBase; use Tests\E2E\Services\Sites\SitesBase; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -24,7 +23,6 @@ class UsageTest extends Scope use ProjectCustom; use SideServer; use FunctionsBase; - use RealtimeBase; use SitesBase { FunctionsBase::createDeployment insteadof SitesBase; FunctionsBase::setupDeployment insteadof SitesBase; From ee2e41fa45fc358ce2266f4b4a2b6367b227e408 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Tue, 10 Mar 2026 09:20:57 +0000 Subject: [PATCH 13/51] feat: add messaging resource migration support --- composer.json | 2 +- composer.lock | 149 ++-- src/Appwrite/Platform/Workers/Migrations.php | 10 + .../Utopia/Response/Model/MigrationReport.php | 24 + .../Services/Migrations/MigrationsBase.php | 838 ++++++++++++++++++ 5 files changed, 947 insertions(+), 76 deletions(-) diff --git a/composer.json b/composer.json index d7f0ae66b5..abe51e500b 100644 --- a/composer.json +++ b/composer.json @@ -70,7 +70,7 @@ "utopia-php/locale": "0.8.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.20.*", - "utopia-php/migration": "1.6.*", + "utopia-php/migration": "1.7.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "1.*", "utopia-php/span": "1.1.*", diff --git a/composer.lock b/composer.lock index 86093899d0..738db8f287 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1cc64e07484256225f56bd525674c3b8", + "content-hash": "b99693284208ff3d006260a089a4f7b9", "packages": [ { "name": "adhocore/jwt", @@ -4058,16 +4058,16 @@ }, { "name": "utopia-php/domains", - "version": "1.0.5", + "version": "1.0.2", "source": { "type": "git", "url": "https://github.com/utopia-php/domains.git", - "reference": "0edf6bb2b07f30db849a267027077bf5abb994c6" + "reference": "b4896a6746f0fbe29dfd5e32f7790bd94c1af1e6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/domains/zipball/0edf6bb2b07f30db849a267027077bf5abb994c6", - "reference": "0edf6bb2b07f30db849a267027077bf5abb994c6", + "url": "https://api.github.com/repos/utopia-php/domains/zipball/b4896a6746f0fbe29dfd5e32f7790bd94c1af1e6", + "reference": "b4896a6746f0fbe29dfd5e32f7790bd94c1af1e6", "shasum": "" }, "require": { @@ -4114,9 +4114,9 @@ ], "support": { "issues": "https://github.com/utopia-php/domains/issues", - "source": "https://github.com/utopia-php/domains/tree/1.0.5" + "source": "https://github.com/utopia-php/domains/tree/1.0.2" }, - "time": "2026-03-03T09:20:50+00:00" + "time": "2026-02-25T08:18:25+00:00" }, { "name": "utopia-php/dsn", @@ -4517,16 +4517,16 @@ }, { "name": "utopia-php/migration", - "version": "1.6.3", + "version": "1.7.0", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "c2d016944cb029fa5ff822ceee704785a06ef289" + "reference": "97583ae502e40621ea91a71de19d053c5ae2e558" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/c2d016944cb029fa5ff822ceee704785a06ef289", - "reference": "c2d016944cb029fa5ff822ceee704785a06ef289", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/97583ae502e40621ea91a71de19d053c5ae2e558", + "reference": "97583ae502e40621ea91a71de19d053c5ae2e558", "shasum": "" }, "require": { @@ -4566,9 +4566,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/1.6.3" + "source": "https://github.com/utopia-php/migration/tree/1.7.0" }, - "time": "2026-03-04T07:08:22+00:00" + "time": "2026-03-10T06:36:27+00:00" }, { "name": "utopia-php/mongo", @@ -5215,16 +5215,16 @@ }, { "name": "utopia-php/vcs", - "version": "2.0.1", + "version": "2.0.0", "source": { "type": "git", "url": "https://github.com/utopia-php/vcs.git", - "reference": "92a1650824ba0c5e6a1bc46e622ac87c50a08920" + "reference": "058049326e04a2a0c2f0ce8ad00c7e84825aba14" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/vcs/zipball/92a1650824ba0c5e6a1bc46e622ac87c50a08920", - "reference": "92a1650824ba0c5e6a1bc46e622ac87c50a08920", + "url": "https://api.github.com/repos/utopia-php/vcs/zipball/058049326e04a2a0c2f0ce8ad00c7e84825aba14", + "reference": "058049326e04a2a0c2f0ce8ad00c7e84825aba14", "shasum": "" }, "require": { @@ -5258,9 +5258,9 @@ ], "support": { "issues": "https://github.com/utopia-php/vcs/issues", - "source": "https://github.com/utopia-php/vcs/tree/2.0.1" + "source": "https://github.com/utopia-php/vcs/tree/2.0.0" }, - "time": "2026-02-27T12:18:49+00:00" + "time": "2026-02-25T11:36:45+00:00" }, { "name": "utopia-php/websocket", @@ -5438,16 +5438,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "1.11.6", + "version": "1.11.1", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "f80e302d000cdc2f98b4bb5ff2fc3bd0bdff7b38" + "reference": "6ff411f26f2750eea05c7598c14bb3a2ada898cb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/f80e302d000cdc2f98b4bb5ff2fc3bd0bdff7b38", - "reference": "f80e302d000cdc2f98b4bb5ff2fc3bd0bdff7b38", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/6ff411f26f2750eea05c7598c14bb3a2ada898cb", + "reference": "6ff411f26f2750eea05c7598c14bb3a2ada898cb", "shasum": "" }, "require": { @@ -5483,22 +5483,22 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/1.11.6" + "source": "https://github.com/appwrite/sdk-generator/tree/1.11.1" }, - "time": "2026-03-09T07:12:51+00:00" + "time": "2026-02-25T07:15:19+00:00" }, { "name": "brianium/paratest", - "version": "v7.19.1", + "version": "v7.19.0", "source": { "type": "git", "url": "https://github.com/paratestphp/paratest.git", - "reference": "95b03194f4cdf5c83175ceead673e21cb66465e7" + "reference": "7c6c29af7c4b406b49ce0c6b0a3a81d3684474e6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/paratestphp/paratest/zipball/95b03194f4cdf5c83175ceead673e21cb66465e7", - "reference": "95b03194f4cdf5c83175ceead673e21cb66465e7", + "url": "https://api.github.com/repos/paratestphp/paratest/zipball/7c6c29af7c4b406b49ce0c6b0a3a81d3684474e6", + "reference": "7c6c29af7c4b406b49ce0c6b0a3a81d3684474e6", "shasum": "" }, "require": { @@ -5512,7 +5512,7 @@ "phpunit/php-code-coverage": "^12.5.3 || ^13.0.1", "phpunit/php-file-iterator": "^6.0.1 || ^7", "phpunit/php-timer": "^8 || ^9", - "phpunit/phpunit": "^12.5.14 || ^13.0.5", + "phpunit/phpunit": "^12.5.9 || ^13", "sebastian/environment": "^8.0.3 || ^9", "symfony/console": "^7.4.4 || ^8.0.4", "symfony/process": "^7.4.5 || ^8.0.5" @@ -5522,10 +5522,10 @@ "ext-pcntl": "*", "ext-pcov": "*", "ext-posix": "*", - "phpstan/phpstan": "^2.1.40", - "phpstan/phpstan-deprecation-rules": "^2.0.4", - "phpstan/phpstan-phpunit": "^2.0.16", - "phpstan/phpstan-strict-rules": "^2.0.10", + "phpstan/phpstan": "^2.1.38", + "phpstan/phpstan-deprecation-rules": "^2.0.3", + "phpstan/phpstan-phpunit": "^2.0.12", + "phpstan/phpstan-strict-rules": "^2.0.8", "symfony/filesystem": "^7.4.0 || ^8.0.1" }, "bin": [ @@ -5566,7 +5566,7 @@ ], "support": { "issues": "https://github.com/paratestphp/paratest/issues", - "source": "https://github.com/paratestphp/paratest/tree/v7.19.1" + "source": "https://github.com/paratestphp/paratest/tree/v7.19.0" }, "funding": [ { @@ -5578,7 +5578,7 @@ "type": "paypal" } ], - "time": "2026-02-25T14:53:45+00:00" + "time": "2026-02-06T10:53:26+00:00" }, { "name": "czproject/git-php", @@ -6398,16 +6398,16 @@ }, { "name": "phpbench/phpbench", - "version": "1.5.1", + "version": "1.4.3", "source": { "type": "git", "url": "https://github.com/phpbench/phpbench.git", - "reference": "9a28fd0833f11171b949843c6fd663eb69b6d14c" + "reference": "b641dde59d969ea42eed70a39f9b51950bc96878" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpbench/phpbench/zipball/9a28fd0833f11171b949843c6fd663eb69b6d14c", - "reference": "9a28fd0833f11171b949843c6fd663eb69b6d14c", + "url": "https://api.github.com/repos/phpbench/phpbench/zipball/b641dde59d969ea42eed70a39f9b51950bc96878", + "reference": "b641dde59d969ea42eed70a39f9b51950bc96878", "shasum": "" }, "require": { @@ -6418,7 +6418,7 @@ "ext-reflection": "*", "ext-spl": "*", "ext-tokenizer": "*", - "php": "^8.2", + "php": "^8.1", "phpbench/container": "^2.2", "psr/log": "^1.1 || ^2.0 || ^3.0", "seld/jsonlint": "^1.1", @@ -6438,9 +6438,8 @@ "phpstan/extension-installer": "^1.1", "phpstan/phpstan": "^1.0", "phpstan/phpstan-phpunit": "^1.0", - "phpunit/phpunit": "^11.5", + "phpunit/phpunit": "^10.4 || ^11.0", "rector/rector": "^1.2", - "sebastian/exporter": "^6.3.2", "symfony/error-handler": "^6.1 || ^7.0 || ^8.0", "symfony/var-dumper": "^6.1 || ^7.0 || ^8.0" }, @@ -6485,7 +6484,7 @@ ], "support": { "issues": "https://github.com/phpbench/phpbench/issues", - "source": "https://github.com/phpbench/phpbench/tree/1.5.1" + "source": "https://github.com/phpbench/phpbench/tree/1.4.3" }, "funding": [ { @@ -6493,15 +6492,15 @@ "type": "github" } ], - "time": "2026-03-05T08:18:58+00:00" + "time": "2025-11-06T19:07:31+00:00" }, { "name": "phpstan/phpstan", - "version": "1.12.33", + "version": "1.12.32", "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/37982d6fc7cbb746dda7773530cda557cdf119e1", - "reference": "37982d6fc7cbb746dda7773530cda557cdf119e1", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/2770dcdf5078d0b0d53f94317e06affe88419aa8", + "reference": "2770dcdf5078d0b0d53f94317e06affe88419aa8", "shasum": "" }, "require": { @@ -6546,7 +6545,7 @@ "type": "github" } ], - "time": "2026-02-28T20:30:03+00:00" + "time": "2025-09-30T10:16:31+00:00" }, { "name": "phpunit/php-code-coverage", @@ -8096,16 +8095,16 @@ }, { "name": "symfony/console", - "version": "v8.0.7", + "version": "v8.0.4", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "15ed9008a4ebe2d6a78e4937f74e0c13ef2e618a" + "reference": "ace03c4cf9805080ff40cbeec69fca180c339a3b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/15ed9008a4ebe2d6a78e4937f74e0c13ef2e618a", - "reference": "15ed9008a4ebe2d6a78e4937f74e0c13ef2e618a", + "url": "https://api.github.com/repos/symfony/console/zipball/ace03c4cf9805080ff40cbeec69fca180c339a3b", + "reference": "ace03c4cf9805080ff40cbeec69fca180c339a3b", "shasum": "" }, "require": { @@ -8162,7 +8161,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v8.0.7" + "source": "https://github.com/symfony/console/tree/v8.0.4" }, "funding": [ { @@ -8182,20 +8181,20 @@ "type": "tidelift" } ], - "time": "2026-03-06T14:06:22+00:00" + "time": "2026-01-13T13:06:50+00:00" }, { "name": "symfony/filesystem", - "version": "v8.0.6", + "version": "v8.0.1", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "7bf9162d7a0dff98d079b72948508fa48018a770" + "reference": "d937d400b980523dc9ee946bb69972b5e619058d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/7bf9162d7a0dff98d079b72948508fa48018a770", - "reference": "7bf9162d7a0dff98d079b72948508fa48018a770", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/d937d400b980523dc9ee946bb69972b5e619058d", + "reference": "d937d400b980523dc9ee946bb69972b5e619058d", "shasum": "" }, "require": { @@ -8232,7 +8231,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v8.0.6" + "source": "https://github.com/symfony/filesystem/tree/v8.0.1" }, "funding": [ { @@ -8252,20 +8251,20 @@ "type": "tidelift" } ], - "time": "2026-02-25T16:59:43+00:00" + "time": "2025-12-01T09:13:36+00:00" }, { "name": "symfony/finder", - "version": "v8.0.6", + "version": "v8.0.5", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "441404f09a54de6d1bd6ad219e088cdf4c91f97c" + "reference": "8bd576e97c67d45941365bf824e18dc8538e6eb0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/441404f09a54de6d1bd6ad219e088cdf4c91f97c", - "reference": "441404f09a54de6d1bd6ad219e088cdf4c91f97c", + "url": "https://api.github.com/repos/symfony/finder/zipball/8bd576e97c67d45941365bf824e18dc8538e6eb0", + "reference": "8bd576e97c67d45941365bf824e18dc8538e6eb0", "shasum": "" }, "require": { @@ -8300,7 +8299,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v8.0.6" + "source": "https://github.com/symfony/finder/tree/v8.0.5" }, "funding": [ { @@ -8320,7 +8319,7 @@ "type": "tidelift" } ], - "time": "2026-01-29T09:41:02+00:00" + "time": "2026-01-26T15:08:38+00:00" }, { "name": "symfony/options-resolver", @@ -8790,16 +8789,16 @@ }, { "name": "symfony/string", - "version": "v8.0.6", + "version": "v8.0.4", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "6c9e1108041b5dce21a9a4984b531c4923aa9ec4" + "reference": "758b372d6882506821ed666032e43020c4f57194" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/6c9e1108041b5dce21a9a4984b531c4923aa9ec4", - "reference": "6c9e1108041b5dce21a9a4984b531c4923aa9ec4", + "url": "https://api.github.com/repos/symfony/string/zipball/758b372d6882506821ed666032e43020c4f57194", + "reference": "758b372d6882506821ed666032e43020c4f57194", "shasum": "" }, "require": { @@ -8856,7 +8855,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v8.0.6" + "source": "https://github.com/symfony/string/tree/v8.0.4" }, "funding": [ { @@ -8876,7 +8875,7 @@ "type": "tidelift" } ], - "time": "2026-02-09T10:14:57+00:00" + "time": "2026-01-12T12:37:40+00:00" }, { "name": "textalk/websocket", @@ -9108,7 +9107,7 @@ ], "aliases": [], "minimum-stability": "dev", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": true, "prefer-lowest": false, "platform": { @@ -9132,5 +9131,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.9.0" + "plugin-api-version": "2.6.0" } diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index d87edaf788..c4cb9ce415 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -317,6 +317,16 @@ class Migrations extends Action 'sites.write', 'tokens.read', 'tokens.write', + 'providers.read', + 'providers.write', + 'topics.read', + 'topics.write', + 'subscribers.read', + 'subscribers.write', + 'messages.read', + 'messages.write', + 'targets.read', + 'targets.write', ] ]); diff --git a/src/Appwrite/Utopia/Response/Model/MigrationReport.php b/src/Appwrite/Utopia/Response/Model/MigrationReport.php index 7ebc22d22e..850e4b5ae9 100644 --- a/src/Appwrite/Utopia/Response/Model/MigrationReport.php +++ b/src/Appwrite/Utopia/Response/Model/MigrationReport.php @@ -59,6 +59,30 @@ class MigrationReport extends Model 'default' => 0, 'example' => 5, ]) + ->addRule(Resource::TYPE_PROVIDER, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of providers to be migrated.', + 'default' => 0, + 'example' => 5, + ]) + ->addRule(Resource::TYPE_TOPIC, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of topics to be migrated.', + 'default' => 0, + 'example' => 10, + ]) + ->addRule(Resource::TYPE_SUBSCRIBER, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of subscribers to be migrated.', + 'default' => 0, + 'example' => 100, + ]) + ->addRule(Resource::TYPE_MESSAGE, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of messages to be migrated.', + 'default' => 0, + 'example' => 50, + ]) ->addRule('size', [ 'type' => self::TYPE_INTEGER, 'description' => 'Size of files to be migrated in mb.', diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 606f9e8127..d5fe7753a4 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1694,4 +1694,842 @@ trait MigrationsBase 'x-appwrite-key' => $this->getProject()['apiKey'] ]); } + + /** + * Messaging + */ + public function testAppwriteMigrationMessagingProvider(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid', + 'apiKey' => 'my-apikey', + 'from' => 'migration@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $this->assertNotEmpty($provider['body']['$id']); + + $providerId = $provider['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertEquals([Resource::TYPE_PROVIDER], $result['resources']); + $this->assertArrayHasKey(Resource::TYPE_PROVIDER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_PROVIDER]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($providerId, $response['body']['$id']); + $this->assertEquals('Migration Sendgrid', $response['body']['name']); + $this->assertEquals('email', $response['body']['type']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingProviderSMTP(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/smtp', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration SMTP', + 'host' => 'smtp.test.com', + 'port' => 587, + 'from' => 'migration-smtp@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_PROVIDER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_PROVIDER]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($providerId, $response['body']['$id']); + $this->assertEquals('Migration SMTP', $response['body']['name']); + $this->assertEquals('email', $response['body']['type']); + $this->assertEquals('smtp', $response['body']['provider']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingProviderTwilio(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/twilio', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Twilio', + 'from' => '+15551234567', + 'accountSid' => 'test-account-sid', + 'authToken' => 'test-auth-token', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_PROVIDER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_PROVIDER]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($providerId, $response['body']['$id']); + $this->assertEquals('Migration Twilio', $response['body']['name']); + $this->assertEquals('sms', $response['body']['type']); + $this->assertEquals('twilio', $response['body']['provider']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingTopic(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Topic', + 'apiKey' => 'my-apikey', + 'from' => 'migration-topic@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $this->assertNotEmpty($topic['body']['$id']); + + $topicId = $topic['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_TOPIC, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_TOPIC]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($topicId, $response['body']['$id']); + $this->assertEquals('Migration Topic', $response['body']['name']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingSubscriber(): void + { + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-sub@test.com', + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $this->assertEquals(1, \count($user['body']['targets'])); + $targetId = $user['body']['targets'][0]['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Subscriber', + 'apiKey' => 'my-apikey', + 'from' => uniqid() . '-migration-sub@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Subscriber Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $subscriber = $this->client->call(Client::METHOD_POST, '/messaging/topics/' . $topicId . '/subscribers', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'subscriberId' => ID::unique(), + 'targetId' => $targetId, + ]); + + $this->assertEquals(201, $subscriber['headers']['status-code']); + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_SUBSCRIBER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($topicId, $response['body']['$id']); + $this->assertGreaterThanOrEqual(1, $response['body']['emailTotal']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingMessage(): void + { + $this->getDestinationProject(true); + + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-msg@test.com', + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $this->assertEquals(1, \count($user['body']['targets'])); + $targetId = $user['body']['targets'][0]['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Message', + 'apiKey' => 'my-apikey', + 'from' => 'migration-msg@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Message Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $message = $this->client->call(Client::METHOD_POST, '/messaging/messages/email', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'messageId' => ID::unique(), + 'targets' => [$targetId], + 'topics' => [$topicId], + 'subject' => 'Migration Test Email', + 'content' => 'This is a migration test email', + 'draft' => true, + ]); + + $this->assertEquals(201, $message['headers']['status-code']); + $this->assertNotEmpty($message['body']['$id']); + + $messageId = $message['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + Resource::TYPE_MESSAGE, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_MESSAGE, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_MESSAGE]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($messageId, $response['body']['$id']); + $this->assertEquals('draft', $response['body']['status']); + $this->assertEquals('Migration Test Email', $response['body']['data']['subject']); + $this->assertEquals('This is a migration test email', $response['body']['data']['content']); + $this->assertContains($topicId, $response['body']['topics']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingSmsMessage(): void + { + $this->getDestinationProject(true); + + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-sms@test.com', + 'phone' => '+1' . str_pad((string) rand(200000000, 999999999), 10, '0', STR_PAD_LEFT), + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $this->assertGreaterThanOrEqual(1, \count($user['body']['targets'])); + + $smsTarget = null; + foreach ($user['body']['targets'] as $target) { + if ($target['providerType'] === 'sms') { + $smsTarget = $target; + break; + } + } + $this->assertNotNull($smsTarget); + $targetId = $smsTarget['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/twilio', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Twilio SMS Msg', + 'from' => '+15559876543', + 'accountSid' => 'test-account-sid', + 'authToken' => 'test-auth-token', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration SMS Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $message = $this->client->call(Client::METHOD_POST, '/messaging/messages/sms', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'messageId' => ID::unique(), + 'targets' => [$targetId], + 'topics' => [$topicId], + 'content' => 'Migration SMS test content', + 'draft' => true, + ]); + + $this->assertEquals(201, $message['headers']['status-code']); + $messageId = $message['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + Resource::TYPE_MESSAGE, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_MESSAGE, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_MESSAGE]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($messageId, $response['body']['$id']); + $this->assertEquals('draft', $response['body']['status']); + $this->assertEquals('Migration SMS test content', $response['body']['data']['content']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingScheduledMessage(): void + { + $this->getDestinationProject(true); + + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-sched@test.com', + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $targetId = $user['body']['targets'][0]['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Scheduled', + 'apiKey' => 'my-apikey', + 'from' => 'migration-sched@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Scheduled Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $subscriber = $this->client->call(Client::METHOD_POST, '/messaging/topics/' . $topicId . '/subscribers', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'subscriberId' => ID::unique(), + 'targetId' => $targetId, + ]); + + $this->assertEquals(201, $subscriber['headers']['status-code']); + + // Create a scheduled message with a future date using topics only + // Direct targets use source IDs which won't resolve in the destination via API + $futureDate = (new \DateTime('+1 year'))->format(\DateTime::ATOM); + $message = $this->client->call(Client::METHOD_POST, '/messaging/messages/email', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'messageId' => ID::unique(), + 'topics' => [$topicId], + 'subject' => 'Migration Scheduled Email', + 'content' => 'This is a scheduled migration test email', + 'scheduledAt' => $futureDate, + ]); + + $this->assertEquals(201, $message['headers']['status-code']); + $messageId = $message['body']['$id']; + $this->assertEquals('scheduled', $message['body']['status']); + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + Resource::TYPE_MESSAGE, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_MESSAGE, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_MESSAGE]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($messageId, $response['body']['$id']); + $this->assertEquals('scheduled', $response['body']['status']); + $this->assertEquals('Migration Scheduled Email', $response['body']['data']['subject']); + $this->assertEquals( + (new \DateTime($futureDate))->getTimestamp(), + (new \DateTime($response['body']['scheduledAt']))->getTimestamp(), + ); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } } From f4b8992cdee45ab90729ad20a8170e6f374c3753 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 10 Mar 2026 12:19:18 +0000 Subject: [PATCH 14/51] Enable SMTP keep-alive to reuse connections across mail jobs Reduces job processing time by avoiding repeated TCP connect, TLS handshake, and SMTP AUTH on every email sent. Co-Authored-By: Claude Opus 4.6 --- app/init/registers.php | 1 + src/Appwrite/Platform/Workers/Mails.php | 1 + 2 files changed, 2 insertions(+) diff --git a/app/init/registers.php b/app/init/registers.php index 26a9329270..7b68c2af9a 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -420,6 +420,7 @@ $register->set('smtp', function () { $mail->Password = $password; $mail->SMTPSecure = System::getEnv('_APP_SMTP_SECURE', ''); $mail->SMTPAutoTLS = false; + $mail->SMTPKeepAlive = true; $mail->CharSet = 'UTF-8'; $mail->Timeout = 10; /* Connection timeout */ $mail->getSMTPInstance()->Timelimit = 30; /* Timeout for each individual SMTP command (e.g. HELO, EHLO, etc.) */ diff --git a/src/Appwrite/Platform/Workers/Mails.php b/src/Appwrite/Platform/Workers/Mails.php index 72f7cddd06..f144c58e1b 100644 --- a/src/Appwrite/Platform/Workers/Mails.php +++ b/src/Appwrite/Platform/Workers/Mails.php @@ -214,6 +214,7 @@ class Mails extends Action $mail->Password = $password; $mail->SMTPSecure = $smtp['secure']; $mail->SMTPAutoTLS = false; + $mail->SMTPKeepAlive = true; $mail->CharSet = 'UTF-8'; $mail->Timeout = 10; /* Connection timeout */ $mail->getSMTPInstance()->Timelimit = 30; /* Timeout for each individual SMTP command (e.g. HELO, EHLO, etc.) */ From 1be142e4becf9aaf1278853f9e8da9ed4791da2f Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Tue, 10 Mar 2026 12:33:30 +0000 Subject: [PATCH 15/51] test: use database fix branch for preserveDates datetime format --- composer.json | 2 +- composer.lock | 27 ++++++++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/composer.json b/composer.json index abe51e500b..c2d2749a74 100644 --- a/composer.json +++ b/composer.json @@ -58,7 +58,7 @@ "utopia-php/compression": "0.1.*", "utopia-php/config": "1.*", "utopia-php/console": "0.1.*", - "utopia-php/database": "5.*", + "utopia-php/database": "dev-fix-preserve-dates-datetime-format as 5.3.7", "utopia-php/detector": "0.2.*", "utopia-php/domains": "1.*", "utopia-php/emails": "0.6.*", diff --git a/composer.lock b/composer.lock index 738db8f287..3caf85b8cb 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b99693284208ff3d006260a089a4f7b9", + "content-hash": "09f5b42e8589d72026a0b3731fc4f2af", "packages": [ { "name": "adhocore/jwt", @@ -3850,16 +3850,16 @@ }, { "name": "utopia-php/database", - "version": "5.3.7", + "version": "dev-fix-preserve-dates-datetime-format", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "438cc82af2981cd41ad200dd9b0df5bf00f3046a" + "reference": "49aa3181862c1370c17c2156a3b3228dca48df1e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/438cc82af2981cd41ad200dd9b0df5bf00f3046a", - "reference": "438cc82af2981cd41ad200dd9b0df5bf00f3046a", + "url": "https://api.github.com/repos/utopia-php/database/zipball/49aa3181862c1370c17c2156a3b3228dca48df1e", + "reference": "49aa3181862c1370c17c2156a3b3228dca48df1e", "shasum": "" }, "require": { @@ -3902,9 +3902,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/5.3.7" + "source": "https://github.com/utopia-php/database/tree/fix-preserve-dates-datetime-format" }, - "time": "2026-03-09T04:28:56+00:00" + "time": "2026-03-10T12:28:50+00:00" }, { "name": "utopia-php/detector", @@ -9105,9 +9105,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-fix-preserve-dates-datetime-format", + "alias": "5.3.7", + "alias_normalized": "5.3.7.0" + } + ], "minimum-stability": "dev", - "stability-flags": [], + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": true, "prefer-lowest": false, "platform": { From 33cafdfc0e8e036ddc30d4cc4ba3a20590feda72 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 10 Mar 2026 22:17:26 +0000 Subject: [PATCH 16/51] Initial plan From 1113bffb78ebf8c4caaf0e54dabf463a1951efbb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 10 Mar 2026 22:19:06 +0000 Subject: [PATCH 17/51] fix: use message timestamp for receivedAt in StatsUsage worker Co-authored-by: stnguyen90 <1477010+stnguyen90@users.noreply.github.com> --- src/Appwrite/Platform/Workers/StatsUsage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 018c192647..801790f188 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -160,7 +160,7 @@ class StatsUsage extends Action } $this->stats[$projectId]['project'] = $project; - $this->stats[$projectId]['receivedAt'] = DateTime::now(); + $this->stats[$projectId]['receivedAt'] = DateTime::format(new \DateTime('@' . $message->getTimestamp())); foreach ($payload['metrics'] ?? [] as $metric) { $this->keys++; if (!isset($this->stats[$projectId]['keys'][$metric['key']])) { From 4bf3c72196498987bdc1aab54ba117622c370c9b Mon Sep 17 00:00:00 2001 From: eldadfux Date: Wed, 11 Mar 2026 06:25:08 +0100 Subject: [PATCH 18/51] fix: allow users to update phone number to empty without causing duplicate errors --- app/controllers/api/users.php | 13 +++--- tests/e2e/Services/Users/UsersBase.php | 56 +++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index d0e5e19a51..9d04018b10 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1560,12 +1560,15 @@ Http::patch('/v1/users/:userId/phone') $oldPhone = $user->getAttribute('phone'); + // Store null instead of empty string so unique constraint allows multiple users without phone + $phoneValue = $number !== '' ? $number : null; + $user - ->setAttribute('phone', $number) + ->setAttribute('phone', $phoneValue) ->setAttribute('phoneVerification', false) ; - if (\strlen($number) !== 0) { + if ($number !== '') { $target = $dbForProject->findOne('targets', [ Query::equal('identifier', [$number]), ]); @@ -1577,7 +1580,7 @@ Http::patch('/v1/users/:userId/phone') try { $user = $dbForProject->updateDocument('users', $user->getId(), new Document([ - 'phone' => $user->getAttribute('phone'), + 'phone' => $phoneValue, 'phoneVerification' => $user->getAttribute('phoneVerification'), ])); /** @@ -1586,14 +1589,14 @@ Http::patch('/v1/users/:userId/phone') $oldTarget = $user->find('identifier', $oldPhone, 'targets'); if ($oldTarget instanceof Document && !$oldTarget->isEmpty()) { - if (\strlen($number) !== 0) { + if ($number !== '') { $dbForProject->updateDocument('targets', $oldTarget->getId(), new Document(['identifier' => $number])); $oldTarget->setAttribute('identifier', $number); } else { $dbForProject->deleteDocument('targets', $oldTarget->getId()); } } else { - if (\strlen($number) !== 0) { + if ($number !== '') { $target = $dbForProject->createDocument('targets', new Document([ '$permissions' => [ Permission::read(Role::user($user->getId())), diff --git a/tests/e2e/Services/Users/UsersBase.php b/tests/e2e/Services/Users/UsersBase.php index 5c7b289722..866ee591a2 100644 --- a/tests/e2e/Services/Users/UsersBase.php +++ b/tests/e2e/Services/Users/UsersBase.php @@ -1596,7 +1596,7 @@ trait UsersBase ]); $this->assertEquals($user['headers']['status-code'], 200); - $this->assertEquals($user['body']['phone'], $updatedNumber); + $this->assertEmpty($user['body']['phone'] ?? ''); $user = $this->client->call(Client::METHOD_GET, '/users/' . $data['userId'], array_merge([ 'content-type' => 'application/json', @@ -1604,7 +1604,7 @@ trait UsersBase ], $this->getHeaders())); $this->assertEquals($user['headers']['status-code'], 200); - $this->assertEquals($user['body']['phone'], $updatedNumber); + $this->assertEmpty($user['body']['phone'] ?? ''); $updatedNumber = "+910000000000"; //dummy number $user = $this->client->call(Client::METHOD_PATCH, '/users/' . $data['userId'] . '/phone', array_merge([ @@ -1648,6 +1648,58 @@ trait UsersBase static::$userNumberUpdated = true; } + public function testUpdateTwoUsersPhoneToEmpty(): void + { + $projectId = $this->getProject()['$id']; + $headers = array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders()); + + // Create two users with distinct valid phone numbers + $user1 = $this->client->call(Client::METHOD_POST, '/users', $headers, [ + 'userId' => ID::unique(), + 'email' => 'user1-phone-empty-test@appwrite.io', + 'password' => 'password', + 'name' => 'User One', + 'phone' => '+16175551201', + ]); + $this->assertEquals(201, $user1['headers']['status-code']); + $this->assertEquals('+16175551201', $user1['body']['phone']); + + $user2 = $this->client->call(Client::METHOD_POST, '/users', $headers, [ + 'userId' => ID::unique(), + 'email' => 'user2-phone-empty-test@appwrite.io', + 'password' => 'password', + 'name' => 'User Two', + 'phone' => '+16175551202', + ]); + $this->assertEquals(201, $user2['headers']['status-code']); + $this->assertEquals('+16175551202', $user2['body']['phone']); + + // Update first user's phone to empty - must succeed + $response1 = $this->client->call(Client::METHOD_PATCH, '/users/' . $user1['body']['$id'] . '/phone', $headers, [ + 'number' => '', + ]); + $this->assertEquals(200, $response1['headers']['status-code'], 'First user phone should update to empty'); + $this->assertEmpty($response1['body']['phone'] ?? ''); + + // Update second user's phone to empty - must succeed (would fail with duplicate if empty was stored as '') + $response2 = $this->client->call(Client::METHOD_PATCH, '/users/' . $user2['body']['$id'] . '/phone', $headers, [ + 'number' => '', + ]); + $this->assertEquals(200, $response2['headers']['status-code'], 'Second user phone should update to empty without duplicate error'); + $this->assertEmpty($response2['body']['phone'] ?? ''); + + // Verify both users have empty phone via GET + $get1 = $this->client->call(Client::METHOD_GET, '/users/' . $user1['body']['$id'], $headers); + $get2 = $this->client->call(Client::METHOD_GET, '/users/' . $user2['body']['$id'], $headers); + $this->assertEquals(200, $get1['headers']['status-code']); + $this->assertEquals(200, $get2['headers']['status-code']); + $this->assertEmpty($get1['body']['phone'] ?? ''); + $this->assertEmpty($get2['body']['phone'] ?? ''); + } + public function testUpdateUserNumberSearch(): void { $data = $this->ensureUserNumberUpdated(); From 0ebda89adf98c475d3ea35157be4a4a60a621aa5 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 11 Mar 2026 05:56:44 +0000 Subject: [PATCH 19/51] update utopia-php/database to released 5.3.8 --- composer.json | 2 +- composer.lock | 29 ++++++++++------------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/composer.json b/composer.json index c2d2749a74..abe51e500b 100644 --- a/composer.json +++ b/composer.json @@ -58,7 +58,7 @@ "utopia-php/compression": "0.1.*", "utopia-php/config": "1.*", "utopia-php/console": "0.1.*", - "utopia-php/database": "dev-fix-preserve-dates-datetime-format as 5.3.7", + "utopia-php/database": "5.*", "utopia-php/detector": "0.2.*", "utopia-php/domains": "1.*", "utopia-php/emails": "0.6.*", diff --git a/composer.lock b/composer.lock index 3caf85b8cb..0e25d3bc5f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "09f5b42e8589d72026a0b3731fc4f2af", + "content-hash": "b99693284208ff3d006260a089a4f7b9", "packages": [ { "name": "adhocore/jwt", @@ -3850,16 +3850,16 @@ }, { "name": "utopia-php/database", - "version": "dev-fix-preserve-dates-datetime-format", + "version": "5.3.8", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "49aa3181862c1370c17c2156a3b3228dca48df1e" + "reference": "4920bb60afb98d4bd81f4d331765716ae1d40255" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/49aa3181862c1370c17c2156a3b3228dca48df1e", - "reference": "49aa3181862c1370c17c2156a3b3228dca48df1e", + "url": "https://api.github.com/repos/utopia-php/database/zipball/4920bb60afb98d4bd81f4d331765716ae1d40255", + "reference": "4920bb60afb98d4bd81f4d331765716ae1d40255", "shasum": "" }, "require": { @@ -3902,9 +3902,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/fix-preserve-dates-datetime-format" + "source": "https://github.com/utopia-php/database/tree/5.3.8" }, - "time": "2026-03-10T12:28:50+00:00" + "time": "2026-03-11T01:03:34+00:00" }, { "name": "utopia-php/detector", @@ -9105,18 +9105,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-fix-preserve-dates-datetime-format", - "alias": "5.3.7", - "alias_normalized": "5.3.7.0" - } - ], + "aliases": [], "minimum-stability": "dev", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": {}, "prefer-stable": true, "prefer-lowest": false, "platform": { @@ -9140,5 +9131,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.9.0" } From 7d15d7e1f57dc3bc52b0c8f6b9d41b4cab12b60e Mon Sep 17 00:00:00 2001 From: Hemachandar <132386067+hmacr@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:15:50 +0530 Subject: [PATCH 20/51] Rename time-travel task (#11523) * Rename time-travel task * rename --- Dockerfile | 2 +- bin/{time-travel => task-time-travel} | 0 tests/e2e/Services/Functions/FunctionsConsoleClientTest.php | 2 +- tests/e2e/Services/Sites/SitesConsoleClientTest.php | 4 ++-- 4 files changed, 4 insertions(+), 4 deletions(-) rename bin/{time-travel => task-time-travel} (100%) diff --git a/Dockerfile b/Dockerfile index 266d4501d0..210c2bc3d9 100755 --- a/Dockerfile +++ b/Dockerfile @@ -70,7 +70,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/sdks && \ chmod +x /usr/local/bin/specs && \ chmod +x /usr/local/bin/ssl && \ - chmod +x /usr/local/bin/time-travel && \ + chmod +x /usr/local/bin/task-time-travel && \ chmod +x /usr/local/bin/screenshot && \ chmod +x /usr/local/bin/test && \ chmod +x /usr/local/bin/upgrade && \ diff --git a/bin/time-travel b/bin/task-time-travel similarity index 100% rename from bin/time-travel rename to bin/task-time-travel diff --git a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php index cc18d14a8e..06044d9984 100644 --- a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php @@ -686,7 +686,7 @@ class FunctionsConsoleClientTest extends Scope $stdout = ''; $stderr = ''; - $code = Console::execute("docker exec appwrite time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); + $code = Console::execute("docker exec appwrite task-time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); $this->assertSame(0, $code, "Time-travel command failed with code $code: $stderr ($stdout)"); $stdout = ''; diff --git a/tests/e2e/Services/Sites/SitesConsoleClientTest.php b/tests/e2e/Services/Sites/SitesConsoleClientTest.php index 2a94dded5f..2e0e1a892d 100644 --- a/tests/e2e/Services/Sites/SitesConsoleClientTest.php +++ b/tests/e2e/Services/Sites/SitesConsoleClientTest.php @@ -180,12 +180,12 @@ class SitesConsoleClientTest extends Scope $stdout = ''; $stderr = ''; - $code = Console::execute("docker exec appwrite-task-maintenance time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); + $code = Console::execute("docker exec appwrite task-time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); $this->assertSame(0, $code, "Time-travel command failed with code $code: $stderr ($stdout)"); $stdout = ''; $stderr = ''; - $code = Console::execute("docker exec appwrite-task-maintenance maintenance --type=trigger", '', $stdout, $stderr); + $code = Console::execute("docker exec appwrite maintenance --type=trigger", '', $stdout, $stderr); $this->assertSame(0, $code, "Maintenance command failed with code $code: $stderr ($stdout)"); $this->assertEventually(function () use ($siteId) { From e557a5fc6e47293a82d5257e3d2672e37c946e1b Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 19:41:49 +0000 Subject: [PATCH 21/51] Add Docker Hub login and parallel image pulls to CI Login to Docker Hub in all test jobs to avoid rate limits, add `docker compose pull --quiet` to parallelize image downloads before `docker compose up`, and replace sed-based .env overrides with native GitHub Actions env fields. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 74 +++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0cff6288e2..8e937b261a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,6 +53,12 @@ jobs: with: submodules: recursive + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -97,10 +103,17 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -142,10 +155,17 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -249,12 +269,19 @@ jobs: echo "_APP_DB_PORT=5432" >> $GITHUB_ENV fi + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 env: _APP_BROWSER_HOST: http://invalid-browser/v1 run: | docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -356,10 +383,17 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -434,11 +468,19 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 + env: + _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -499,11 +541,19 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 + env: + _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -562,11 +612,19 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 + env: + _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -636,11 +694,19 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 + env: + _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." From 4e1b710503ba1ee880842a394645b1ea57aba8ee Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 19:59:23 +0000 Subject: [PATCH 22/51] Fix Docker Hub login credentials to match repo config Use `vars.DOCKERHUB_USERNAME` and `secrets.DOCKERHUB_TOKEN` to match the existing publish and release workflows. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8e937b261a..55254ec35a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -56,8 +56,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -106,8 +106,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -158,8 +158,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -272,8 +272,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -386,8 +386,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -471,8 +471,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -544,8 +544,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -615,8 +615,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -697,8 +697,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 From 0b416e44753a0a7b00c869ae43e69b0cc8145876 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:01:25 +0000 Subject: [PATCH 23/51] Skip locally-built images during docker compose pull Add --ignore-buildable flag so docker compose pull skips images with build directives (appwrite-dev) instead of trying to pull them from Docker Hub. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 55254ec35a..982e19f832 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -113,7 +113,7 @@ jobs: timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -165,7 +165,7 @@ jobs: timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -281,7 +281,7 @@ jobs: _APP_BROWSER_HOST: http://invalid-browser/v1 run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -393,7 +393,7 @@ jobs: timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -480,7 +480,7 @@ jobs: _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -553,7 +553,7 @@ jobs: _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -624,7 +624,7 @@ jobs: _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -706,7 +706,7 @@ jobs: _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." From d22642590f76a3903ccca6a45be98305ea3a5f3b Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:07:13 +0000 Subject: [PATCH 24/51] Add build directive to all appwrite-dev services for --ignore-buildable Add x-build YAML anchor and apply it to all services using the appwrite-dev image so docker compose pull --ignore-buildable correctly skips them instead of trying to pull from Docker Hub. Co-Authored-By: Claude Opus 4.6 --- docker-compose.yml | 60 ++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4a38757737..c744a9c5a5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,15 @@ x-logging: &x-logging max-file: "5" max-size: "10m" +x-build: &x-build + build: + context: . + target: development + args: + DEBUG: false + TESTING: true + VERSION: dev + services: traefik: image: traefik:3.6 @@ -50,15 +59,8 @@ services: appwrite: container_name: appwrite - <<: *x-logging + <<: [*x-logging, *x-build] image: appwrite-dev - build: - context: . - target: development - args: - DEBUG: false - TESTING: true - VERSION: dev ports: - 9501:80 networks: @@ -260,7 +262,7 @@ services: appwrite-realtime: entrypoint: realtime - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-realtime image: appwrite-dev restart: unless-stopped @@ -312,7 +314,7 @@ services: appwrite-worker-audits: entrypoint: worker-audits - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-audits image: appwrite-dev networks: @@ -343,7 +345,7 @@ services: appwrite-worker-webhooks: entrypoint: worker-webhooks - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-webhooks image: appwrite-dev networks: @@ -378,7 +380,7 @@ services: appwrite-worker-deletes: entrypoint: worker-deletes - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-deletes image: appwrite-dev networks: @@ -443,7 +445,7 @@ services: appwrite-worker-databases: entrypoint: worker-databases - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-databases image: appwrite-dev networks: @@ -476,7 +478,7 @@ services: appwrite-worker-builds: entrypoint: worker-builds - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-builds image: appwrite-dev networks: @@ -551,7 +553,7 @@ services: appwrite-worker-screenshots: entrypoint: worker-screenshots - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-screenshots image: appwrite-dev networks: @@ -614,7 +616,7 @@ services: appwrite-worker-certificates: entrypoint: worker-certificates - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-certificates image: appwrite-dev networks: @@ -656,7 +658,7 @@ services: appwrite-worker-executions: entrypoint: worker-executions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-executions image: appwrite-dev networks: @@ -686,7 +688,7 @@ services: appwrite-worker-functions: entrypoint: worker-functions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-functions image: appwrite-dev networks: @@ -730,7 +732,7 @@ services: appwrite-worker-mails: entrypoint: worker-mails - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-mails image: appwrite-dev networks: @@ -772,7 +774,7 @@ services: appwrite-worker-messaging: entrypoint: worker-messaging - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-messaging restart: unless-stopped image: appwrite-dev @@ -829,7 +831,7 @@ services: appwrite-worker-migrations: entrypoint: worker-migrations - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-migrations restart: unless-stopped image: appwrite-dev @@ -874,7 +876,7 @@ services: appwrite-task-maintenance: entrypoint: maintenance - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-maintenance image: appwrite-dev networks: @@ -920,7 +922,7 @@ services: appwrite-task-interval: entrypoint: interval - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-interval image: appwrite-dev networks: @@ -961,7 +963,7 @@ services: appwrite-task-stats-resources: container_name: appwrite-task-stats-resources entrypoint: stats-resources - <<: *x-logging + <<: [*x-logging, *x-build] image: appwrite-dev networks: - appwrite @@ -993,7 +995,7 @@ services: appwrite-worker-stats-resources: entrypoint: worker-stats-resources - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-stats-resources image: appwrite-dev networks: @@ -1026,7 +1028,7 @@ services: appwrite-worker-stats-usage: entrypoint: worker-stats-usage - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-stats-usage image: appwrite-dev networks: @@ -1059,7 +1061,7 @@ services: appwrite-task-scheduler-functions: entrypoint: schedule-functions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-scheduler-functions image: appwrite-dev networks: @@ -1089,7 +1091,7 @@ services: appwrite-task-scheduler-executions: entrypoint: schedule-executions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-scheduler-executions image: appwrite-dev networks: @@ -1118,7 +1120,7 @@ services: appwrite-task-scheduler-messages: entrypoint: schedule-messages - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-scheduler-messages image: appwrite-dev networks: From 6aac7ea6ad9f25ed2e1fc15c1ace0bc682b7b169 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:17:03 +0000 Subject: [PATCH 25/51] Update GitHub Actions to latest versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - actions/cache: v4 → v5 - docker/setup-buildx-action: v3 → v4 Fixes Node.js 20 deprecation warnings. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 982e19f832..3ccf652ccb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -60,7 +60,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Build Appwrite uses: docker/build-push-action@v6 @@ -79,7 +79,7 @@ jobs: VERSION=dev - name: Cache Docker Image - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -97,7 +97,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -149,7 +149,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -243,7 +243,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -377,7 +377,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -462,7 +462,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -535,7 +535,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -606,7 +606,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -688,7 +688,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar From 3613a645d049c5c2705c7412837a91b3f24fd938 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:19:04 +0000 Subject: [PATCH 26/51] Remove php-retry action and run test commands directly Replace itznotabug/php-retry with native run steps and timeout-minutes. Also remove pull-requests: write permission that was only needed by php-retry to post PR comments. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 108 ++++++++---------------------------- 1 file changed, 22 insertions(+), 86 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3ccf652ccb..d252c72a42 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -90,7 +90,6 @@ jobs: needs: setup permissions: contents: read - pull-requests: write steps: - name: checkout @@ -124,18 +123,11 @@ jobs: run: docker compose exec -T appwrite vars - name: Run Unit Tests - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/unit - command: >- - docker compose exec - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/unit + timeout-minutes: 15 + run: >- + docker compose exec + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/unit e2e_general_test: name: E2E General Test @@ -143,7 +135,6 @@ jobs: needs: setup permissions: contents: read - pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -181,18 +172,11 @@ jobs: done - name: Run General Tests - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/General - command: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/General --debug + timeout-minutes: 15 + run: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/General --debug - name: Failure Logs if: failure() @@ -206,7 +190,6 @@ jobs: needs: setup permissions: contents: read - pull-requests: write strategy: fail-fast: false matrix: @@ -297,15 +280,8 @@ jobs: done - name: Run ${{ matrix.service }} tests with Project table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 20 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/${{ matrix.service }} - command: | + timeout-minutes: 20 + run: | echo "Using project tables" SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" @@ -339,7 +315,6 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read - pull-requests: write strategy: fail-fast: false matrix: @@ -409,15 +384,8 @@ jobs: done - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 20 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/${{ matrix.service }} - command: | + timeout-minutes: 20 + run: | if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then echo "Using shared tables V1" export _APP_DATABASE_SHARED_TABLES=database_db_main @@ -456,7 +424,6 @@ jobs: needs: setup permissions: contents: read - pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -488,15 +455,8 @@ jobs: done - name: Run Projects tests in dedicated table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Projects - command: | + timeout-minutes: 15 + run: | echo "Using project tables" export _APP_DATABASE_SHARED_TABLES= export _APP_DATABASE_SHARED_TABLES_V1= @@ -522,7 +482,6 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read - pull-requests: write strategy: fail-fast: false matrix: @@ -561,15 +520,8 @@ jobs: done - name: Run Projects tests in ${{ matrix.tables-mode }} table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Projects - command: | + timeout-minutes: 15 + run: | if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then echo "Using shared tables V1" export _APP_DATABASE_SHARED_TABLES=database_db_main @@ -600,7 +552,6 @@ jobs: needs: setup permissions: contents: read - pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -640,15 +591,8 @@ jobs: done - name: Run Site tests with browser connected in dedicated table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Sites - command: | + timeout-minutes: 15 + run: | echo "Keeping original value of _APP_BROWSER_HOST" echo "Using project tables" export _APP_DATABASE_SHARED_TABLES= @@ -675,7 +619,6 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read - pull-requests: write strategy: fail-fast: false matrix: @@ -722,15 +665,8 @@ jobs: done - name: Run Site tests with browser connected in ${{ matrix.tables-mode }} table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Sites - command: | + timeout-minutes: 15 + run: | echo "Keeping original value of _APP_BROWSER_HOST" if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then echo "Using shared tables V1" From dc9a1c03d18bc6f3ef5f1452f9ab89ef00e9d0cb Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:20:23 +0000 Subject: [PATCH 27/51] Replace shell exports with GitHub Actions env fields Use step-level env: fields and GHA expressions for conditional values instead of shell export statements and if/elif blocks. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 103 ++++++++++++++---------------------- 1 file changed, 39 insertions(+), 64 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d252c72a42..e78a23575c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -385,17 +385,10 @@ jobs: - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode timeout-minutes: 20 + env: + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | - if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then - echo "Using shared tables V1" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1=database_db_main - elif [ "${{ matrix.tables-mode }}" == "Shared V2" ]; then - echo "Using shared tables V2" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1= - fi - SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" # Services that rely on sequential test method execution (shared static state) @@ -456,16 +449,15 @@ jobs: - name: Run Projects tests in dedicated table mode timeout-minutes: 15 - run: | - echo "Using project tables" - export _APP_DATABASE_SHARED_TABLES= - export _APP_DATABASE_SHARED_TABLES_V1= - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled + env: + _APP_DATABASE_SHARED_TABLES: "" + _APP_DATABASE_SHARED_TABLES_V1: "" + run: >- + docker compose exec -T + -e _APP_DATABASE_SHARED_TABLES + -e _APP_DATABASE_SHARED_TABLES_V1 + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled - name: Failure Logs if: failure() @@ -521,22 +513,15 @@ jobs: - name: Run Projects tests in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 - run: | - if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then - echo "Using shared tables V1" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1=database_db_main - elif [ "${{ matrix.tables-mode }}" == "Shared V2" ]; then - echo "Using shared tables V2" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1= - fi - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled + env: + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} + run: >- + docker compose exec -T + -e _APP_DATABASE_SHARED_TABLES + -e _APP_DATABASE_SHARED_TABLES_V1 + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled - name: Failure Logs if: failure() @@ -592,17 +577,15 @@ jobs: - name: Run Site tests with browser connected in dedicated table mode timeout-minutes: 15 - run: | - echo "Keeping original value of _APP_BROWSER_HOST" - echo "Using project tables" - export _APP_DATABASE_SHARED_TABLES= - export _APP_DATABASE_SHARED_TABLES_V1= - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots + env: + _APP_DATABASE_SHARED_TABLES: "" + _APP_DATABASE_SHARED_TABLES_V1: "" + run: >- + docker compose exec -T + -e _APP_DATABASE_SHARED_TABLES + -e _APP_DATABASE_SHARED_TABLES_V1 + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots - name: Failure Logs if: failure() @@ -666,23 +649,15 @@ jobs: - name: Run Site tests with browser connected in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 - run: | - echo "Keeping original value of _APP_BROWSER_HOST" - if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then - echo "Using shared tables V1" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1=database_db_main - elif [ "${{ matrix.tables-mode }}" == "Shared V2" ]; then - echo "Using shared tables V2" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1= - fi - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots + env: + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} + run: >- + docker compose exec -T + -e _APP_DATABASE_SHARED_TABLES + -e _APP_DATABASE_SHARED_TABLES_V1 + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots - name: Failure Logs if: failure() From d30df31b826f24962f27089feefc99fbdb503818 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:25:47 +0000 Subject: [PATCH 28/51] Move compose-time env vars to docker compose up step _APP_DATABASE_SHARED_TABLES and _APP_DATABASE_SHARED_TABLES_V1 are read by the server at boot time via System::getEnv(), not by the test runner. Passing them via docker compose exec -e had no effect on the already-running Swoole server. Move them to the Load and Start Appwrite step so they're set at docker compose up time. Keep _APP_E2E_RESPONSE_FORMAT on exec since it's read by the test runner process (tests/e2e/Scopes/Scope.php). Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 46 +++++++++++-------------------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e78a23575c..201a47c8ba 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -262,6 +262,8 @@ jobs: timeout-minutes: 3 env: _APP_BROWSER_HOST: http://invalid-browser/v1 + _APP_DATABASE_SHARED_TABLES: "" + _APP_DATABASE_SHARED_TABLES_V1: "" run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -282,7 +284,6 @@ jobs: - name: Run ${{ matrix.service }} tests with Project table mode timeout-minutes: 20 run: | - echo "Using project tables" SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" # Services that rely on sequential test method execution (shared static state) @@ -291,14 +292,7 @@ jobs: Databases|Functions|Realtime) FUNCTIONAL_FLAG="" ;; esac - echo "Running with paratest (parallel) for: ${{ matrix.service }} ${FUNCTIONAL_FLAG:+(functional)}" docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES="" \ - -e _APP_DATABASE_SHARED_TABLES_V1="" \ - -e _APP_DB_ADAPTER="${{ env._APP_DB_ADAPTER }}" \ - -e _APP_DB_HOST="${{ env._APP_DB_HOST }}" \ - -e _APP_DB_PORT="${{ env._APP_DB_PORT }}" \ - -e _APP_DB_SCHEMA=appwrite \ -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ appwrite vendor/bin/paratest --processes $(nproc) $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --exclude-group ciIgnore --log-junit tests/e2e/Services/${{ matrix.service }}/junit.xml @@ -366,6 +360,9 @@ jobs: - name: Load and Start Appwrite timeout-minutes: 3 + env: + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -385,9 +382,6 @@ jobs: - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode timeout-minutes: 20 - env: - _APP_DATABASE_SHARED_TABLES: database_db_main - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" @@ -398,8 +392,6 @@ jobs: esac docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ appwrite vendor/bin/paratest --processes $(nproc) $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --exclude-group ciIgnore --log-junit tests/e2e/Services/${{ matrix.service }}/junit.xml @@ -438,6 +430,8 @@ jobs: timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled + _APP_DATABASE_SHARED_TABLES: "" + _APP_DATABASE_SHARED_TABLES_V1: "" run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -449,13 +443,8 @@ jobs: - name: Run Projects tests in dedicated table mode timeout-minutes: 15 - env: - _APP_DATABASE_SHARED_TABLES: "" - _APP_DATABASE_SHARED_TABLES_V1: "" run: >- docker compose exec -T - -e _APP_DATABASE_SHARED_TABLES - -e _APP_DATABASE_SHARED_TABLES_V1 -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled @@ -502,6 +491,8 @@ jobs: timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -513,13 +504,8 @@ jobs: - name: Run Projects tests in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 - env: - _APP_DATABASE_SHARED_TABLES: database_db_main - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: >- docker compose exec -T - -e _APP_DATABASE_SHARED_TABLES - -e _APP_DATABASE_SHARED_TABLES_V1 -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled @@ -558,6 +544,8 @@ jobs: timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled + _APP_DATABASE_SHARED_TABLES: "" + _APP_DATABASE_SHARED_TABLES_V1: "" run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -577,13 +565,8 @@ jobs: - name: Run Site tests with browser connected in dedicated table mode timeout-minutes: 15 - env: - _APP_DATABASE_SHARED_TABLES: "" - _APP_DATABASE_SHARED_TABLES_V1: "" run: >- docker compose exec -T - -e _APP_DATABASE_SHARED_TABLES - -e _APP_DATABASE_SHARED_TABLES_V1 -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots @@ -630,6 +613,8 @@ jobs: timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -649,13 +634,8 @@ jobs: - name: Run Site tests with browser connected in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 - env: - _APP_DATABASE_SHARED_TABLES: database_db_main - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: >- docker compose exec -T - -e _APP_DATABASE_SHARED_TABLES - -e _APP_DATABASE_SHARED_TABLES_V1 -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots From 534dc55f17780a49b7d800316f7bbcb67ad5a9e7 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:35:42 +0000 Subject: [PATCH 29/51] Remove unnecessary abuse enabled from screenshot tests The screenshot tests have no abuse-related code. Abuse was only enabled on these jobs as a side effect of the original sed command applying to all jobs below e2e_service_test. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 201a47c8ba..721c25cadf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -543,7 +543,6 @@ jobs: - name: Load and Start Appwrite timeout-minutes: 3 env: - _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: "" _APP_DATABASE_SHARED_TABLES_V1: "" run: | @@ -612,7 +611,6 @@ jobs: - name: Load and Start Appwrite timeout-minutes: 3 env: - _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | From ecca0d80363fcd88030989d22c6a6729241f15e0 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:48:39 +0000 Subject: [PATCH 30/51] Increase Load and Start Appwrite timeout from 3 to 10 minutes The 3-minute timeout was too tight with the added docker compose pull step for downloading third-party images. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 721c25cadf..5c20ca094d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -109,7 +109,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -153,7 +153,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -259,7 +259,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_BROWSER_HOST: http://invalid-browser/v1 _APP_DATABASE_SHARED_TABLES: "" @@ -359,7 +359,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} @@ -427,7 +427,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: "" @@ -488,7 +488,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: database_db_main @@ -541,7 +541,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_DATABASE_SHARED_TABLES: "" _APP_DATABASE_SHARED_TABLES_V1: "" @@ -609,7 +609,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} From 0bbcc3f570455e241776460669a5683d74e3676a Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 21:06:44 +0000 Subject: [PATCH 31/51] Move dev tools to docker-compose.override.yml Move adminer, redis-insight, mongo-express, and graphql-explorer to an override file. CI sets COMPOSE_FILE=docker-compose.yml explicitly so these are excluded from test runs, reducing the number of images to pull from 14 to 10. Local docker compose auto-loads both files so dev workflow is unchanged. Co-Authored-By: Claude Opus 4.6 --- docker-compose.override.yml | 144 ++++++++++++++++++++++++++++++++++++ docker-compose.yml | 142 +---------------------------------- 2 files changed, 147 insertions(+), 139 deletions(-) create mode 100644 docker-compose.override.yml diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000000..29182f01d2 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,144 @@ +# Dev tools for local development only. +# This file is automatically loaded by `docker compose` alongside docker-compose.yml. +# CI sets COMPOSE_FILE=docker-compose.yml explicitly, so these services are excluded from test runs. + +services: + appwrite-mongo-express: + profiles: ["mongodb"] + image: mongo-express + container_name: appwrite-mongo-express + networks: + - appwrite + ports: + - "8082:8081" + environment: + ME_CONFIG_MONGODB_URL: "mongodb://root:${_APP_DB_ROOT_PASS}@appwrite-mongodb:27017/?replicaSet=rs0&directConnection=true" + ME_CONFIG_BASICAUTH_USERNAME: ${_APP_DB_USER} + ME_CONFIG_BASICAUTH_PASSWORD: ${_APP_DB_PASS} + depends_on: + - mongodb + + adminer: + image: adminer + container_name: appwrite-adminer + restart: always + ports: + - 9506:8080 + networks: + - appwrite + - gateway + environment: + - ADMINER_DESIGN=pepa-linha + - ADMINER_DEFAULT_SERVER=mariadb + - ADMINER_DEFAULT_USERNAME=root + - ADMINER_DEFAULT_PASSWORD=rootsecretpassword + - ADMINER_DEFAULT_DB=appwrite + configs: + - source: adminer-index.php + target: /var/www/html/index.php + mode: 0755 + labels: + - "traefik.enable=true" + - "traefik.constraint-label-stack=appwrite" + - "traefik.docker.network=gateway" + - "traefik.http.services.appwrite_adminer.loadbalancer.server.port=8080" + - "traefik.http.routers.appwrite_adminer_http.entrypoints=appwrite_web" + - "traefik.http.routers.appwrite_adminer_http.rule=Host(`mysql.localhost`)" + - "traefik.http.routers.appwrite_adminer_http.service=appwrite_adminer" + - "traefik.http.routers.appwrite_adminer_https.entrypoints=appwrite_websecure" + - "traefik.http.routers.appwrite_adminer_https.rule=Host(`mysql.localhost`)" + - "traefik.http.routers.appwrite_adminer_https.service=appwrite_adminer" + - "traefik.http.routers.appwrite_adminer_https.tls=true" + + redis-insight: + image: redis/redisinsight:latest + restart: unless-stopped + networks: + - appwrite + - gateway + environment: + - RI_PRE_SETUP_DATABASES_PATH=/mnt/connections.json + configs: + - source: redisinsight-connections.json + target: /mnt/connections.json + mode: 0755 + labels: + - "traefik.enable=true" + - "traefik.constraint-label-stack=appwrite" + - "traefik.docker.network=gateway" + - "traefik.http.services.appwrite_redisinsight.loadbalancer.server.port=5540" + - "traefik.http.routers.appwrite_redisinsight_http.entrypoints=appwrite_web" + - "traefik.http.routers.appwrite_redisinsight_http.rule=Host(`redis.localhost`)" + - "traefik.http.routers.appwrite_redisinsight_http.service=appwrite_redisinsight" + - "traefik.http.routers.appwrite_redisinsight_https.entrypoints=appwrite_websecure" + - "traefik.http.routers.appwrite_redisinsight_https.rule=Host(`redis.localhost`)" + - "traefik.http.routers.appwrite_redisinsight_https.service=appwrite_redisinsight" + - "traefik.http.routers.appwrite_redisinsight_https.tls=true" + ports: + - "8081:5540" + + graphql-explorer: + container_name: appwrite-graphql-explorer + image: appwrite/altair:0.3.0 + restart: unless-stopped + networks: + - appwrite + ports: + - "9509:3000" + environment: + - SERVER_URL=http://localhost/v1/graphql + +configs: + redisinsight-connections.json: + content: | + [ + { + "compressor": "NONE", + "id": "104dc90a-21ef-4d5e-8912-b30baabb152f", + "host": "redis", + "port": 6379, + "name": "redis:6379", + "db": 0, + "username": "default", + "password": null, + "connectionType": "STANDALONE", + "nameFromProvider": null, + "provider": "REDIS", + "lastConnection": "2025-10-16T09:22:02.591Z", + "modules": [ + { + "name": "ReJSON", + "version": 20808, + "semanticVersion": "2.8.8" + }, + { + "name": "search", + "version": 21015, + "semanticVersion": "2.10.15" + } + ], + "tls": false, + "tlsServername": null, + "verifyServerCert": null, + "caCert": null, + "clientCert": null, + "ssh": false, + "sshOptions": null, + "forceStandalone": false, + "tags": [] + } + ] + + adminer-index.php: + content: | + $$_ENV['ADMINER_DEFAULT_SERVER'], + 'driver' => 'server', /* seems to autodetect the driver from server settings */ + 'username' => $$_ENV['ADMINER_DEFAULT_USERNAME'], + 'password' => $$_ENV['ADMINER_DEFAULT_PASSWORD'], + 'db' => $$_ENV['ADMINER_DEFAULT_DB'], + ]; + } + include './adminer.php'; diff --git a/docker-compose.yml b/docker-compose.yml index c744a9c5a5..0013fdfa64 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1277,20 +1277,7 @@ services: retries: 10 start_period: 30s - appwrite-mongo-express: - profiles: ["mongodb"] - image: mongo-express - container_name: appwrite-mongo-express - networks: - - appwrite - ports: - - "8082:8081" - environment: - ME_CONFIG_MONGODB_URL: "mongodb://root:${_APP_DB_ROOT_PASS}@appwrite-mongodb:27017/?replicaSet=rs0&directConnection=true" - ME_CONFIG_BASICAUTH_USERNAME: ${_APP_DB_USER} - ME_CONFIG_BASICAUTH_PASSWORD: ${_APP_DB_PASS} - depends_on: - - mongodb + postgresql: profiles: ["postgresql"] @@ -1398,78 +1385,8 @@ services: networks: - appwrite - adminer: - image: adminer - container_name: appwrite-adminer - <<: *x-logging - restart: always - ports: - - 9506:8080 - networks: - - appwrite - - gateway - environment: - - ADMINER_DESIGN=pepa-linha - - ADMINER_DEFAULT_SERVER=mariadb - - ADMINER_DEFAULT_USERNAME=root - - ADMINER_DEFAULT_PASSWORD=rootsecretpassword - - ADMINER_DEFAULT_DB=appwrite - configs: - - source: adminer-index.php - target: /var/www/html/index.php - mode: 0755 - labels: - - "traefik.enable=true" - - "traefik.constraint-label-stack=appwrite" - - "traefik.docker.network=gateway" - - "traefik.http.services.appwrite_adminer.loadbalancer.server.port=8080" - - "traefik.http.routers.appwrite_adminer_http.entrypoints=appwrite_web" - - "traefik.http.routers.appwrite_adminer_http.rule=Host(`mysql.localhost`)" - - "traefik.http.routers.appwrite_adminer_http.service=appwrite_adminer" - - "traefik.http.routers.appwrite_adminer_https.entrypoints=appwrite_websecure" - - "traefik.http.routers.appwrite_adminer_https.rule=Host(`mysql.localhost`)" - - "traefik.http.routers.appwrite_adminer_https.service=appwrite_adminer" - - "traefik.http.routers.appwrite_adminer_https.tls=true" - - redis-insight: - image: redis/redisinsight:latest - restart: unless-stopped - networks: - - appwrite - - gateway - environment: - - RI_PRE_SETUP_DATABASES_PATH=/mnt/connections.json - configs: - - source: redisinsight-connections.json - target: /mnt/connections.json - mode: 0755 - labels: - - "traefik.enable=true" - - "traefik.constraint-label-stack=appwrite" - - "traefik.docker.network=gateway" - - "traefik.http.services.appwrite_redisinsight.loadbalancer.server.port=5540" - - "traefik.http.routers.appwrite_redisinsight_http.entrypoints=appwrite_web" - - "traefik.http.routers.appwrite_redisinsight_http.rule=Host(`redis.localhost`)" - - "traefik.http.routers.appwrite_redisinsight_http.service=appwrite_redisinsight" - - "traefik.http.routers.appwrite_redisinsight_https.entrypoints=appwrite_websecure" - - "traefik.http.routers.appwrite_redisinsight_https.rule=Host(`redis.localhost`)" - - "traefik.http.routers.appwrite_redisinsight_https.service=appwrite_redisinsight" - - "traefik.http.routers.appwrite_redisinsight_https.tls=true" - ports: - - "8081:5540" - - graphql-explorer: - container_name: appwrite-graphql-explorer - image: appwrite/altair:0.3.0 - restart: unless-stopped - networks: - - appwrite - ports: - - "9509:3000" - environment: - - SERVER_URL=http://localhost/v1/graphql - - # Dev Tools End ------------------------------------------------------------------------------------------ + # Dev tools (adminer, redis-insight, mongo-express, graphql-explorer) + # are defined in docker-compose.override.yml networks: gateway: @@ -1482,60 +1399,7 @@ networks: runtimes: name: runtimes -configs: - redisinsight-connections.json: - content: | - [ - { - "compressor": "NONE", - "id": "104dc90a-21ef-4d5e-8912-b30baabb152f", - "host": "redis", - "port": 6379, - "name": "redis:6379", - "db": 0, - "username": "default", - "password": null, - "connectionType": "STANDALONE", - "nameFromProvider": null, - "provider": "REDIS", - "lastConnection": "2025-10-16T09:22:02.591Z", - "modules": [ - { - "name": "ReJSON", - "version": 20808, - "semanticVersion": "2.8.8" - }, - { - "name": "search", - "version": 21015, - "semanticVersion": "2.10.15" - } - ], - "tls": false, - "tlsServername": null, - "verifyServerCert": null, - "caCert": null, - "clientCert": null, - "ssh": false, - "sshOptions": null, - "forceStandalone": false, - "tags": [] - } - ] - adminer-index.php: - content: | - $$_ENV['ADMINER_DEFAULT_SERVER'], - 'driver' => 'server', /* seems to autodetect the driver from server settings */ - 'username' => $$_ENV['ADMINER_DEFAULT_USERNAME'], - 'password' => $$_ENV['ADMINER_DEFAULT_PASSWORD'], - 'db' => $$_ENV['ADMINER_DEFAULT_DB'], - ]; - } - include './adminer.php'; volumes: appwrite-mariadb: From 33ce469ab0206b161b60983fabca86d666fdc839 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 21:44:15 +0000 Subject: [PATCH 32/51] Add Docker healthcheck and use --wait instead of polling loop Replace the manual shell polling loop (until doctor > /dev/null) with a proper Docker healthcheck on the appwrite service and `docker compose up --wait`, which blocks until healthchecks pass. Also reverts the timeout back to 3 minutes now that image pulls are cached. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 64 ++++++++++--------------------------- docker-compose.yml | 5 +++ 2 files changed, 21 insertions(+), 48 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5c20ca094d..cebcd2a62c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -109,15 +109,11 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Environment Variables run: docker compose exec -T appwrite vars @@ -153,15 +149,11 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Wait for Open Runtimes timeout-minutes: 3 @@ -259,7 +251,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_BROWSER_HOST: http://invalid-browser/v1 _APP_DATABASE_SHARED_TABLES: "" @@ -267,11 +259,7 @@ jobs: run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Wait for Open Runtimes timeout-minutes: 3 @@ -359,18 +347,14 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Wait for Open Runtimes timeout-minutes: 3 @@ -427,7 +411,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: "" @@ -435,11 +419,7 @@ jobs: run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Run Projects tests in dedicated table mode timeout-minutes: 15 @@ -488,7 +468,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: database_db_main @@ -496,11 +476,7 @@ jobs: run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Run Projects tests in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 @@ -541,18 +517,14 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_DATABASE_SHARED_TABLES: "" _APP_DATABASE_SHARED_TABLES_V1: "" run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Wait for Open Runtimes timeout-minutes: 3 @@ -609,18 +581,14 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Wait for Open Runtimes timeout-minutes: 3 diff --git a/docker-compose.yml b/docker-compose.yml index 0013fdfa64..583b558e60 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,6 +61,11 @@ services: container_name: appwrite <<: [*x-logging, *x-build] image: appwrite-dev + healthcheck: + test: ["CMD", "doctor"] + interval: 5s + timeout: 5s + retries: 12 ports: - 9501:80 networks: From 38798c899396ee48b3151185ecaa632495c3f939 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 21:58:30 +0000 Subject: [PATCH 33/51] Add healthchecks for MariaDB, PostgreSQL, and Redis Add proper healthchecks to infrastructure services and use condition: service_healthy for redis in appwrite's depends_on so it waits for Redis to be ready before starting. Co-Authored-By: Claude Opus 4.6 --- docker-compose.yml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 583b558e60..c0b0560a7f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,10 +108,10 @@ services: - ./dev:/usr/src/code/dev depends_on: - - ${_APP_DB_HOST:-mongodb} - - redis - - coredns - # - clamav + redis: + condition: service_healthy + coredns: + condition: service_started entrypoint: - php - -e @@ -1244,6 +1244,11 @@ services: - MYSQL_PASSWORD=${_APP_DB_PASS} - MARIADB_AUTO_UPGRADE=1 command: "mysqld --innodb-flush-method=fsync" + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 5s + timeout: 5s + retries: 12 mongodb: profiles: ["mongodb"] @@ -1303,6 +1308,11 @@ services: - POSTGRES_USER=${_APP_DB_USER} - POSTGRES_PASSWORD=${_APP_DB_PASS} command: "postgres" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${_APP_DB_USER}"] + interval: 5s + timeout: 5s + retries: 12 redis: image: redis:7.4.7-alpine @@ -1319,6 +1329,11 @@ services: - appwrite volumes: - appwrite-redis:/data:rw + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 12 coredns: # DNS server for testing purposes (Proxy APIs) image: coredns/coredns:1.12.4 From fd28ad8a6681c9cf8d678cc80c4c29bfc9e5b00d Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 21:59:33 +0000 Subject: [PATCH 34/51] Remove --debug flag from test commands for quieter CI output Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cebcd2a62c..39c9aee9ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -168,8 +168,7 @@ jobs: run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/General --debug - + appwrite test /usr/src/code/tests/e2e/General - name: Failure Logs if: failure() run: | From 16929bc42074ef739151d4021aa9d92ee91082f7 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 22:00:00 +0000 Subject: [PATCH 35/51] Remove remaining --debug flags from test commands Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 39c9aee9ca..879b72327d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -425,7 +425,7 @@ jobs: run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled + appwrite test /usr/src/code/tests/e2e/Services/Projects --group=abuseEnabled - name: Failure Logs if: failure() @@ -482,7 +482,7 @@ jobs: run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled + appwrite test /usr/src/code/tests/e2e/Services/Projects --group=abuseEnabled - name: Failure Logs if: failure() @@ -538,7 +538,7 @@ jobs: run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots + appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots - name: Failure Logs if: failure() @@ -602,7 +602,7 @@ jobs: run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots + appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots - name: Failure Logs if: failure() From 4cbe50193a163f3e9e44e97c520a46a200515e5f Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 22:24:52 +0000 Subject: [PATCH 36/51] Increase Load and Start Appwrite timeout to 5 minutes The docker compose pull step alone can take over 2.5 minutes on CI (e.g. openruntimes-executor, traefik), leaving no time for docker compose up --wait within 3 minutes. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 879b72327d..73c136bb93 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -109,7 +109,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -149,7 +149,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable From 7dfe44cb363a8a4350e4339fbc504f794143959f Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 22:44:42 +0000 Subject: [PATCH 37/51] Run abuse-enabled tests across entire test suite, not just Projects The abuseEnabled jobs previously only ran tests in Services/Projects, missing the Account abuse test and any future abuseEnabled tests in other services. Also rename jobs to "Abuse" for consistency. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 73c136bb93..c13370f3f0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -387,7 +387,7 @@ jobs: docker compose logs openruntimes-executor e2e_abuse_enabled: - name: E2E Service Test (Abuse enabled) + name: E2E Service Test (Abuse) runs-on: ubuntu-latest needs: setup permissions: @@ -420,12 +420,12 @@ jobs: docker compose pull --quiet --ignore-buildable docker compose up -d --quiet-pull --wait - - name: Run Projects tests in dedicated table mode + - name: Run abuse-enabled tests in dedicated table mode timeout-minutes: 15 run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Projects --group=abuseEnabled + appwrite test /usr/src/code/tests/e2e --group=abuseEnabled - name: Failure Logs if: failure() @@ -436,7 +436,7 @@ jobs: docker compose logs openruntimes-executor e2e_abuse_enabled_shared_mode: - name: E2E Shared Mode Service Test (Abuse enabled) + name: E2E Shared Mode Service Test (Abuse) runs-on: ubuntu-latest needs: [ setup, check_database_changes ] if: needs.check_database_changes.outputs.database_changed == 'true' @@ -477,12 +477,12 @@ jobs: docker compose pull --quiet --ignore-buildable docker compose up -d --quiet-pull --wait - - name: Run Projects tests in ${{ matrix.tables-mode }} table mode + - name: Run abuse-enabled tests in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Projects --group=abuseEnabled + appwrite test /usr/src/code/tests/e2e --group=abuseEnabled - name: Failure Logs if: failure() From 89419344c2b3f045d5ce8c048eb1dd2a7782c4de Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 23:20:51 +0000 Subject: [PATCH 38/51] Restore MariaDB defaults, fix remaining timeouts, filter listener span logs - Change .env defaults back from MongoDB to MariaDB - Bump all remaining "Load and Start Appwrite" timeouts from 3 to 5 minutes - Filter listener.* span logs to only export on error Co-Authored-By: Claude Opus 4.6 --- .env | 8 ++++---- .github/workflows/tests.yml | 12 ++++++------ app/init/span.php | 7 ++++++- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.env b/.env index 0df9cb42f4..3c8bef5b68 100644 --- a/.env +++ b/.env @@ -39,10 +39,10 @@ _APP_REDIS_HOST=redis _APP_REDIS_PORT=6379 _APP_REDIS_PASS= _APP_REDIS_USER= -COMPOSE_PROFILES=mongodb -_APP_DB_ADAPTER=mongodb -_APP_DB_HOST=mongodb -_APP_DB_PORT=27017 +COMPOSE_PROFILES=mariadb +_APP_DB_ADAPTER=mariadb +_APP_DB_HOST=mariadb +_APP_DB_PORT=3306 _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c13370f3f0..9f2556f3e5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -250,7 +250,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_BROWSER_HOST: http://invalid-browser/v1 _APP_DATABASE_SHARED_TABLES: "" @@ -346,7 +346,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} @@ -410,7 +410,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: "" @@ -467,7 +467,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: database_db_main @@ -516,7 +516,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_DATABASE_SHARED_TABLES: "" _APP_DATABASE_SHARED_TABLES_V1: "" @@ -580,7 +580,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} diff --git a/app/init/span.php b/app/init/span.php index 76f37f5300..8afa01b2df 100644 --- a/app/init/span.php +++ b/app/init/span.php @@ -5,4 +5,9 @@ use Utopia\Span\Span; use Utopia\Span\Storage; Span::setStorage(new Storage\Coroutine()); -Span::addExporter(new Exporter\Pretty()); +Span::addExporter(new Exporter\Pretty(), function (Span $span): bool { + if (\str_starts_with($span->getAction(), 'listener.')) { + return $span->getError() !== null; + } + return true; +}); From 8cb36835cc4eaa92345bae1b2413e7aa89b11d41 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 23:41:26 +0000 Subject: [PATCH 39/51] Restore php-retry action for flaky test resilience Re-add itznotabug/php-retry@v3 wrapping all test steps with max_attempts: 2 and retry_wait_seconds: 300. Also restore pull-requests: write permission needed by the action. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 135 ++++++++++++++++++++++++++---------- 1 file changed, 100 insertions(+), 35 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9f2556f3e5..20d3c71a9a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -90,6 +90,7 @@ jobs: needs: setup permissions: contents: read + pull-requests: write steps: - name: checkout @@ -119,11 +120,18 @@ jobs: run: docker compose exec -T appwrite vars - name: Run Unit Tests - timeout-minutes: 15 - run: >- - docker compose exec - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/unit + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/unit + command: >- + docker compose exec + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/unit e2e_general_test: name: E2E General Test @@ -131,6 +139,7 @@ jobs: needs: setup permissions: contents: read + pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -164,11 +173,19 @@ jobs: done - name: Run General Tests - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/General + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/General + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/General + - name: Failure Logs if: failure() run: | @@ -181,6 +198,7 @@ jobs: needs: setup permissions: contents: read + pull-requests: write strategy: fail-fast: false matrix: @@ -228,7 +246,7 @@ jobs: run: | DB_ADAPTER_LOWER=$(echo "${{ matrix.db_adapter }}" | tr 'A-Z' 'a-z') echo "COMPOSE_PROFILES=${DB_ADAPTER_LOWER}" >> $GITHUB_ENV - + if [ "${{ matrix.db_adapter }}" = "MARIADB" ]; then echo "_APP_DB_ADAPTER=mariadb" >> $GITHUB_ENV echo "_APP_DB_HOST=mariadb" >> $GITHUB_ENV @@ -269,8 +287,15 @@ jobs: done - name: Run ${{ matrix.service }} tests with Project table mode - timeout-minutes: 20 - run: | + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 20 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/Services/${{ matrix.service }} + command: | SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" # Services that rely on sequential test method execution (shared static state) @@ -296,6 +321,7 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read + pull-requests: write strategy: fail-fast: false matrix: @@ -364,8 +390,15 @@ jobs: done - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode - timeout-minutes: 20 - run: | + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 20 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/Services/${{ matrix.service }} + command: | SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" # Services that rely on sequential test method execution (shared static state) @@ -392,6 +425,7 @@ jobs: needs: setup permissions: contents: read + pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -421,11 +455,18 @@ jobs: docker compose up -d --quiet-pull --wait - name: Run abuse-enabled tests in dedicated table mode - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e --group=abuseEnabled + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e --group=abuseEnabled - name: Failure Logs if: failure() @@ -442,6 +483,7 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read + pull-requests: write strategy: fail-fast: false matrix: @@ -478,11 +520,18 @@ jobs: docker compose up -d --quiet-pull --wait - name: Run abuse-enabled tests in ${{ matrix.tables-mode }} table mode - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e --group=abuseEnabled + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e --group=abuseEnabled - name: Failure Logs if: failure() @@ -498,6 +547,7 @@ jobs: needs: setup permissions: contents: read + pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -534,11 +584,18 @@ jobs: done - name: Run Site tests with browser connected in dedicated table mode - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/Services/Sites + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots - name: Failure Logs if: failure() @@ -555,6 +612,7 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read + pull-requests: write strategy: fail-fast: false matrix: @@ -598,11 +656,18 @@ jobs: done - name: Run Site tests with browser connected in ${{ matrix.tables-mode }} table mode - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/Services/Sites + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots - name: Failure Logs if: failure() From 9f4ba3a4a285042b8e79327be96d81a0ac51db09 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 23:47:03 +0000 Subject: [PATCH 40/51] Reduce php-retry wait time from 300s to 60s Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 20d3c71a9a..1b112eebbd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -123,7 +123,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -176,7 +176,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -290,7 +290,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 20 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -393,7 +393,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 20 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -458,7 +458,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -523,7 +523,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -587,7 +587,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -659,7 +659,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} From f09b9a994eaefed1ec506db792facc823d707858 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 23:59:31 +0000 Subject: [PATCH 41/51] Revert .env defaults back to MongoDB Co-Authored-By: Claude Opus 4.6 --- .env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 3c8bef5b68..0df9cb42f4 100644 --- a/.env +++ b/.env @@ -39,10 +39,10 @@ _APP_REDIS_HOST=redis _APP_REDIS_PORT=6379 _APP_REDIS_PASS= _APP_REDIS_USER= -COMPOSE_PROFILES=mariadb -_APP_DB_ADAPTER=mariadb -_APP_DB_HOST=mariadb -_APP_DB_PORT=3306 +COMPOSE_PROFILES=mongodb +_APP_DB_ADAPTER=mongodb +_APP_DB_HOST=mongodb +_APP_DB_PORT=27017 _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password From 20c65bac37959979a4d3e76703fdd69ef4f99742 Mon Sep 17 00:00:00 2001 From: eldadfux Date: Thu, 12 Mar 2026 09:02:16 +0100 Subject: [PATCH 42/51] Allow null branch --- src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php index 47195a3eb5..ba99cefb42 100644 --- a/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php +++ b/src/Appwrite/Platform/Modules/Proxy/Http/Rules/Site/Create.php @@ -75,7 +75,7 @@ class Create extends Action ->callback($this->action(...)); } - public function action(string $domain, string $siteId, string $branch, Response $response, Document $project, Certificate $queueForCertificates, Event $queueForEvents, Database $dbForPlatform, Database $dbForProject, array $platform, Log $log) + public function action(string $domain, string $siteId, ?string $branch, Response $response, Document $project, Certificate $queueForCertificates, Event $queueForEvents, Database $dbForPlatform, Database $dbForProject, array $platform, Log $log) { $this->validateDomainRestrictions($domain, $platform); From e5f0c2df12f9296cb16e6466c35a9f1412035484 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 00:47:54 +0000 Subject: [PATCH 43/51] Consolidate CI test matrix with dynamic database and mode dimensions Merge 6 E2E jobs into 3 by combining dedicated/shared mode variants into a single matrix dimension. Database adapters and table modes expand dynamically based on whether utopia-php/database changed. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 328 +++++------------------------------- 1 file changed, 42 insertions(+), 286 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1b112eebbd..10cf958652 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,19 +28,21 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v6 - - - name: Fetch base branch - run: git fetch origin ${{ github.event.pull_request.base.ref }} + with: + fetch-depth: 0 - name: Check for utopia-php/database changes id: check run: | - if git diff origin/${{ github.event.pull_request.base.ref }} HEAD -- composer.lock | grep -q '"name": "utopia-php/database"'; then - echo "Database version changed, going to run all mode tests." - echo "database_changed=true" >> "$GITHUB_ENV" + BASE_REF="${{ github.event.pull_request.base.ref }}" + if [ -z "$BASE_REF" ]; then + echo "database_changed=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if git diff "origin/${BASE_REF}" HEAD -- composer.lock | grep -q '"name": "utopia-php/database"'; then echo "database_changed=true" >> "$GITHUB_OUTPUT" else - echo "database_changed=false" >> "$GITHUB_ENV" echo "database_changed=false" >> "$GITHUB_OUTPUT" fi @@ -193,20 +195,17 @@ jobs: docker compose logs e2e_service_test: - name: E2E Service Test + name: E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service }} runs-on: ubuntu-latest - needs: setup + needs: [setup, check_database_changes] permissions: contents: read pull-requests: write strategy: fail-fast: false matrix: - db_adapter: [ - MARIADB, - POSTGRESQL, - MONGODB - ] + database: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["MariaDB","PostgreSQL","MongoDB"]' || '["MongoDB"]') }} + mode: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["dedicated","shared_v1","shared_v2"]' || '["dedicated"]') }} service: [ Account, Avatars, @@ -241,23 +240,18 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true - - name: Set DB Adapter environment - id: set-db-env + - name: Set database environment run: | - DB_ADAPTER_LOWER=$(echo "${{ matrix.db_adapter }}" | tr 'A-Z' 'a-z') - echo "COMPOSE_PROFILES=${DB_ADAPTER_LOWER}" >> $GITHUB_ENV + DB_LOWER=$(echo "${{ matrix.database }}" | tr 'A-Z' 'a-z') + echo "COMPOSE_PROFILES=${DB_LOWER}" >> $GITHUB_ENV + echo "_APP_DB_ADAPTER=${DB_LOWER}" >> $GITHUB_ENV + echo "_APP_DB_HOST=${DB_LOWER}" >> $GITHUB_ENV - if [ "${{ matrix.db_adapter }}" = "MARIADB" ]; then - echo "_APP_DB_ADAPTER=mariadb" >> $GITHUB_ENV - echo "_APP_DB_HOST=mariadb" >> $GITHUB_ENV + if [ "${{ matrix.database }}" = "MariaDB" ]; then echo "_APP_DB_PORT=3306" >> $GITHUB_ENV - elif [ "${{ matrix.db_adapter }}" = "MONGODB" ]; then - echo "_APP_DB_ADAPTER=mongodb" >> $GITHUB_ENV - echo "_APP_DB_HOST=mongodb" >> $GITHUB_ENV + elif [ "${{ matrix.database }}" = "MongoDB" ]; then echo "_APP_DB_PORT=27017" >> $GITHUB_ENV - elif [ "${{ matrix.db_adapter }}" = "POSTGRESQL" ]; then - echo "_APP_DB_ADAPTER=postgresql" >> $GITHUB_ENV - echo "_APP_DB_HOST=postgresql" >> $GITHUB_ENV + elif [ "${{ matrix.database }}" = "PostgreSQL" ]; then echo "_APP_DB_PORT=5432" >> $GITHUB_ENV fi @@ -271,8 +265,8 @@ jobs: timeout-minutes: 5 env: _APP_BROWSER_HOST: http://invalid-browser/v1 - _APP_DATABASE_SHARED_TABLES: "" - _APP_DATABASE_SHARED_TABLES_V1: "" + _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -286,7 +280,7 @@ jobs: sleep 1 done - - name: Run ${{ matrix.service }} tests with Project table mode + - name: Run tests uses: itznotabug/php-retry@v3 with: max_attempts: 2 @@ -314,185 +308,19 @@ jobs: echo "=== Appwrite Logs ===" docker compose logs - e2e_shared_mode_test: - name: E2E Shared Mode Service Test - runs-on: ubuntu-latest - needs: [ setup, check_database_changes ] - if: needs.check_database_changes.outputs.database_changed == 'true' - permissions: - contents: read - pull-requests: write - strategy: - fail-fast: false - matrix: - service: - [ - Account, - Avatars, - Console, - Databases, - Functions, - FunctionsSchedule, - GraphQL, - Health, - Locale, - Projects, - Realtime, - Sites, - Proxy, - Storage, - Teams, - Users, - Webhooks, - VCS, - Messaging, - Migrations, - Tokens - ] - tables-mode: [ - 'Shared V1', - 'Shared V2', - ] - - steps: - - name: checkout - uses: actions/checkout@v6 - - - name: Load Cache - uses: actions/cache@v5 - with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true - - - name: Login to Docker Hub - uses: docker/login-action@v4 - with: - username: ${{ vars.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Load and Start Appwrite - timeout-minutes: 5 - env: - _APP_DATABASE_SHARED_TABLES: database_db_main - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} - run: | - docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet --ignore-buildable - docker compose up -d --quiet-pull --wait - - - name: Wait for Open Runtimes - timeout-minutes: 3 - run: | - while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do - echo "Waiting for Executor to come online" - sleep 1 - done - - - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 60 - timeout_minutes: 20 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/${{ matrix.service }} - command: | - SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" - - # Services that rely on sequential test method execution (shared static state) - FUNCTIONAL_FLAG="--functional" - case "${{ matrix.service }}" in - Databases|Functions|Realtime) FUNCTIONAL_FLAG="" ;; - esac - - docker compose exec -T \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite vendor/bin/paratest --processes $(nproc) $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --exclude-group ciIgnore --log-junit tests/e2e/Services/${{ matrix.service }}/junit.xml - - - name: Failure Logs - if: failure() - run: | - echo "=== Appwrite Worker Builds Logs ===" - docker compose logs appwrite-worker-builds - echo "=== OpenRuntimes Executor Logs ===" - docker compose logs openruntimes-executor - e2e_abuse_enabled: - name: E2E Service Test (Abuse) + name: E2E / Abuse (${{ matrix.mode }}) runs-on: ubuntu-latest - needs: setup - permissions: - contents: read - pull-requests: write - steps: - - name: checkout - uses: actions/checkout@v6 - - - name: Load Cache - uses: actions/cache@v5 - with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true - - - name: Login to Docker Hub - uses: docker/login-action@v4 - with: - username: ${{ vars.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Load and Start Appwrite - timeout-minutes: 5 - env: - _APP_OPTIONS_ABUSE: enabled - _APP_DATABASE_SHARED_TABLES: "" - _APP_DATABASE_SHARED_TABLES_V1: "" - run: | - docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet --ignore-buildable - docker compose up -d --quiet-pull --wait - - - name: Run abuse-enabled tests in dedicated table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 60 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e - command: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e --group=abuseEnabled - - - name: Failure Logs - if: failure() - run: | - echo "=== Appwrite Worker Builds Logs ===" - docker compose logs appwrite-worker-builds - echo "=== OpenRuntimes Executor Logs ===" - docker compose logs openruntimes-executor - - e2e_abuse_enabled_shared_mode: - name: E2E Shared Mode Service Test (Abuse) - runs-on: ubuntu-latest - needs: [ setup, check_database_changes ] - if: needs.check_database_changes.outputs.database_changed == 'true' + needs: [setup, check_database_changes] permissions: contents: read pull-requests: write strategy: fail-fast: false matrix: - tables-mode: [ - 'Shared V1', - 'Shared V2', - ] + mode: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["dedicated","shared_v1","shared_v2"]' || '["dedicated"]') }} steps: - - name: checkout + - name: Checkout repository uses: actions/checkout@v6 - name: Load Cache @@ -512,14 +340,14 @@ jobs: timeout-minutes: 5 env: _APP_OPTIONS_ABUSE: enabled - _APP_DATABASE_SHARED_TABLES: database_db_main - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} + _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable docker compose up -d --quiet-pull --wait - - name: Run abuse-enabled tests in ${{ matrix.tables-mode }} table mode + - name: Run tests uses: itznotabug/php-retry@v3 with: max_attempts: 2 @@ -536,92 +364,22 @@ jobs: - name: Failure Logs if: failure() run: | - echo "=== Appwrite Worker Builds Logs ===" - docker compose logs appwrite-worker-builds - echo "=== OpenRuntimes Executor Logs ===" - docker compose logs openruntimes-executor + echo "=== Appwrite Logs ===" + docker compose logs e2e_screenshots: - name: E2E Service Test (Site Screenshots) + name: E2E / Screenshots (${{ matrix.mode }}) runs-on: ubuntu-latest - needs: setup - permissions: - contents: read - pull-requests: write - steps: - - name: checkout - uses: actions/checkout@v6 - - - name: Load Cache - uses: actions/cache@v5 - with: - key: ${{ env.CACHE_KEY }} - path: /tmp/${{ env.IMAGE }}.tar - fail-on-cache-miss: true - - - name: Login to Docker Hub - uses: docker/login-action@v4 - with: - username: ${{ vars.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Load and Start Appwrite - timeout-minutes: 5 - env: - _APP_DATABASE_SHARED_TABLES: "" - _APP_DATABASE_SHARED_TABLES_V1: "" - run: | - docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet --ignore-buildable - docker compose up -d --quiet-pull --wait - - - name: Wait for Open Runtimes - timeout-minutes: 3 - run: | - while ! docker compose logs openruntimes-executor | grep -q "Executor is ready."; do - echo "Waiting for Executor to come online" - sleep 1 - done - - - name: Run Site tests with browser connected in dedicated table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 60 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Sites - command: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots - - - name: Failure Logs - if: failure() - run: | - echo "=== Appwrite Worker Builds Logs ===" - docker compose logs appwrite-worker-builds - echo "=== OpenRuntimes Executor Logs ===" - docker compose logs openruntimes-executor - - e2e_screenshots_shared_mode: - name: E2E Shared Mode Service Test (Site Screenshots) - runs-on: ubuntu-latest - needs: [ setup, check_database_changes ] - if: needs.check_database_changes.outputs.database_changed == 'true' + needs: [setup, check_database_changes] permissions: contents: read pull-requests: write strategy: fail-fast: false matrix: - tables-mode: [ - 'Shared V1', - 'Shared V2', - ] + mode: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["dedicated","shared_v1","shared_v2"]' || '["dedicated"]') }} steps: - - name: checkout + - name: Checkout repository uses: actions/checkout@v6 - name: Load Cache @@ -640,8 +398,8 @@ jobs: - name: Load and Start Appwrite timeout-minutes: 5 env: - _APP_DATABASE_SHARED_TABLES: database_db_main - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} + _APP_DATABASE_SHARED_TABLES: ${{ matrix.mode != 'dedicated' && 'database_db_main' || '' }} + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.mode == 'shared_v1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -655,7 +413,7 @@ jobs: sleep 1 done - - name: Run Site tests with browser connected in ${{ matrix.tables-mode }} table mode + - name: Run tests uses: itznotabug/php-retry@v3 with: max_attempts: 2 @@ -672,7 +430,5 @@ jobs: - name: Failure Logs if: failure() run: | - echo "=== Appwrite Worker Builds Logs ===" - docker compose logs appwrite-worker-builds - echo "=== OpenRuntimes Executor Logs ===" - docker compose logs openruntimes-executor + echo "=== Appwrite Logs ===" + docker compose logs From 09317f290a2355037b92ff7dae62f979b7254924 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 08:55:47 +0000 Subject: [PATCH 44/51] Clean up database env setup and improve matrix naming Hardcode lowercase env vars per database branch instead of using tr. Use proper casing for database matrix values (MongoDB, MariaDB, PostgreSQL). Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 10cf958652..7d119f78f5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -242,16 +242,20 @@ jobs: - name: Set database environment run: | - DB_LOWER=$(echo "${{ matrix.database }}" | tr 'A-Z' 'a-z') - echo "COMPOSE_PROFILES=${DB_LOWER}" >> $GITHUB_ENV - echo "_APP_DB_ADAPTER=${DB_LOWER}" >> $GITHUB_ENV - echo "_APP_DB_HOST=${DB_LOWER}" >> $GITHUB_ENV - if [ "${{ matrix.database }}" = "MariaDB" ]; then + echo "COMPOSE_PROFILES=mariadb" >> $GITHUB_ENV + echo "_APP_DB_ADAPTER=mariadb" >> $GITHUB_ENV + echo "_APP_DB_HOST=mariadb" >> $GITHUB_ENV echo "_APP_DB_PORT=3306" >> $GITHUB_ENV elif [ "${{ matrix.database }}" = "MongoDB" ]; then + echo "COMPOSE_PROFILES=mongodb" >> $GITHUB_ENV + echo "_APP_DB_ADAPTER=mongodb" >> $GITHUB_ENV + echo "_APP_DB_HOST=mongodb" >> $GITHUB_ENV echo "_APP_DB_PORT=27017" >> $GITHUB_ENV elif [ "${{ matrix.database }}" = "PostgreSQL" ]; then + echo "COMPOSE_PROFILES=postgresql" >> $GITHUB_ENV + echo "_APP_DB_ADAPTER=postgresql" >> $GITHUB_ENV + echo "_APP_DB_HOST=postgresql" >> $GITHUB_ENV echo "_APP_DB_PORT=5432" >> $GITHUB_ENV fi From edd948557e4bc142ce722d59826b108cd8e1eb16 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 11:01:36 +0000 Subject: [PATCH 45/51] Refactor matrix job to use GitHub API and clean up test config Replace shell-based database change detection with github-script using the GitHub API, eliminating the need for a full checkout. Restructure matrix generation with guard clauses and no mutation. Remove ciIgnore exclude group from test command. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 65 ++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7d119f78f5..7b16ac4982 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,31 +20,42 @@ on: default: '' jobs: - check_database_changes: - name: Check if utopia-php/database changed + matrix: + name: Generate test matrix runs-on: ubuntu-latest outputs: - database_changed: ${{ steps.check.outputs.database_changed }} + databases: ${{ steps.generate.outputs.databases }} + modes: ${{ steps.generate.outputs.modes }} steps: - - name: Checkout repository - uses: actions/checkout@v6 + - name: Generate matrix + id: generate + uses: actions/github-script@v7 with: - fetch-depth: 0 + script: | + const allDatabases = ['MariaDB', 'PostgreSQL', 'MongoDB']; + const allModes = ['dedicated', 'shared_v1', 'shared_v2']; - - name: Check for utopia-php/database changes - id: check - run: | - BASE_REF="${{ github.event.pull_request.base.ref }}" - if [ -z "$BASE_REF" ]; then - echo "database_changed=true" >> "$GITHUB_OUTPUT" - exit 0 - fi + const defaultDatabases = ['MongoDB']; + const defaultModes = ['dedicated']; - if git diff "origin/${BASE_REF}" HEAD -- composer.lock | grep -q '"name": "utopia-php/database"'; then - echo "database_changed=true" >> "$GITHUB_OUTPUT" - else - echo "database_changed=false" >> "$GITHUB_OUTPUT" - fi + const pr = context.payload.pull_request; + if (!pr) { + core.setOutput('databases', JSON.stringify(allDatabases)); + core.setOutput('modes', JSON.stringify(allModes)); + return; + } + + const files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pr.number, + }); + + const lockFile = files.find(f => f.filename === 'composer.lock'); + const databaseChanged = lockFile?.patch?.includes('"name": "utopia-php/database"') ?? false; + + core.setOutput('databases', JSON.stringify(databaseChanged ? allDatabases : defaultDatabases)); + core.setOutput('modes', JSON.stringify(databaseChanged ? allModes : defaultModes)); setup: name: Setup & Build Appwrite Image @@ -197,15 +208,15 @@ jobs: e2e_service_test: name: E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service }} runs-on: ubuntu-latest - needs: [setup, check_database_changes] + needs: [setup, matrix] permissions: contents: read pull-requests: write strategy: fail-fast: false matrix: - database: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["MariaDB","PostgreSQL","MongoDB"]' || '["MongoDB"]') }} - mode: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["dedicated","shared_v1","shared_v2"]' || '["dedicated"]') }} + database: ${{ fromJSON(needs.matrix.outputs.databases) }} + mode: ${{ fromJSON(needs.matrix.outputs.modes) }} service: [ Account, Avatars, @@ -304,7 +315,7 @@ jobs: docker compose exec -T \ -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite vendor/bin/paratest --processes $(nproc) $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --exclude-group ciIgnore --log-junit tests/e2e/Services/${{ matrix.service }}/junit.xml + appwrite vendor/bin/paratest --processes $(nproc) $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --log-junit tests/e2e/Services/${{ matrix.service }}/junit.xml - name: Failure Logs if: failure() @@ -315,14 +326,14 @@ jobs: e2e_abuse_enabled: name: E2E / Abuse (${{ matrix.mode }}) runs-on: ubuntu-latest - needs: [setup, check_database_changes] + needs: [setup, matrix] permissions: contents: read pull-requests: write strategy: fail-fast: false matrix: - mode: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["dedicated","shared_v1","shared_v2"]' || '["dedicated"]') }} + mode: ${{ fromJSON(needs.matrix.outputs.modes) }} steps: - name: Checkout repository uses: actions/checkout@v6 @@ -374,14 +385,14 @@ jobs: e2e_screenshots: name: E2E / Screenshots (${{ matrix.mode }}) runs-on: ubuntu-latest - needs: [setup, check_database_changes] + needs: [setup, matrix] permissions: contents: read pull-requests: write strategy: fail-fast: false matrix: - mode: ${{ fromJSON(needs.check_database_changes.outputs.database_changed == 'true' && '["dedicated","shared_v1","shared_v2"]' || '["dedicated"]') }} + mode: ${{ fromJSON(needs.matrix.outputs.modes) }} steps: - name: Checkout repository uses: actions/checkout@v6 From aecca2f503c08edca8c179b6ec8e90259a2c2d50 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:39:14 +0000 Subject: [PATCH 46/51] Consolidate PR workflows into single CI workflow Merge linter, static-analysis, tests, and benchmark workflows into ci.yml with structured job naming (Checks / Format, Tests / E2E / ..., etc.). Shared Docker image build between tests and benchmark. Update actions to latest versions. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/benchmark.yml | 123 ------------------ .github/workflows/{tests.yml => ci.yml} | 161 +++++++++++++++++++++--- .github/workflows/linter.yml | 28 ----- .github/workflows/stale.yml | 2 +- .github/workflows/static-analysis.yml | 21 ---- 5 files changed, 147 insertions(+), 188 deletions(-) delete mode 100644 .github/workflows/benchmark.yml rename .github/workflows/{tests.yml => ci.yml} (70%) delete mode 100644 .github/workflows/linter.yml delete mode 100644 .github/workflows/static-analysis.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index b7b4fa0d2f..0000000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: Benchmark -concurrency: - group: '${{ github.workflow }}-${{ github.ref }}' - cancel-in-progress: true -env: - COMPOSE_FILE: docker-compose.yml - IMAGE: appwrite-dev - CACHE_KEY: 'appwrite-dev-${{ github.event.pull_request.head.sha }}' -'on': - - pull_request -jobs: - setup: - name: Setup & Build Appwrite Image - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - submodules: recursive - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Build Appwrite - uses: docker/build-push-action@v6 - with: - context: . - push: false - tags: '${{ env.IMAGE }}' - load: true - cache-from: type=gha - cache-to: 'type=gha,mode=max' - outputs: 'type=docker,dest=/tmp/${{ env.IMAGE }}.tar' - target: development - build-args: | - DEBUG=false - TESTING=true - VERSION=dev - - name: Cache Docker Image - uses: actions/cache@v4 - with: - key: '${{ env.CACHE_KEY }}' - path: '/tmp/${{ env.IMAGE }}.tar' - benchmarking: - name: Benchmark - runs-on: ubuntu-latest - needs: setup - permissions: - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - name: Load Cache - uses: actions/cache@v4 - with: - key: '${{ env.CACHE_KEY }}' - path: '/tmp/${{ env.IMAGE }}.tar' - fail-on-cache-miss: true - - name: Load and Start Appwrite - run: | - sed -i 's/traefik/localhost/g' .env - docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose up -d - sleep 10 - - name: Install Oha - run: | - echo "deb [signed-by=/usr/share/keyrings/azlux-archive-keyring.gpg] http://packages.azlux.fr/debian/ stable main" | sudo tee /etc/apt/sources.list.d/azlux.list - sudo wget -O /usr/share/keyrings/azlux-archive-keyring.gpg https://azlux.fr/repo.gpg - sudo apt update - sudo apt install oha - oha --version - - name: Benchmark PR - run: 'oha -z 180s http://localhost/v1/health/version --output-format json > benchmark.json' - - name: Cleaning - run: docker compose down -v - - name: Installing latest version - run: | - rm docker-compose.yml - rm .env - curl https://appwrite.io/install/compose -o docker-compose.yml - curl https://appwrite.io/install/env -o .env - sed -i 's/_APP_OPTIONS_ABUSE=enabled/_APP_OPTIONS_ABUSE=disabled/g' .env - docker compose up -d - sleep 10 - - name: Benchmark Latest - run: oha -z 180s http://localhost/v1/health/version --output-format json > benchmark-latest.json - - name: Prepare comment - run: | - echo '## :sparkles: Benchmark results' > benchmark.txt - echo ' ' >> benchmark.txt - echo "- Requests per second: $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt - echo "- Requests with 200 status code: $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt - echo "- P99 latency: $(jq -r '.latencyPercentiles.p99' benchmark.json )" >> benchmark.txt - echo " " >> benchmark.txt - echo " " >> benchmark.txt - echo "## :zap: Benchmark Comparison" >> benchmark.txt - echo " " >> benchmark.txt - echo "| Metric | This PR | Latest version | " >> benchmark.txt - echo "| --- | --- | --- | " >> benchmark.txt - echo "| RPS | $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.summary.requestsPerSec|tonumber|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt - echo "| 200 | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt - echo "| P99 | $(jq -r '.latencyPercentiles.p99' benchmark.json ) | $(jq -r '.latencyPercentiles.p99' benchmark-latest.json ) | " >> benchmark.txt - - name: Save results - uses: actions/upload-artifact@v6 - if: '${{ !cancelled() }}' - with: - name: benchmark.json - path: benchmark.json - retention-days: 7 - - name: Find Comment - if: github.event.pull_request.head.repo.full_name == github.repository - uses: peter-evans/find-comment@v3 - id: fc - with: - issue-number: '${{ github.event.pull_request.number }}' - comment-author: 'github-actions[bot]' - body-includes: Benchmark results - - name: Comment on PR - if: github.event.pull_request.head.repo.full_name == github.repository - uses: peter-evans/create-or-update-comment@v4 - with: - comment-id: '${{ steps.fc.outputs.comment-id }}' - issue-number: '${{ github.event.pull_request.number }}' - body-path: benchmark.txt - edit-mode: replace diff --git a/.github/workflows/tests.yml b/.github/workflows/ci.yml similarity index 70% rename from .github/workflows/tests.yml rename to .github/workflows/ci.yml index 7b16ac4982..233b365a9a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ -name: "Tests" +name: CI concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ci-${{ github.ref }} cancel-in-progress: true env: @@ -20,8 +20,46 @@ on: default: '' jobs: + format: + name: Checks / Format + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 2 + + - run: git checkout HEAD^2 + + - name: Validate composer.json and composer.lock + run: | + docker run --rm -v $PWD:/app composer:2.8 sh -c \ + "composer validate" + + - name: Run Linter + run: | + docker run --rm -v $PWD:/app composer:2.8 sh -c \ + "composer install --profile --ignore-platform-reqs && composer lint" + + analyze: + name: Checks / Analyze + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v6 + + - name: Run CodeQL + run: | + docker run --rm -v $PWD:/app composer:2.8 sh -c \ + "composer install --profile --ignore-platform-reqs && composer check" + + - name: Run Locale check + run: | + docker run --rm -v $PWD:/app node:24-alpine sh -c \ + "cd /app/.github/workflows/static-analysis/locale && node index.js" + matrix: - name: Generate test matrix + name: Tests / Matrix runs-on: ubuntu-latest outputs: databases: ${{ steps.generate.outputs.databases }} @@ -29,7 +67,7 @@ jobs: steps: - name: Generate matrix id: generate - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | const allDatabases = ['MariaDB', 'PostgreSQL', 'MongoDB']; @@ -58,7 +96,7 @@ jobs: core.setOutput('modes', JSON.stringify(databaseChanged ? allModes : defaultModes)); setup: - name: Setup & Build Appwrite Image + name: Setup runs-on: ubuntu-latest steps: - name: Checkout repository @@ -97,14 +135,13 @@ jobs: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar - unit_test: - name: Unit Test + unit: + name: Tests / Unit runs-on: ubuntu-latest needs: setup permissions: contents: read pull-requests: write - steps: - name: checkout uses: actions/checkout@v6 @@ -146,8 +183,8 @@ jobs: -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" appwrite test /usr/src/code/tests/unit - e2e_general_test: - name: E2E General Test + e2e_general: + name: Tests / E2E / General runs-on: ubuntu-latest needs: setup permissions: @@ -205,8 +242,8 @@ jobs: echo "=== Appwrite Logs ===" docker compose logs - e2e_service_test: - name: E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service }} + e2e_service: + name: Tests / E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service }} runs-on: ubuntu-latest needs: [setup, matrix] permissions: @@ -323,8 +360,8 @@ jobs: echo "=== Appwrite Logs ===" docker compose logs - e2e_abuse_enabled: - name: E2E / Abuse (${{ matrix.mode }}) + e2e_abuse: + name: Tests / E2E / Abuse (${{ matrix.mode }}) runs-on: ubuntu-latest needs: [setup, matrix] permissions: @@ -383,7 +420,7 @@ jobs: docker compose logs e2e_screenshots: - name: E2E / Screenshots (${{ matrix.mode }}) + name: Tests / E2E / Screenshots (${{ matrix.mode }}) runs-on: ubuntu-latest needs: [setup, matrix] permissions: @@ -447,3 +484,97 @@ jobs: run: | echo "=== Appwrite Logs ===" docker compose logs + + benchmark: + name: Benchmark + runs-on: ubuntu-latest + needs: setup + permissions: + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Load Cache + uses: actions/cache@v5 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + fail-on-cache-miss: true + + - name: Load and Start Appwrite + run: | + sed -i 's/traefik/localhost/g' .env + docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose up -d + sleep 10 + + - name: Install Oha + run: | + echo "deb [signed-by=/usr/share/keyrings/azlux-archive-keyring.gpg] http://packages.azlux.fr/debian/ stable main" | sudo tee /etc/apt/sources.list.d/azlux.list + sudo wget -O /usr/share/keyrings/azlux-archive-keyring.gpg https://azlux.fr/repo.gpg + sudo apt update + sudo apt install oha + oha --version + + - name: Benchmark PR + run: 'oha -z 180s http://localhost/v1/health/version --output-format json > benchmark.json' + + - name: Cleaning + run: docker compose down -v + + - name: Installing latest version + run: | + rm docker-compose.yml + rm .env + curl https://appwrite.io/install/compose -o docker-compose.yml + curl https://appwrite.io/install/env -o .env + sed -i 's/_APP_OPTIONS_ABUSE=enabled/_APP_OPTIONS_ABUSE=disabled/g' .env + docker compose up -d + sleep 10 + + - name: Benchmark Latest + run: oha -z 180s http://localhost/v1/health/version --output-format json > benchmark-latest.json + + - name: Prepare comment + run: | + echo '## :sparkles: Benchmark results' > benchmark.txt + echo ' ' >> benchmark.txt + echo "- Requests per second: $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt + echo "- Requests with 200 status code: $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt + echo "- P99 latency: $(jq -r '.latencyPercentiles.p99' benchmark.json )" >> benchmark.txt + echo " " >> benchmark.txt + echo " " >> benchmark.txt + echo "## :zap: Benchmark Comparison" >> benchmark.txt + echo " " >> benchmark.txt + echo "| Metric | This PR | Latest version | " >> benchmark.txt + echo "| --- | --- | --- | " >> benchmark.txt + echo "| RPS | $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.summary.requestsPerSec|tonumber|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt + echo "| 200 | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt + echo "| P99 | $(jq -r '.latencyPercentiles.p99' benchmark.json ) | $(jq -r '.latencyPercentiles.p99' benchmark-latest.json ) | " >> benchmark.txt + + - name: Save results + uses: actions/upload-artifact@v7 + if: ${{ !cancelled() }} + with: + name: benchmark.json + path: benchmark.json + retention-days: 7 + + - name: Find Comment + if: github.event.pull_request.head.repo.full_name == github.repository + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: Benchmark results + + - name: Comment on PR + if: github.event.pull_request.head.repo.full_name == github.repository + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: benchmark.txt + edit-mode: replace diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml deleted file mode 100644 index f4ae5df1ce..0000000000 --- a/.github/workflows/linter.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: "Linter" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: [pull_request] -jobs: - lint: - name: Linter - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - fetch-depth: 2 - - - run: git checkout HEAD^2 - - - name: Validate composer.json and composer.lock - run: | - docker run --rm -v $PWD:/app composer:2.8 sh -c \ - "composer validate" - - name: Run Linter - run: | - docker run --rm -v $PWD:/app composer:2.8 sh -c \ - "composer install --profile --ignore-platform-reqs && composer lint" diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 5987eeeb0c..6e4a8ba73b 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/stale@v9 + - uses: actions/stale@v10 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: "This issue has been labeled as a 'question', indicating that it requires additional information from the requestor. It has been inactive for 7 days. If no further activity occurs, this issue will be closed in 14 days." diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml deleted file mode 100644 index a0dc38b3b4..0000000000 --- a/.github/workflows/static-analysis.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: "Static code analysis" - -on: [pull_request] -jobs: - lint: - name: CodeQL - runs-on: ubuntu-latest - - steps: - - name: Check out the repo - uses: actions/checkout@v6 - - - name: Run CodeQL - run: | - docker run --rm -v $PWD:/app composer:2.8 sh -c \ - "composer install --profile --ignore-platform-reqs && composer check" - - - name: Run Locale check - run: | - docker run --rm -v $PWD:/app node:24-alpine sh -c \ - "cd /app/.github/workflows/static-analysis/locale && node index.js" From 8d0a4d7f92bd11f3f6e58ac5205585a14840c945 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:56:41 +0000 Subject: [PATCH 47/51] Consolidate remaining PR workflows and simplify Trivy scan - Move check-dependencies into ci.yml as Checks / Dependencies (upgrade to osv-scanner-reusable-pr.yml@v2.3.3, drop merge_group) - Move pr-scan into ci.yml as Checks / Image (upgrade Trivy to 0.33.1, use SARIF + upload-sarif instead of custom PR comment logic) - Rename Setup job to Build - Fix format job git checkout HEAD^2 to only run on pull_request - Rename PHPStan step correctly (was mislabeled CodeQL) - Add Docker Hub login to benchmark job - Remove no-op pull_request trigger from ai-moderator Co-Authored-By: Claude Opus 4.6 --- .github/workflows/ai-moderator.yml | 2 - .github/workflows/check-dependencies.yml | 19 ---- .github/workflows/ci.yml | 80 +++++++++++++++-- .github/workflows/pr-scan.yml | 106 ----------------------- 4 files changed, 71 insertions(+), 136 deletions(-) delete mode 100644 .github/workflows/check-dependencies.yml delete mode 100644 .github/workflows/pr-scan.yml diff --git a/.github/workflows/ai-moderator.yml b/.github/workflows/ai-moderator.yml index d0b180985f..483f3dbeee 100644 --- a/.github/workflows/ai-moderator.yml +++ b/.github/workflows/ai-moderator.yml @@ -5,8 +5,6 @@ on: types: [opened, edited] issue_comment: types: [created, edited] - pull_request: - types: [opened, edited] pull_request_review: types: [submitted, edited] pull_request_review_comment: diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml deleted file mode 100644 index 17caf3aa6b..0000000000 --- a/.github/workflows/check-dependencies.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Check dependencies - -# Adapted from https://google.github.io/osv-scanner/github-action/#scan-on-pull-request - -on: - pull_request: - branches: [main, 1.*.x] - merge_group: - branches: [main, 1.*.x] - -permissions: - # Require writing security events to upload SARIF file to security tab - security-events: write - # Only need to read contents - contents: read - -jobs: - scan-pr: - uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@v1.7.1" \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 233b365a9a..859ace8c4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,61 @@ on: default: '' jobs: + dependencies: + name: Checks / Dependencies + if: github.event_name == 'pull_request' + permissions: + security-events: write + contents: read + uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@v2.3.3" + + security: + name: Checks / Image + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + steps: + - name: Check out code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: 'recursive' + + - name: Build the Docker image + uses: docker/build-push-action@v6 + with: + context: . + push: false + load: true + tags: pr_image:${{ github.sha }} + target: production + + - name: Run Trivy vulnerability scanner on image + uses: aquasecurity/trivy-action@0.33.1 + with: + image-ref: 'pr_image:${{ github.sha }}' + format: 'sarif' + output: 'trivy-image-results.sarif' + severity: 'CRITICAL,HIGH' + + - name: Run Trivy vulnerability scanner on source code + uses: aquasecurity/trivy-action@0.33.1 + with: + scan-type: 'fs' + scan-ref: '.' + format: 'sarif' + output: 'trivy-fs-results.sarif' + severity: 'CRITICAL,HIGH' + skip-setup-trivy: true + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v4 + if: always() + with: + sarif_file: '.' + format: name: Checks / Format runs-on: ubuntu-latest @@ -30,6 +85,7 @@ jobs: fetch-depth: 2 - run: git checkout HEAD^2 + if: github.event_name == 'pull_request' - name: Validate composer.json and composer.lock run: | @@ -48,7 +104,7 @@ jobs: - name: Check out the repo uses: actions/checkout@v6 - - name: Run CodeQL + - name: Run PHPStan run: | docker run --rm -v $PWD:/app composer:2.8 sh -c \ "composer install --profile --ignore-platform-reqs && composer check" @@ -95,8 +151,8 @@ jobs: core.setOutput('databases', JSON.stringify(databaseChanged ? allDatabases : defaultDatabases)); core.setOutput('modes', JSON.stringify(databaseChanged ? allModes : defaultModes)); - setup: - name: Setup + build: + name: Build runs-on: ubuntu-latest steps: - name: Checkout repository @@ -138,7 +194,7 @@ jobs: unit: name: Tests / Unit runs-on: ubuntu-latest - needs: setup + needs: build permissions: contents: read pull-requests: write @@ -186,7 +242,7 @@ jobs: e2e_general: name: Tests / E2E / General runs-on: ubuntu-latest - needs: setup + needs: build permissions: contents: read pull-requests: write @@ -245,7 +301,7 @@ jobs: e2e_service: name: Tests / E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service }} runs-on: ubuntu-latest - needs: [setup, matrix] + needs: [build, matrix] permissions: contents: read pull-requests: write @@ -363,7 +419,7 @@ jobs: e2e_abuse: name: Tests / E2E / Abuse (${{ matrix.mode }}) runs-on: ubuntu-latest - needs: [setup, matrix] + needs: [build, matrix] permissions: contents: read pull-requests: write @@ -422,7 +478,7 @@ jobs: e2e_screenshots: name: Tests / E2E / Screenshots (${{ matrix.mode }}) runs-on: ubuntu-latest - needs: [setup, matrix] + needs: [build, matrix] permissions: contents: read pull-requests: write @@ -488,7 +544,7 @@ jobs: benchmark: name: Benchmark runs-on: ubuntu-latest - needs: setup + needs: build permissions: pull-requests: write steps: @@ -502,6 +558,12 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Load and Start Appwrite run: | sed -i 's/traefik/localhost/g' .env diff --git a/.github/workflows/pr-scan.yml b/.github/workflows/pr-scan.yml deleted file mode 100644 index 51f3460d03..0000000000 --- a/.github/workflows/pr-scan.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: PR Security Scan -on: - pull_request_target: - types: [opened, synchronize, reopened] - -jobs: - scan: - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - steps: - - name: Check out code - uses: actions/checkout@v6 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - submodules: 'recursive' - - - name: Build the Docker image - uses: docker/build-push-action@v6 - with: - context: . - push: false - load: true - tags: pr_image:${{ github.sha }} - target: production - - - name: Run Trivy vulnerability scanner on image - uses: aquasecurity/trivy-action@0.20.0 - with: - image-ref: 'pr_image:${{ github.sha }}' - format: 'json' - output: 'trivy-image-results.json' - severity: 'CRITICAL,HIGH' - - - name: Run Trivy vulnerability scanner on source code - uses: aquasecurity/trivy-action@0.20.0 - with: - scan-type: 'fs' - scan-ref: '.' - format: 'json' - output: 'trivy-fs-results.json' - severity: 'CRITICAL,HIGH' - - - name: Process Trivy scan results - id: process-results - uses: actions/github-script@v8 - with: - script: | - const fs = require('fs'); - let commentBody = '## Security Scan Results for PR\n\n'; - - function processResults(results, title) { - let sectionBody = `### ${title}\n\n`; - if (results.Results && results.Results.some(result => result.Vulnerabilities && result.Vulnerabilities.length > 0)) { - sectionBody += '| Package | Version | Vulnerability | Severity |\n'; - sectionBody += '|---------|---------|----------------|----------|\n'; - - const uniqueVulns = new Set(); - results.Results.forEach(result => { - if (result.Vulnerabilities) { - result.Vulnerabilities.forEach(vuln => { - const vulnKey = `${vuln.PkgName}-${vuln.InstalledVersion}-${vuln.VulnerabilityID}`; - if (!uniqueVulns.has(vulnKey)) { - uniqueVulns.add(vulnKey); - sectionBody += `| ${vuln.PkgName} | ${vuln.InstalledVersion} | [${vuln.VulnerabilityID}](https://nvd.nist.gov/vuln/detail/${vuln.VulnerabilityID}) | ${vuln.Severity} |\n`; - } - }); - } - }); - } else { - sectionBody += '🎉 No vulnerabilities found!\n'; - } - return sectionBody; - } - - try { - const imageResults = JSON.parse(fs.readFileSync('trivy-image-results.json', 'utf8')); - const fsResults = JSON.parse(fs.readFileSync('trivy-fs-results.json', 'utf8')); - - commentBody += processResults(imageResults, "Docker Image Scan Results"); - commentBody += '\n'; - commentBody += processResults(fsResults, "Source Code Scan Results"); - - } catch (error) { - commentBody += `There was an error while running the security scan: ${error.message}\n`; - commentBody += 'Please contact the core team for assistance.'; - } - - core.setOutput('comment-body', commentBody); - - name: Find Comment - uses: peter-evans/find-comment@v3 - id: fc - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: Security Scan Results for PR - - - name: Create or update comment - uses: peter-evans/create-or-update-comment@v3 - with: - issue-number: ${{ github.event.pull_request.number }} - comment-id: ${{ steps.fc.outputs.comment-id }} - body: ${{ steps.process-results.outputs.comment-body }} - edit-mode: replace From e67ed2660a89bcb92a15726983b80f110fe91802 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:00:44 +0000 Subject: [PATCH 48/51] Add actions: read permission for osv-scanner reusable workflow Co-Authored-By: Claude Opus 4.6 --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 859ace8c4f..f99ee24513 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ jobs: name: Checks / Dependencies if: github.event_name == 'pull_request' permissions: + actions: read security-events: write contents: read uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@v2.3.3" From 26326d05e93dbc643852def0adad296eb407c316 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:04:33 +0000 Subject: [PATCH 49/51] Guard SARIF upload against missing files from failed Trivy scans Co-Authored-By: Claude Opus 4.6 --- .github/workflows/ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f99ee24513..4aa874a286 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,9 +70,17 @@ jobs: severity: 'CRITICAL,HIGH' skip-setup-trivy: true + - name: Check for SARIF files + id: sarif-check + if: always() + run: | + if ls *.sarif 1>/dev/null 2>&1; then + echo "exists=true" >> $GITHUB_OUTPUT + fi + - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v4 - if: always() + if: always() && steps.sarif-check.outputs.exists == 'true' with: sarif_file: '.' From e99f682cd655db92256cb3d3ff37faeebb6b12c0 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:27:38 +0000 Subject: [PATCH 50/51] Update trivy-action to v0.35.0 Co-Authored-By: Claude Opus 4.6 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4aa874a286..8c33a0ff08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,7 +53,7 @@ jobs: target: production - name: Run Trivy vulnerability scanner on image - uses: aquasecurity/trivy-action@0.33.1 + uses: aquasecurity/trivy-action@0.35.0 with: image-ref: 'pr_image:${{ github.sha }}' format: 'sarif' @@ -61,7 +61,7 @@ jobs: severity: 'CRITICAL,HIGH' - name: Run Trivy vulnerability scanner on source code - uses: aquasecurity/trivy-action@0.33.1 + uses: aquasecurity/trivy-action@0.35.0 with: scan-type: 'fs' scan-ref: '.' From 1abbca9318b63bf52252581755d319658626a144 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:42:28 +0000 Subject: [PATCH 51/51] Split SARIF uploads with unique categories to fix codeql-action error Co-Authored-By: Claude Opus 4.6 --- .github/workflows/ci.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c33a0ff08..e59b14e550 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,19 +70,19 @@ jobs: severity: 'CRITICAL,HIGH' skip-setup-trivy: true - - name: Check for SARIF files - id: sarif-check - if: always() - run: | - if ls *.sarif 1>/dev/null 2>&1; then - echo "exists=true" >> $GITHUB_OUTPUT - fi - - - name: Upload Trivy scan results to GitHub Security tab + - name: Upload image scan results uses: github/codeql-action/upload-sarif@v4 - if: always() && steps.sarif-check.outputs.exists == 'true' + if: always() && hashFiles('trivy-image-results.sarif') != '' with: - sarif_file: '.' + sarif_file: 'trivy-image-results.sarif' + category: 'trivy-image' + + - name: Upload source code scan results + uses: github/codeql-action/upload-sarif@v4 + if: always() && hashFiles('trivy-fs-results.sarif') != '' + with: + sarif_file: 'trivy-fs-results.sarif' + category: 'trivy-source' format: name: Checks / Format