From 7644e0fe48eb0a2dbba50d6db71d4c41a8560d50 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 3 Mar 2026 18:48:37 +0530 Subject: [PATCH 01/40] Add realtime metrics for connections, messages, and bandwidth in project usage --- app/controllers/api/project.php | 34 ++++ app/init/constants.php | 6 + app/realtime.php | 107 ++++++++++- .../Utopia/Response/Model/UsageProject.php | 39 ++++ tests/e2e/General/UsageTest.php | 172 ++++++++++++++++++ 5 files changed, 355 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index d24519e3fb..1fd33a5db0 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -72,6 +72,10 @@ Http::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, + METRIC_REALTIME_CONNECTIONS, + METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, + METRIC_REALTIME_INBOUND, + METRIC_REALTIME_OUTBOUND, ], 'period' => [ METRIC_NETWORK_REQUESTS, @@ -85,6 +89,10 @@ Http::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, + METRIC_REALTIME_CONNECTIONS, + METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, + METRIC_REALTIME_INBOUND, + METRIC_REALTIME_OUTBOUND, ] ]; @@ -347,6 +355,26 @@ Http::get('/v1/project/usage') ]; } + // Realtime bandwidth = realtime.inbound + realtime.outbound (per bucket) + $realtimeProjectBandwidth = []; + foreach ($usage[METRIC_REALTIME_INBOUND] as $item) { + $realtimeProjectBandwidth[$item['date']] ??= 0; + $realtimeProjectBandwidth[$item['date']] += $item['value']; + } + + foreach ($usage[METRIC_REALTIME_OUTBOUND] as $item) { + $realtimeProjectBandwidth[$item['date']] ??= 0; + $realtimeProjectBandwidth[$item['date']] += $item['value']; + } + + $realtimeBandwidth = []; + foreach ($realtimeProjectBandwidth as $date => $value) { + $realtimeBandwidth[] = [ + 'date' => $date, + 'value' => $value + ]; + } + $response->dynamic(new Document([ 'requests' => ($usage[METRIC_NETWORK_REQUESTS]), 'network' => $network, @@ -367,10 +395,16 @@ Http::get('/v1/project/usage') 'deploymentsStorageTotal' => $total[METRIC_DEPLOYMENTS_STORAGE], 'databasesReadsTotal' => $total[METRIC_DATABASES_OPERATIONS_READS], 'databasesWritesTotal' => $total[METRIC_DATABASES_OPERATIONS_WRITES], + 'realtimeConnectionsTotal' => $total[METRIC_REALTIME_CONNECTIONS], + 'realtimeMessagesTotal' => $total[METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT], + 'realtimeBandwidthTotal' => ($total[METRIC_REALTIME_INBOUND] ?? 0) + ($total[METRIC_REALTIME_OUTBOUND] ?? 0), 'executionsBreakdown' => $executionsBreakdown, 'bucketsBreakdown' => $bucketsBreakdown, 'databasesReads' => $usage[METRIC_DATABASES_OPERATIONS_READS], 'databasesWrites' => $usage[METRIC_DATABASES_OPERATIONS_WRITES], + 'realtimeConnections' => $usage[METRIC_REALTIME_CONNECTIONS], + 'realtimeMessages' => $usage[METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT], + 'realtimeBandwidth' => $realtimeBandwidth, 'databasesStorageBreakdown' => $databasesStorageBreakdown, 'executionsMbSecondsBreakdown' => $executionsMbSecondsBreakdown, 'buildsMbSecondsBreakdown' => $buildsMbSecondsBreakdown, diff --git a/app/init/constants.php b/app/init/constants.php index c6a5bc853e..cd77b05746 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -361,6 +361,12 @@ const METRIC_AVATARS_SCREENSHOTS_GENERATED = 'avatars.screenshotsGenerated'; const METRIC_FUNCTIONS_RUNTIME = 'functions.runtimes.{runtime}'; const METRIC_SITES_FRAMEWORK = 'sites.frameworks.{framework}'; +// Realtime metrics +const METRIC_REALTIME_CONNECTIONS = 'realtime.connections'; +const METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT = 'realtime.messages.sent'; +const METRIC_REALTIME_INBOUND = 'realtime.inbound'; +const METRIC_REALTIME_OUTBOUND = 'realtime.outbound'; + // Resource types const RESOURCE_TYPE_PROJECTS = 'projects'; const RESOURCE_TYPE_FUNCTIONS = 'functions'; diff --git a/app/realtime.php b/app/realtime.php index 7ec24d03c8..8a066bcbab 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -1,5 +1,6 @@ getSequence()])) { + return $ctx['queueForStatsUsage'][$project->getSequence()]; + } + + global $register; + + /** @var Group $pools */ + $pools = $register->get('pools'); + + $queue = new StatsUsage(new BrokerPool(publisher: $pools->get('publisher'))); + $queue->setProject($project); + + return $ctx['queueForStatsUsage'][$project->getSequence()] = $queue; + } +} + $realtime = getRealtime(); /** @@ -545,20 +572,59 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, } $total = 0; + $outboundBytes = 0; + foreach ($groups as $group) { $data = $event['data']; $data['subscriptions'] = $group['subscriptions']; - $server->send($group['ids'], json_encode([ + $payloadJson = json_encode([ 'type' => 'event', 'data' => $data - ])); - $total += count($group['ids']); + ]); + + $server->send($group['ids'], $payloadJson); + + $count = count($group['ids']); + $total += $count; + $outboundBytes += strlen($payloadJson) * $count; } if ($total > 0) { $register->get('telemetry.messageSentCounter')->add($total); $stats->incr($event['project'], 'messages', $total); + + $projectId = $event['project'] ?? null; + + if (!empty($projectId)) { + try { + $consoleDB = getConsoleDB(); + /** @var Document $project */ + $project = $consoleDB->getAuthorization()->skip( + fn () => $consoleDB->getDocument('projects', $projectId) + ); + + if (!$project->isEmpty()) { + $queueForStatsUsage = getQueueForStatsUsageForProject($project); + + $queueForStatsUsage->addMetric( + METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, + $total + ); + + if ($outboundBytes > 0) { + $queueForStatsUsage->addMetric( + METRIC_REALTIME_OUTBOUND, + $outboundBytes + ); + } + + $queueForStatsUsage->trigger(); + } + } catch (Throwable $th) { + logError($th, 'realtimeUsageOutbound', tags: ['projectId' => $projectId]); + } + } } }); } catch (Throwable $th) { @@ -707,6 +773,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, ]); $stats->incr($project->getId(), 'connections'); $stats->incr($project->getId(), 'connectionsTotal'); + + try { + $queueForStatsUsage = getQueueForStatsUsageForProject($project); + $queueForStatsUsage + ->addMetric(METRIC_REALTIME_CONNECTIONS, 1) + ->trigger(); + } catch (\Throwable $th) { + logError($th, 'realtimeUsageConnections', project: $project); + } + } catch (Throwable $th) { logError($th, 'realtime', project: $project, user: $logUser, authorization: $authorization); @@ -748,6 +824,7 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re $authorization = null; try { + $rawSize = \strlen($message); $response = new Response(new SwooleResponse()); $projectId = $realtime->connections[$connection]['projectId'] ?? null; @@ -786,6 +863,18 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many messages.'); } + // Record realtime inbound bytes for this project + if ($project !== null && !$project->isEmpty()) { + try { + $queueForStatsUsage = getQueueForStatsUsageForProject($project); + $queueForStatsUsage + ->addMetric(METRIC_REALTIME_INBOUND, $rawSize) + ->trigger(); + } catch (Throwable $th) { + logError($th, 'realtimeUsageInbound', project: $project); + } + } + $message = json_decode($message, true); if (is_null($message) || (!array_key_exists('type', $message) && !array_key_exists('data', $message))) { @@ -905,6 +994,18 @@ $server->onClose(function (int $connection) use ($realtime, $stats, $register) { if (array_key_exists($connection, $realtime->connections)) { $stats->decr($realtime->connections[$connection]['projectId'], 'connectionsTotal'); $register->get('telemetry.connectionCounter')->add(-1); + + $projectId = $realtime->connections[$connection]['projectId']; + + $consoleDB = getConsoleDB(); + $project = $consoleDB->getAuthorization()->skip( + fn () => $consoleDB->getDocument('projects', $projectId) + ); + + if (!$project->isEmpty()) { + $queue = getQueueForStatsUsageForProject($project); + $queue->addMetric(METRIC_REALTIME_CONNECTIONS, -1)->trigger(); + } } $realtime->unsubscribe($connection); diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index ee644aa845..c219a35f29 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -100,6 +100,24 @@ class UsageProject extends Model 'default' => 0, 'example' => 0, ]) + ->addRule('realtimeConnectionsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Current aggregated number of open Realtime connections.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('realtimeMessagesTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total number of Realtime messages sent to clients.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('realtimeBandwidthTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total consumed Realtime bandwidth (in bytes).', + 'default' => 0, + 'example' => 0, + ]) ->addRule('requests', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of requests per period.', @@ -114,6 +132,27 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) + ->addRule('realtimeConnections', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of open Realtime connections per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('realtimeMessages', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of Realtime messages sent to clients per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('realtimeBandwidth', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated consumed Realtime bandwidth (in bytes) per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ->addRule('users', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of users per period.', diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index e7b5b1d844..29229ae2bd 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -11,6 +11,7 @@ use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideServer; use Tests\E2E\Services\Functions\FunctionsBase; +use Tests\E2E\Services\Realtime\RealtimeBase; use Tests\E2E\Services\Sites\SitesBase; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -23,6 +24,7 @@ class UsageTest extends Scope use ProjectCustom; use SideServer; use FunctionsBase; + use RealtimeBase; use SitesBase { FunctionsBase::createDeployment insteadof SitesBase; FunctionsBase::setupDeployment insteadof SitesBase; @@ -1408,6 +1410,176 @@ class UsageTest extends Scope }); } + public function testRealtimeUsageMetrics(): void + { + $user = $this->getUser(); + $session = $user['session'] ?? ''; + $projectId = $this->getProject()['$id']; + + // Baseline realtime usage before opening a new connection + $baseline = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $connectionsBefore = $baseline['body']['realtimeConnectionsTotal'] ?? 0; + $messagesBefore = $baseline['body']['realtimeMessagesTotal'] ?? 0; + + $connectionCount = 3; + $clients = []; + + for ($i = 0; $i < $connectionCount; $i++) { + $client = $this->getWebsocket(['documents'], [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ], null); + + $connected = json_decode($client->receive(), true); + $this->assertEquals('connected', $connected['type']); + + $clients[] = $client; + } + + try { + $database = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Realtime Usage DB', + ]); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Realtime Usage Collection', + 'permissions' => [ + Permission::create(Role::user($user['$id'])), + ], + 'documentSecurity' => true, + ]); + + $collectionId = $collection['body']['$id']; + + $attribute = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'key' => 'name', + 'size' => 256, + 'required' => true, + ]); + + $this->assertEquals(202, $attribute['headers']['status-code']); + + $this->assertEventually(function () use ($databaseId, $collectionId, $projectId) { + $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/name', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + $this->assertEquals('available', $response['body']['status']); + }, 30000, 250); + + $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders()), [ + 'documentId' => ID::unique(), + 'data' => [ + 'name' => 'Realtime Usage Doc', + ], + 'permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + ]); + + $this->assertEquals(201, $document['headers']['status-code']); + + $event = json_decode($clients[0]->receive(), true); + $this->assertEquals('event', $event['type']); + + // After creating a document we expect all connections to receive an event + $this->assertEventually(function () use ($connectionsBefore, $messagesBefore, $connectionCount) { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertEquals(200, $response['headers']['status-code']); + + $this->assertArrayHasKey('realtimeConnectionsTotal', $response['body']); + $this->assertArrayHasKey('realtimeMessagesTotal', $response['body']); + $this->assertArrayHasKey('realtimeBandwidthTotal', $response['body']); + $this->assertArrayHasKey('realtimeConnections', $response['body']); + $this->assertArrayHasKey('realtimeMessages', $response['body']); + $this->assertArrayHasKey('realtimeBandwidth', $response['body']); + + // We expect exactly $connectionCount additional open connections and $connectionCount additional message deliveries + $this->assertEquals($connectionsBefore + $connectionCount, $response['body']['realtimeConnectionsTotal']); + $this->assertEquals($messagesBefore + $connectionCount, $response['body']['realtimeMessagesTotal']); + $this->assertGreaterThan(0, $response['body']['realtimeBandwidthTotal']); + + $this->validateDates($response['body']['realtimeConnections']); + $this->validateDates($response['body']['realtimeMessages']); + $this->validateDates($response['body']['realtimeBandwidth']); + }, 60000, 2000); + + // Now close a single connection and ensure the counters reflect it + $clients[0]->close(); + + $this->assertEventually(function () use ($connectionsBefore, $messagesBefore, $connectionCount) { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertEquals(200, $response['headers']['status-code']); + + $this->assertArrayHasKey('realtimeConnectionsTotal', $response['body']); + $this->assertArrayHasKey('realtimeMessagesTotal', $response['body']); + $this->assertArrayHasKey('realtimeBandwidthTotal', $response['body']); + + // One of the connections is closed, so we expect one less open connection. + // Messages and bandwidth are cumulative and should not decrease. + $this->assertEquals($connectionsBefore + $connectionCount - 1, $response['body']['realtimeConnectionsTotal']); + $this->assertEquals($messagesBefore + $connectionCount, $response['body']['realtimeMessagesTotal']); + $this->assertGreaterThan(0, $response['body']['realtimeBandwidthTotal']); + }, 60000, 2000); + } finally { + foreach ($clients as $client) { + $client->close(); + } + } + } + public function tearDown(): void { $this->projectId = ''; From 82db411517641559f9ebbf52d2f3a809f74e8830 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 3 Mar 2026 19:36:49 +0530 Subject: [PATCH 02/40] updated --- app/realtime.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/realtime.php b/app/realtime.php index 8a066bcbab..33833d27ea 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -226,7 +226,7 @@ if (!function_exists('getTelemetry')) { } } -if (!function_exists('queueForStatsUsage')) { +if (!function_exists('getQueueForStatsUsageForProject')) { function getQueueForStatsUsageForProject(Document $project): StatsUsage { $ctx = Coroutine::getContext(); From 9fdd7c1c6e55181eea155a820352e870b5fe0e8d Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 5 Mar 2026 11:30:15 +0530 Subject: [PATCH 03/40] added try catch connection close metric --- app/realtime.php | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 33833d27ea..76a84e1d9d 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -997,14 +997,18 @@ $server->onClose(function (int $connection) use ($realtime, $stats, $register) { $projectId = $realtime->connections[$connection]['projectId']; - $consoleDB = getConsoleDB(); - $project = $consoleDB->getAuthorization()->skip( - fn () => $consoleDB->getDocument('projects', $projectId) - ); + try { + $consoleDB = getConsoleDB(); + $project = $consoleDB->getAuthorization()->skip( + fn () => $consoleDB->getDocument('projects', $projectId) + ); - if (!$project->isEmpty()) { - $queue = getQueueForStatsUsageForProject($project); - $queue->addMetric(METRIC_REALTIME_CONNECTIONS, -1)->trigger(); + if (!$project->isEmpty()) { + $queue = getQueueForStatsUsageForProject($project); + $queue->addMetric(METRIC_REALTIME_CONNECTIONS, -1)->trigger(); + } + } catch (Throwable $th) { + logError($th, 'realtimeUsageConnectionClose', tags: ['projectId' => $projectId]); } } $realtime->unsubscribe($connection); From b5c2cc971610df03dabe1045624eb20b2a2d2953 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 5 Mar 2026 18:10:09 +0530 Subject: [PATCH 04/40] Add triggerStats function for realtime usage metrics tracking --- app/realtime.php | 160 +++++++++++++++++++++++++++++------------------ 1 file changed, 100 insertions(+), 60 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 76a84e1d9d..8250a395a9 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -251,6 +251,43 @@ if (!function_exists('getQueueForStatsUsageForProject')) { } } +if (!function_exists('triggerStats')) { + /** + * Trigger realtime usage stats with a generic metric map. + * + * @param array $event Metrics in the form METRIC_CONSTANT => value + */ + function triggerStats(array $event, string $projectId): void + { + if (empty($projectId)) { + return; + } + + try { + $consoleDB = getConsoleDB(); + /** @var Document $project */ + $project = $consoleDB->getAuthorization()->skip( + fn () => $consoleDB->getDocument('projects', $projectId) + ); + + if ($project->isEmpty()) { + return; + } + + $queueForStatsUsage = getQueueForStatsUsageForProject($project); + + foreach ($event as $metric => $value) { + $queueForStatsUsage->addMetric($metric, $value); + } + + $queueForStatsUsage->trigger(); + $queueForStatsUsage->reset(); + } catch (Throwable $th) { + logError($th, 'realtimeStats', tags: ['projectId' => $projectId]); + } + } +} + $realtime = getRealtime(); /** @@ -597,33 +634,15 @@ $server->onWorkerStart(function (int $workerId) use ($server, $register, $stats, $projectId = $event['project'] ?? null; if (!empty($projectId)) { - try { - $consoleDB = getConsoleDB(); - /** @var Document $project */ - $project = $consoleDB->getAuthorization()->skip( - fn () => $consoleDB->getDocument('projects', $projectId) - ); + $metrics = [ + METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT => $total, + ]; - if (!$project->isEmpty()) { - $queueForStatsUsage = getQueueForStatsUsageForProject($project); - - $queueForStatsUsage->addMetric( - METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, - $total - ); - - if ($outboundBytes > 0) { - $queueForStatsUsage->addMetric( - METRIC_REALTIME_OUTBOUND, - $outboundBytes - ); - } - - $queueForStatsUsage->trigger(); - } - } catch (Throwable $th) { - logError($th, 'realtimeUsageOutbound', tags: ['projectId' => $projectId]); + if ($outboundBytes > 0) { + $metrics[METRIC_REALTIME_OUTBOUND] = $outboundBytes; } + + triggerStats($metrics, $projectId); } } }); @@ -701,6 +720,19 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many requests'); } + // Record realtime inbound bytes for this project (WS handshake + query params) + try { + $rawSize = $request->getSize(); + } catch (Throwable) { + $rawSize = \strlen((string) $request->getURI()); + } + + if ($rawSize > 0) { + triggerStats([ + METRIC_REALTIME_INBOUND => $rawSize, + ], $project->getId()); + } + /* * Validate Client Domain - Check to avoid CSRF attack. * Adding Appwrite API domains to allow XDOMAIN communication. @@ -755,14 +787,16 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $user = empty($user->getId()) ? null : $response->output($user, Response::MODEL_ACCOUNT); - $server->send([$connection], json_encode([ + $connectedPayloadJson = json_encode([ 'type' => 'connected', 'data' => [ 'channels' => $names, 'subscriptions' => $mapping, 'user' => $user ] - ])); + ]); + + $server->send([$connection], $connectedPayloadJson); $register->get('telemetry.connectionCounter')->add(1); $register->get('telemetry.connectionCreatedCounter')->add(1); @@ -774,14 +808,10 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, $stats->incr($project->getId(), 'connections'); $stats->incr($project->getId(), 'connectionsTotal'); - try { - $queueForStatsUsage = getQueueForStatsUsageForProject($project); - $queueForStatsUsage - ->addMetric(METRIC_REALTIME_CONNECTIONS, 1) - ->trigger(); - } catch (\Throwable $th) { - logError($th, 'realtimeUsageConnections', project: $project); - } + $connectedOutboundBytes = \strlen($connectedPayloadJson); + + triggerStats([METRIC_REALTIME_CONNECTIONS => 1, METRIC_REALTIME_OUTBOUND => $connectedOutboundBytes], $project->getId()); + } catch (Throwable $th) { logError($th, 'realtime', project: $project, user: $logUser, authorization: $authorization); @@ -865,14 +895,9 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re // Record realtime inbound bytes for this project if ($project !== null && !$project->isEmpty()) { - try { - $queueForStatsUsage = getQueueForStatsUsageForProject($project); - $queueForStatsUsage - ->addMetric(METRIC_REALTIME_INBOUND, $rawSize) - ->trigger(); - } catch (Throwable $th) { - logError($th, 'realtimeUsageInbound', project: $project); - } + triggerStats([ + METRIC_REALTIME_INBOUND => $rawSize, + ], $project->getId()); } $message = json_decode($message, true); @@ -883,9 +908,21 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re switch ($message['type']) { case 'ping': - $server->send([$connection], json_encode([ + $pongPayloadJson = json_encode([ 'type' => 'pong' - ])); + ]); + + $server->send([$connection], $pongPayloadJson); + + if ($project !== null && !$project->isEmpty()) { + $pongOutboundBytes = \strlen($pongPayloadJson); + + if ($pongOutboundBytes > 0) { + triggerStats([ + METRIC_REALTIME_OUTBOUND => $pongOutboundBytes, + ], $project->getId()); + } + } break; case 'authentication': @@ -946,14 +983,27 @@ $server->onMessage(function (int $connection, string $message) use ($server, $re } $user = $response->output($user, Response::MODEL_ACCOUNT); - $server->send([$connection], json_encode([ + + $authResponsePayloadJson = json_encode([ 'type' => 'response', 'data' => [ 'to' => 'authentication', 'success' => true, 'user' => $user ] - ])); + ]); + + $server->send([$connection], $authResponsePayloadJson); + + if ($project !== null && !$project->isEmpty()) { + $authOutboundBytes = \strlen($authResponsePayloadJson); + + if ($authOutboundBytes > 0) { + triggerStats([ + METRIC_REALTIME_OUTBOUND => $authOutboundBytes, + ], $project->getId()); + } + } break; @@ -997,19 +1047,9 @@ $server->onClose(function (int $connection) use ($realtime, $stats, $register) { $projectId = $realtime->connections[$connection]['projectId']; - try { - $consoleDB = getConsoleDB(); - $project = $consoleDB->getAuthorization()->skip( - fn () => $consoleDB->getDocument('projects', $projectId) - ); - - if (!$project->isEmpty()) { - $queue = getQueueForStatsUsageForProject($project); - $queue->addMetric(METRIC_REALTIME_CONNECTIONS, -1)->trigger(); - } - } catch (Throwable $th) { - logError($th, 'realtimeUsageConnectionClose', tags: ['projectId' => $projectId]); - } + triggerStats([ + METRIC_REALTIME_CONNECTIONS => -1, + ], $projectId); } $realtime->unsubscribe($connection); From 03d7bccde2a4465d651a5575b4a3321990ced6fe Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 5 Mar 2026 18:14:16 +0530 Subject: [PATCH 05/40] updated tests --- tests/e2e/General/UsageTest.php | 55 +++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 29229ae2bd..2eb2673c8a 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1546,6 +1546,61 @@ class UsageTest extends Scope $this->validateDates($response['body']['realtimeBandwidth']); }, 60000, 2000); + // Capture a snapshot of usage after the broadcasted document event + $afterEventUsage = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertEquals(200, $afterEventUsage['headers']['status-code']); + + $connectionsAfterEvent = $afterEventUsage['body']['realtimeConnectionsTotal'] ?? 0; + $messagesAfterEvent = $afterEventUsage['body']['realtimeMessagesTotal'] ?? 0; + $bandwidthAfterEvent = $afterEventUsage['body']['realtimeBandwidthTotal'] ?? 0; + + // Send a ping over an existing connection to exercise the ping/pong + // realtime usage metrics path (inbound + outbound bytes) without + // generating additional "messages sent" usage. + $clients[1]->send(json_encode([ + 'type' => 'ping', + ])); + + $firstMessage = json_decode($clients[1]->receive(), true); + // Depending on timing, the first frame we see here can be either + // a broadcast "event" (from another operation) or the "pong" + // response. Both are valid, and in either case the ping/pong + // traffic still exercises the realtime usage metrics paths. + $this->assertContains($firstMessage['type'], ['event']); + + // We expect: + // - connections count to remain the same + // - messages count to remain the same (no new broadcast events) + // - bandwidth total to increase because of ping/pong traffic + $this->assertEventually(function () use ($connectionsAfterEvent, $messagesAfterEvent, $bandwidthAfterEvent) { + $response = $this->client->call( + Client::METHOD_GET, + '/project/usage', + $this->getConsoleHeaders(), + [ + 'period' => '1h', + 'startDate' => self::getToday(), + 'endDate' => self::getTomorrow(), + ] + ); + + $this->assertEquals(200, $response['headers']['status-code']); + + $this->assertEquals($connectionsAfterEvent, $response['body']['realtimeConnectionsTotal']); + $this->assertEquals($messagesAfterEvent, $response['body']['realtimeMessagesTotal']); + $this->assertGreaterThan($bandwidthAfterEvent, $response['body']['realtimeBandwidthTotal']); + }, 60000, 2000); + // Now close a single connection and ensure the counters reflect it $clients[0]->close(); From ece5b8173277ff0d41cbbb0b56c798baf0a0f2fe Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 6 Mar 2026 15:27:30 +0530 Subject: [PATCH 06/40] updated test --- tests/e2e/General/UsageTest.php | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 2eb2673c8a..f51b04b9c6 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1571,12 +1571,23 @@ class UsageTest extends Scope 'type' => 'ping', ])); - $firstMessage = json_decode($clients[1]->receive(), true); - // Depending on timing, the first frame we see here can be either - // a broadcast "event" (from another operation) or the "pong" - // response. Both are valid, and in either case the ping/pong - // traffic still exercises the realtime usage metrics paths. - $this->assertContains($firstMessage['type'], ['event']); + // A broadcast document "event" can still be queued for this connection. + // Read until we see the pong response (and fail fast on unexpected frames). + $pong = null; + for ($i = 0; $i < 5; $i++) { + $frame = json_decode($clients[1]->receive(), true); + $this->assertIsArray($frame); + $this->assertArrayHasKey('type', $frame); + + if ($frame['type'] === 'pong') { + $pong = $frame; + break; + } + + $this->assertEquals('event', $frame['type']); + } + + $this->assertNotNull($pong, 'Expected to receive a pong frame after ping.'); // We expect: // - connections count to remain the same From 0ea196d21c3a76fcee5d04fc3024fba1d6d6eadd Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 6 Mar 2026 15:37:30 +0530 Subject: [PATCH 07/40] updated inbound raw size to the request size --- app/realtime.php | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/app/realtime.php b/app/realtime.php index 8250a395a9..9ab6601ed7 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -720,18 +720,11 @@ $server->onOpen(function (int $connection, SwooleRequest $request) use ($server, throw new Exception(Exception::REALTIME_TOO_MANY_MESSAGES, 'Too many requests'); } - // Record realtime inbound bytes for this project (WS handshake + query params) - try { - $rawSize = $request->getSize(); - } catch (Throwable) { - $rawSize = \strlen((string) $request->getURI()); - } + $rawSize = $request->getSize(); - if ($rawSize > 0) { - triggerStats([ - METRIC_REALTIME_INBOUND => $rawSize, - ], $project->getId()); - } + triggerStats([ + METRIC_REALTIME_INBOUND => $rawSize, + ], $project->getId()); /* * Validate Client Domain - Check to avoid CSRF attack. From 39f3bc7b9dfc09d6c277b01a6efc61a43e30e637 Mon Sep 17 00:00:00 2001 From: eldadfux Date: Mon, 9 Mar 2026 20:08:41 +0100 Subject: [PATCH 08/40] Fix SDK namespace call --- app/controllers/general.php | 3 +++ app/http.php | 3 +++ 2 files changed, 6 insertions(+) diff --git a/app/controllers/general.php b/app/controllers/general.php index 57edd98bc4..f77aa3ec52 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -1399,6 +1399,9 @@ Http::error() $sdk = $route?->getLabel("sdk", false); $action = 'UNKNOWN_NAMESPACE.UNKNOWN.METHOD'; if (!empty($sdk)) { + if (\is_array($sdk)) { + $sdk = $sdk[0]; + } /** @var \Appwrite\SDK\Method $sdk */ $action = $sdk->getNamespace() . '.' . $sdk->getMethodName(); } elseif ($route === null) { diff --git a/app/http.php b/app/http.php index 7f771de130..1302940856 100644 --- a/app/http.php +++ b/app/http.php @@ -581,6 +581,9 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool $action = 'UNKNOWN_NAMESPACE.UNKNOWN.METHOD'; if (!empty($sdk)) { + if (\is_array($sdk)) { + $sdk = $sdk[0]; + } /** @var Appwrite\SDK\Method $sdk */ $action = $sdk->getNamespace() . '.' . $sdk->getMethodName(); } elseif ($route === null) { From a0167d6c6c18560cd8d6d4b8762437c7358916aa Mon Sep 17 00:00:00 2001 From: eldadfux Date: Mon, 9 Mar 2026 20:17:32 +0100 Subject: [PATCH 09/40] Fix for when vcs comment is empty --- src/Appwrite/Vcs/Comment.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Vcs/Comment.php b/src/Appwrite/Vcs/Comment.php index 90f9c8f95d..e6d6996748 100644 --- a/src/Appwrite/Vcs/Comment.php +++ b/src/Appwrite/Vcs/Comment.php @@ -251,7 +251,7 @@ class Comment $json = \base64_decode($state); $builds = \json_decode($json, true); - $this->builds = $builds; + $this->builds = \is_array($builds) ? $builds : []; return $this; } From eccc39a4669db5afb52e35928e071524329a2a05 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 10 Mar 2026 12:15:25 +0530 Subject: [PATCH 10/40] refactor: remove realtime metrics from project usage endpoints and related tests --- app/controllers/api/project.php | 34 --- app/realtime.php | 59 +---- .../Utopia/Response/Model/UsageProject.php | 39 --- tests/e2e/General/UsageTest.php | 236 ------------------ 4 files changed, 1 insertion(+), 367 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 1fd33a5db0..d24519e3fb 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -72,10 +72,6 @@ Http::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, - METRIC_REALTIME_CONNECTIONS, - METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, - METRIC_REALTIME_INBOUND, - METRIC_REALTIME_OUTBOUND, ], 'period' => [ METRIC_NETWORK_REQUESTS, @@ -89,10 +85,6 @@ Http::get('/v1/project/usage') METRIC_DATABASES_OPERATIONS_READS, METRIC_DATABASES_OPERATIONS_WRITES, METRIC_FILES_IMAGES_TRANSFORMED, - METRIC_REALTIME_CONNECTIONS, - METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT, - METRIC_REALTIME_INBOUND, - METRIC_REALTIME_OUTBOUND, ] ]; @@ -355,26 +347,6 @@ Http::get('/v1/project/usage') ]; } - // Realtime bandwidth = realtime.inbound + realtime.outbound (per bucket) - $realtimeProjectBandwidth = []; - foreach ($usage[METRIC_REALTIME_INBOUND] as $item) { - $realtimeProjectBandwidth[$item['date']] ??= 0; - $realtimeProjectBandwidth[$item['date']] += $item['value']; - } - - foreach ($usage[METRIC_REALTIME_OUTBOUND] as $item) { - $realtimeProjectBandwidth[$item['date']] ??= 0; - $realtimeProjectBandwidth[$item['date']] += $item['value']; - } - - $realtimeBandwidth = []; - foreach ($realtimeProjectBandwidth as $date => $value) { - $realtimeBandwidth[] = [ - 'date' => $date, - 'value' => $value - ]; - } - $response->dynamic(new Document([ 'requests' => ($usage[METRIC_NETWORK_REQUESTS]), 'network' => $network, @@ -395,16 +367,10 @@ Http::get('/v1/project/usage') 'deploymentsStorageTotal' => $total[METRIC_DEPLOYMENTS_STORAGE], 'databasesReadsTotal' => $total[METRIC_DATABASES_OPERATIONS_READS], 'databasesWritesTotal' => $total[METRIC_DATABASES_OPERATIONS_WRITES], - 'realtimeConnectionsTotal' => $total[METRIC_REALTIME_CONNECTIONS], - 'realtimeMessagesTotal' => $total[METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT], - 'realtimeBandwidthTotal' => ($total[METRIC_REALTIME_INBOUND] ?? 0) + ($total[METRIC_REALTIME_OUTBOUND] ?? 0), 'executionsBreakdown' => $executionsBreakdown, 'bucketsBreakdown' => $bucketsBreakdown, 'databasesReads' => $usage[METRIC_DATABASES_OPERATIONS_READS], 'databasesWrites' => $usage[METRIC_DATABASES_OPERATIONS_WRITES], - 'realtimeConnections' => $usage[METRIC_REALTIME_CONNECTIONS], - 'realtimeMessages' => $usage[METRIC_REALTIME_CONNECTIONS_MESSAGES_SENT], - 'realtimeBandwidth' => $realtimeBandwidth, 'databasesStorageBreakdown' => $databasesStorageBreakdown, 'executionsMbSecondsBreakdown' => $executionsMbSecondsBreakdown, 'buildsMbSecondsBreakdown' => $buildsMbSecondsBreakdown, diff --git a/app/realtime.php b/app/realtime.php index 9ab6601ed7..1e58832203 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -1,6 +1,5 @@ getSequence()])) { - return $ctx['queueForStatsUsage'][$project->getSequence()]; - } - - global $register; - - /** @var Group $pools */ - $pools = $register->get('pools'); - - $queue = new StatsUsage(new BrokerPool(publisher: $pools->get('publisher'))); - $queue->setProject($project); - - return $ctx['queueForStatsUsage'][$project->getSequence()] = $queue; - } -} - if (!function_exists('triggerStats')) { - /** - * Trigger realtime usage stats with a generic metric map. - * - * @param array $event Metrics in the form METRIC_CONSTANT => value - */ function triggerStats(array $event, string $projectId): void { - if (empty($projectId)) { - return; - } - - try { - $consoleDB = getConsoleDB(); - /** @var Document $project */ - $project = $consoleDB->getAuthorization()->skip( - fn () => $consoleDB->getDocument('projects', $projectId) - ); - - if ($project->isEmpty()) { - return; - } - - $queueForStatsUsage = getQueueForStatsUsageForProject($project); - - foreach ($event as $metric => $value) { - $queueForStatsUsage->addMetric($metric, $value); - } - - $queueForStatsUsage->trigger(); - $queueForStatsUsage->reset(); - } catch (Throwable $th) { - logError($th, 'realtimeStats', tags: ['projectId' => $projectId]); - } + return; } } diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index c219a35f29..ee644aa845 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -100,24 +100,6 @@ class UsageProject extends Model 'default' => 0, 'example' => 0, ]) - ->addRule('realtimeConnectionsTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Current aggregated number of open Realtime connections.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('realtimeMessagesTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total number of Realtime messages sent to clients.', - 'default' => 0, - 'example' => 0, - ]) - ->addRule('realtimeBandwidthTotal', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'Total consumed Realtime bandwidth (in bytes).', - 'default' => 0, - 'example' => 0, - ]) ->addRule('requests', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of requests per period.', @@ -132,27 +114,6 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) - ->addRule('realtimeConnections', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of open Realtime connections per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - ->addRule('realtimeMessages', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated number of Realtime messages sent to clients per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) - ->addRule('realtimeBandwidth', [ - 'type' => Response::MODEL_METRIC, - 'description' => 'Aggregated consumed Realtime bandwidth (in bytes) per period.', - 'default' => [], - 'example' => [], - 'array' => true - ]) ->addRule('users', [ 'type' => Response::MODEL_METRIC, 'description' => 'Aggregated number of users per period.', diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index f51b04b9c6..cabbdf6913 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -1410,242 +1410,6 @@ class UsageTest extends Scope }); } - public function testRealtimeUsageMetrics(): void - { - $user = $this->getUser(); - $session = $user['session'] ?? ''; - $projectId = $this->getProject()['$id']; - - // Baseline realtime usage before opening a new connection - $baseline = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $connectionsBefore = $baseline['body']['realtimeConnectionsTotal'] ?? 0; - $messagesBefore = $baseline['body']['realtimeMessagesTotal'] ?? 0; - - $connectionCount = 3; - $clients = []; - - for ($i = 0; $i < $connectionCount; $i++) { - $client = $this->getWebsocket(['documents'], [ - 'origin' => 'http://localhost', - 'cookie' => 'a_session_' . $projectId . '=' . $session, - ], null); - - $connected = json_decode($client->receive(), true); - $this->assertEquals('connected', $connected['type']); - - $clients[] = $client; - } - - try { - $database = $this->client->call(Client::METHOD_POST, '/databases', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - 'x-appwrite-key' => $this->getProject()['apiKey'], - ], [ - 'databaseId' => ID::unique(), - 'name' => 'Realtime Usage DB', - ]); - - $databaseId = $database['body']['$id']; - - $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - 'x-appwrite-key' => $this->getProject()['apiKey'], - ], [ - 'collectionId' => ID::unique(), - 'name' => 'Realtime Usage Collection', - 'permissions' => [ - Permission::create(Role::user($user['$id'])), - ], - 'documentSecurity' => true, - ]); - - $collectionId = $collection['body']['$id']; - - $attribute = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/string', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - 'x-appwrite-key' => $this->getProject()['apiKey'], - ], [ - 'key' => 'name', - 'size' => 256, - 'required' => true, - ]); - - $this->assertEquals(202, $attribute['headers']['status-code']); - - $this->assertEventually(function () use ($databaseId, $collectionId, $projectId) { - $response = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes/name', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - 'x-appwrite-key' => $this->getProject()['apiKey'], - ]); - $this->assertEquals('available', $response['body']['status']); - }, 30000, 250); - - $document = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $projectId, - ], $this->getHeaders()), [ - 'documentId' => ID::unique(), - 'data' => [ - 'name' => 'Realtime Usage Doc', - ], - 'permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - ]); - - $this->assertEquals(201, $document['headers']['status-code']); - - $event = json_decode($clients[0]->receive(), true); - $this->assertEquals('event', $event['type']); - - // After creating a document we expect all connections to receive an event - $this->assertEventually(function () use ($connectionsBefore, $messagesBefore, $connectionCount) { - $response = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $this->assertEquals(200, $response['headers']['status-code']); - - $this->assertArrayHasKey('realtimeConnectionsTotal', $response['body']); - $this->assertArrayHasKey('realtimeMessagesTotal', $response['body']); - $this->assertArrayHasKey('realtimeBandwidthTotal', $response['body']); - $this->assertArrayHasKey('realtimeConnections', $response['body']); - $this->assertArrayHasKey('realtimeMessages', $response['body']); - $this->assertArrayHasKey('realtimeBandwidth', $response['body']); - - // We expect exactly $connectionCount additional open connections and $connectionCount additional message deliveries - $this->assertEquals($connectionsBefore + $connectionCount, $response['body']['realtimeConnectionsTotal']); - $this->assertEquals($messagesBefore + $connectionCount, $response['body']['realtimeMessagesTotal']); - $this->assertGreaterThan(0, $response['body']['realtimeBandwidthTotal']); - - $this->validateDates($response['body']['realtimeConnections']); - $this->validateDates($response['body']['realtimeMessages']); - $this->validateDates($response['body']['realtimeBandwidth']); - }, 60000, 2000); - - // Capture a snapshot of usage after the broadcasted document event - $afterEventUsage = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $this->assertEquals(200, $afterEventUsage['headers']['status-code']); - - $connectionsAfterEvent = $afterEventUsage['body']['realtimeConnectionsTotal'] ?? 0; - $messagesAfterEvent = $afterEventUsage['body']['realtimeMessagesTotal'] ?? 0; - $bandwidthAfterEvent = $afterEventUsage['body']['realtimeBandwidthTotal'] ?? 0; - - // Send a ping over an existing connection to exercise the ping/pong - // realtime usage metrics path (inbound + outbound bytes) without - // generating additional "messages sent" usage. - $clients[1]->send(json_encode([ - 'type' => 'ping', - ])); - - // A broadcast document "event" can still be queued for this connection. - // Read until we see the pong response (and fail fast on unexpected frames). - $pong = null; - for ($i = 0; $i < 5; $i++) { - $frame = json_decode($clients[1]->receive(), true); - $this->assertIsArray($frame); - $this->assertArrayHasKey('type', $frame); - - if ($frame['type'] === 'pong') { - $pong = $frame; - break; - } - - $this->assertEquals('event', $frame['type']); - } - - $this->assertNotNull($pong, 'Expected to receive a pong frame after ping.'); - - // We expect: - // - connections count to remain the same - // - messages count to remain the same (no new broadcast events) - // - bandwidth total to increase because of ping/pong traffic - $this->assertEventually(function () use ($connectionsAfterEvent, $messagesAfterEvent, $bandwidthAfterEvent) { - $response = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $this->assertEquals(200, $response['headers']['status-code']); - - $this->assertEquals($connectionsAfterEvent, $response['body']['realtimeConnectionsTotal']); - $this->assertEquals($messagesAfterEvent, $response['body']['realtimeMessagesTotal']); - $this->assertGreaterThan($bandwidthAfterEvent, $response['body']['realtimeBandwidthTotal']); - }, 60000, 2000); - - // Now close a single connection and ensure the counters reflect it - $clients[0]->close(); - - $this->assertEventually(function () use ($connectionsBefore, $messagesBefore, $connectionCount) { - $response = $this->client->call( - Client::METHOD_GET, - '/project/usage', - $this->getConsoleHeaders(), - [ - 'period' => '1h', - 'startDate' => self::getToday(), - 'endDate' => self::getTomorrow(), - ] - ); - - $this->assertEquals(200, $response['headers']['status-code']); - - $this->assertArrayHasKey('realtimeConnectionsTotal', $response['body']); - $this->assertArrayHasKey('realtimeMessagesTotal', $response['body']); - $this->assertArrayHasKey('realtimeBandwidthTotal', $response['body']); - - // One of the connections is closed, so we expect one less open connection. - // Messages and bandwidth are cumulative and should not decrease. - $this->assertEquals($connectionsBefore + $connectionCount - 1, $response['body']['realtimeConnectionsTotal']); - $this->assertEquals($messagesBefore + $connectionCount, $response['body']['realtimeMessagesTotal']); - $this->assertGreaterThan(0, $response['body']['realtimeBandwidthTotal']); - }, 60000, 2000); - } finally { - foreach ($clients as $client) { - $client->close(); - } - } - } - public function tearDown(): void { $this->projectId = ''; From 7f4cba276e63700c75052bade5e41a62d11e6414 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 10 Mar 2026 12:19:46 +0530 Subject: [PATCH 11/40] updated tests --- tests/e2e/General/UsageTest.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index cabbdf6913..e7b5b1d844 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -11,7 +11,6 @@ use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideServer; use Tests\E2E\Services\Functions\FunctionsBase; -use Tests\E2E\Services\Realtime\RealtimeBase; use Tests\E2E\Services\Sites\SitesBase; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -24,7 +23,6 @@ class UsageTest extends Scope use ProjectCustom; use SideServer; use FunctionsBase; - use RealtimeBase; use SitesBase { FunctionsBase::createDeployment insteadof SitesBase; FunctionsBase::setupDeployment insteadof SitesBase; From ee2e41fa45fc358ce2266f4b4a2b6367b227e408 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Tue, 10 Mar 2026 09:20:57 +0000 Subject: [PATCH 12/40] feat: add messaging resource migration support --- composer.json | 2 +- composer.lock | 149 ++-- src/Appwrite/Platform/Workers/Migrations.php | 10 + .../Utopia/Response/Model/MigrationReport.php | 24 + .../Services/Migrations/MigrationsBase.php | 838 ++++++++++++++++++ 5 files changed, 947 insertions(+), 76 deletions(-) diff --git a/composer.json b/composer.json index d7f0ae66b5..abe51e500b 100644 --- a/composer.json +++ b/composer.json @@ -70,7 +70,7 @@ "utopia-php/locale": "0.8.*", "utopia-php/logger": "0.6.*", "utopia-php/messaging": "0.20.*", - "utopia-php/migration": "1.6.*", + "utopia-php/migration": "1.7.*", "utopia-php/platform": "0.7.*", "utopia-php/pools": "1.*", "utopia-php/span": "1.1.*", diff --git a/composer.lock b/composer.lock index 86093899d0..738db8f287 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1cc64e07484256225f56bd525674c3b8", + "content-hash": "b99693284208ff3d006260a089a4f7b9", "packages": [ { "name": "adhocore/jwt", @@ -4058,16 +4058,16 @@ }, { "name": "utopia-php/domains", - "version": "1.0.5", + "version": "1.0.2", "source": { "type": "git", "url": "https://github.com/utopia-php/domains.git", - "reference": "0edf6bb2b07f30db849a267027077bf5abb994c6" + "reference": "b4896a6746f0fbe29dfd5e32f7790bd94c1af1e6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/domains/zipball/0edf6bb2b07f30db849a267027077bf5abb994c6", - "reference": "0edf6bb2b07f30db849a267027077bf5abb994c6", + "url": "https://api.github.com/repos/utopia-php/domains/zipball/b4896a6746f0fbe29dfd5e32f7790bd94c1af1e6", + "reference": "b4896a6746f0fbe29dfd5e32f7790bd94c1af1e6", "shasum": "" }, "require": { @@ -4114,9 +4114,9 @@ ], "support": { "issues": "https://github.com/utopia-php/domains/issues", - "source": "https://github.com/utopia-php/domains/tree/1.0.5" + "source": "https://github.com/utopia-php/domains/tree/1.0.2" }, - "time": "2026-03-03T09:20:50+00:00" + "time": "2026-02-25T08:18:25+00:00" }, { "name": "utopia-php/dsn", @@ -4517,16 +4517,16 @@ }, { "name": "utopia-php/migration", - "version": "1.6.3", + "version": "1.7.0", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "c2d016944cb029fa5ff822ceee704785a06ef289" + "reference": "97583ae502e40621ea91a71de19d053c5ae2e558" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/c2d016944cb029fa5ff822ceee704785a06ef289", - "reference": "c2d016944cb029fa5ff822ceee704785a06ef289", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/97583ae502e40621ea91a71de19d053c5ae2e558", + "reference": "97583ae502e40621ea91a71de19d053c5ae2e558", "shasum": "" }, "require": { @@ -4566,9 +4566,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/1.6.3" + "source": "https://github.com/utopia-php/migration/tree/1.7.0" }, - "time": "2026-03-04T07:08:22+00:00" + "time": "2026-03-10T06:36:27+00:00" }, { "name": "utopia-php/mongo", @@ -5215,16 +5215,16 @@ }, { "name": "utopia-php/vcs", - "version": "2.0.1", + "version": "2.0.0", "source": { "type": "git", "url": "https://github.com/utopia-php/vcs.git", - "reference": "92a1650824ba0c5e6a1bc46e622ac87c50a08920" + "reference": "058049326e04a2a0c2f0ce8ad00c7e84825aba14" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/vcs/zipball/92a1650824ba0c5e6a1bc46e622ac87c50a08920", - "reference": "92a1650824ba0c5e6a1bc46e622ac87c50a08920", + "url": "https://api.github.com/repos/utopia-php/vcs/zipball/058049326e04a2a0c2f0ce8ad00c7e84825aba14", + "reference": "058049326e04a2a0c2f0ce8ad00c7e84825aba14", "shasum": "" }, "require": { @@ -5258,9 +5258,9 @@ ], "support": { "issues": "https://github.com/utopia-php/vcs/issues", - "source": "https://github.com/utopia-php/vcs/tree/2.0.1" + "source": "https://github.com/utopia-php/vcs/tree/2.0.0" }, - "time": "2026-02-27T12:18:49+00:00" + "time": "2026-02-25T11:36:45+00:00" }, { "name": "utopia-php/websocket", @@ -5438,16 +5438,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "1.11.6", + "version": "1.11.1", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "f80e302d000cdc2f98b4bb5ff2fc3bd0bdff7b38" + "reference": "6ff411f26f2750eea05c7598c14bb3a2ada898cb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/f80e302d000cdc2f98b4bb5ff2fc3bd0bdff7b38", - "reference": "f80e302d000cdc2f98b4bb5ff2fc3bd0bdff7b38", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/6ff411f26f2750eea05c7598c14bb3a2ada898cb", + "reference": "6ff411f26f2750eea05c7598c14bb3a2ada898cb", "shasum": "" }, "require": { @@ -5483,22 +5483,22 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/1.11.6" + "source": "https://github.com/appwrite/sdk-generator/tree/1.11.1" }, - "time": "2026-03-09T07:12:51+00:00" + "time": "2026-02-25T07:15:19+00:00" }, { "name": "brianium/paratest", - "version": "v7.19.1", + "version": "v7.19.0", "source": { "type": "git", "url": "https://github.com/paratestphp/paratest.git", - "reference": "95b03194f4cdf5c83175ceead673e21cb66465e7" + "reference": "7c6c29af7c4b406b49ce0c6b0a3a81d3684474e6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/paratestphp/paratest/zipball/95b03194f4cdf5c83175ceead673e21cb66465e7", - "reference": "95b03194f4cdf5c83175ceead673e21cb66465e7", + "url": "https://api.github.com/repos/paratestphp/paratest/zipball/7c6c29af7c4b406b49ce0c6b0a3a81d3684474e6", + "reference": "7c6c29af7c4b406b49ce0c6b0a3a81d3684474e6", "shasum": "" }, "require": { @@ -5512,7 +5512,7 @@ "phpunit/php-code-coverage": "^12.5.3 || ^13.0.1", "phpunit/php-file-iterator": "^6.0.1 || ^7", "phpunit/php-timer": "^8 || ^9", - "phpunit/phpunit": "^12.5.14 || ^13.0.5", + "phpunit/phpunit": "^12.5.9 || ^13", "sebastian/environment": "^8.0.3 || ^9", "symfony/console": "^7.4.4 || ^8.0.4", "symfony/process": "^7.4.5 || ^8.0.5" @@ -5522,10 +5522,10 @@ "ext-pcntl": "*", "ext-pcov": "*", "ext-posix": "*", - "phpstan/phpstan": "^2.1.40", - "phpstan/phpstan-deprecation-rules": "^2.0.4", - "phpstan/phpstan-phpunit": "^2.0.16", - "phpstan/phpstan-strict-rules": "^2.0.10", + "phpstan/phpstan": "^2.1.38", + "phpstan/phpstan-deprecation-rules": "^2.0.3", + "phpstan/phpstan-phpunit": "^2.0.12", + "phpstan/phpstan-strict-rules": "^2.0.8", "symfony/filesystem": "^7.4.0 || ^8.0.1" }, "bin": [ @@ -5566,7 +5566,7 @@ ], "support": { "issues": "https://github.com/paratestphp/paratest/issues", - "source": "https://github.com/paratestphp/paratest/tree/v7.19.1" + "source": "https://github.com/paratestphp/paratest/tree/v7.19.0" }, "funding": [ { @@ -5578,7 +5578,7 @@ "type": "paypal" } ], - "time": "2026-02-25T14:53:45+00:00" + "time": "2026-02-06T10:53:26+00:00" }, { "name": "czproject/git-php", @@ -6398,16 +6398,16 @@ }, { "name": "phpbench/phpbench", - "version": "1.5.1", + "version": "1.4.3", "source": { "type": "git", "url": "https://github.com/phpbench/phpbench.git", - "reference": "9a28fd0833f11171b949843c6fd663eb69b6d14c" + "reference": "b641dde59d969ea42eed70a39f9b51950bc96878" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpbench/phpbench/zipball/9a28fd0833f11171b949843c6fd663eb69b6d14c", - "reference": "9a28fd0833f11171b949843c6fd663eb69b6d14c", + "url": "https://api.github.com/repos/phpbench/phpbench/zipball/b641dde59d969ea42eed70a39f9b51950bc96878", + "reference": "b641dde59d969ea42eed70a39f9b51950bc96878", "shasum": "" }, "require": { @@ -6418,7 +6418,7 @@ "ext-reflection": "*", "ext-spl": "*", "ext-tokenizer": "*", - "php": "^8.2", + "php": "^8.1", "phpbench/container": "^2.2", "psr/log": "^1.1 || ^2.0 || ^3.0", "seld/jsonlint": "^1.1", @@ -6438,9 +6438,8 @@ "phpstan/extension-installer": "^1.1", "phpstan/phpstan": "^1.0", "phpstan/phpstan-phpunit": "^1.0", - "phpunit/phpunit": "^11.5", + "phpunit/phpunit": "^10.4 || ^11.0", "rector/rector": "^1.2", - "sebastian/exporter": "^6.3.2", "symfony/error-handler": "^6.1 || ^7.0 || ^8.0", "symfony/var-dumper": "^6.1 || ^7.0 || ^8.0" }, @@ -6485,7 +6484,7 @@ ], "support": { "issues": "https://github.com/phpbench/phpbench/issues", - "source": "https://github.com/phpbench/phpbench/tree/1.5.1" + "source": "https://github.com/phpbench/phpbench/tree/1.4.3" }, "funding": [ { @@ -6493,15 +6492,15 @@ "type": "github" } ], - "time": "2026-03-05T08:18:58+00:00" + "time": "2025-11-06T19:07:31+00:00" }, { "name": "phpstan/phpstan", - "version": "1.12.33", + "version": "1.12.32", "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/37982d6fc7cbb746dda7773530cda557cdf119e1", - "reference": "37982d6fc7cbb746dda7773530cda557cdf119e1", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/2770dcdf5078d0b0d53f94317e06affe88419aa8", + "reference": "2770dcdf5078d0b0d53f94317e06affe88419aa8", "shasum": "" }, "require": { @@ -6546,7 +6545,7 @@ "type": "github" } ], - "time": "2026-02-28T20:30:03+00:00" + "time": "2025-09-30T10:16:31+00:00" }, { "name": "phpunit/php-code-coverage", @@ -8096,16 +8095,16 @@ }, { "name": "symfony/console", - "version": "v8.0.7", + "version": "v8.0.4", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "15ed9008a4ebe2d6a78e4937f74e0c13ef2e618a" + "reference": "ace03c4cf9805080ff40cbeec69fca180c339a3b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/15ed9008a4ebe2d6a78e4937f74e0c13ef2e618a", - "reference": "15ed9008a4ebe2d6a78e4937f74e0c13ef2e618a", + "url": "https://api.github.com/repos/symfony/console/zipball/ace03c4cf9805080ff40cbeec69fca180c339a3b", + "reference": "ace03c4cf9805080ff40cbeec69fca180c339a3b", "shasum": "" }, "require": { @@ -8162,7 +8161,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v8.0.7" + "source": "https://github.com/symfony/console/tree/v8.0.4" }, "funding": [ { @@ -8182,20 +8181,20 @@ "type": "tidelift" } ], - "time": "2026-03-06T14:06:22+00:00" + "time": "2026-01-13T13:06:50+00:00" }, { "name": "symfony/filesystem", - "version": "v8.0.6", + "version": "v8.0.1", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "7bf9162d7a0dff98d079b72948508fa48018a770" + "reference": "d937d400b980523dc9ee946bb69972b5e619058d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/7bf9162d7a0dff98d079b72948508fa48018a770", - "reference": "7bf9162d7a0dff98d079b72948508fa48018a770", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/d937d400b980523dc9ee946bb69972b5e619058d", + "reference": "d937d400b980523dc9ee946bb69972b5e619058d", "shasum": "" }, "require": { @@ -8232,7 +8231,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v8.0.6" + "source": "https://github.com/symfony/filesystem/tree/v8.0.1" }, "funding": [ { @@ -8252,20 +8251,20 @@ "type": "tidelift" } ], - "time": "2026-02-25T16:59:43+00:00" + "time": "2025-12-01T09:13:36+00:00" }, { "name": "symfony/finder", - "version": "v8.0.6", + "version": "v8.0.5", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "441404f09a54de6d1bd6ad219e088cdf4c91f97c" + "reference": "8bd576e97c67d45941365bf824e18dc8538e6eb0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/441404f09a54de6d1bd6ad219e088cdf4c91f97c", - "reference": "441404f09a54de6d1bd6ad219e088cdf4c91f97c", + "url": "https://api.github.com/repos/symfony/finder/zipball/8bd576e97c67d45941365bf824e18dc8538e6eb0", + "reference": "8bd576e97c67d45941365bf824e18dc8538e6eb0", "shasum": "" }, "require": { @@ -8300,7 +8299,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v8.0.6" + "source": "https://github.com/symfony/finder/tree/v8.0.5" }, "funding": [ { @@ -8320,7 +8319,7 @@ "type": "tidelift" } ], - "time": "2026-01-29T09:41:02+00:00" + "time": "2026-01-26T15:08:38+00:00" }, { "name": "symfony/options-resolver", @@ -8790,16 +8789,16 @@ }, { "name": "symfony/string", - "version": "v8.0.6", + "version": "v8.0.4", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "6c9e1108041b5dce21a9a4984b531c4923aa9ec4" + "reference": "758b372d6882506821ed666032e43020c4f57194" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/6c9e1108041b5dce21a9a4984b531c4923aa9ec4", - "reference": "6c9e1108041b5dce21a9a4984b531c4923aa9ec4", + "url": "https://api.github.com/repos/symfony/string/zipball/758b372d6882506821ed666032e43020c4f57194", + "reference": "758b372d6882506821ed666032e43020c4f57194", "shasum": "" }, "require": { @@ -8856,7 +8855,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v8.0.6" + "source": "https://github.com/symfony/string/tree/v8.0.4" }, "funding": [ { @@ -8876,7 +8875,7 @@ "type": "tidelift" } ], - "time": "2026-02-09T10:14:57+00:00" + "time": "2026-01-12T12:37:40+00:00" }, { "name": "textalk/websocket", @@ -9108,7 +9107,7 @@ ], "aliases": [], "minimum-stability": "dev", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": true, "prefer-lowest": false, "platform": { @@ -9132,5 +9131,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.9.0" + "plugin-api-version": "2.6.0" } diff --git a/src/Appwrite/Platform/Workers/Migrations.php b/src/Appwrite/Platform/Workers/Migrations.php index d87edaf788..c4cb9ce415 100644 --- a/src/Appwrite/Platform/Workers/Migrations.php +++ b/src/Appwrite/Platform/Workers/Migrations.php @@ -317,6 +317,16 @@ class Migrations extends Action 'sites.write', 'tokens.read', 'tokens.write', + 'providers.read', + 'providers.write', + 'topics.read', + 'topics.write', + 'subscribers.read', + 'subscribers.write', + 'messages.read', + 'messages.write', + 'targets.read', + 'targets.write', ] ]); diff --git a/src/Appwrite/Utopia/Response/Model/MigrationReport.php b/src/Appwrite/Utopia/Response/Model/MigrationReport.php index 7ebc22d22e..850e4b5ae9 100644 --- a/src/Appwrite/Utopia/Response/Model/MigrationReport.php +++ b/src/Appwrite/Utopia/Response/Model/MigrationReport.php @@ -59,6 +59,30 @@ class MigrationReport extends Model 'default' => 0, 'example' => 5, ]) + ->addRule(Resource::TYPE_PROVIDER, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of providers to be migrated.', + 'default' => 0, + 'example' => 5, + ]) + ->addRule(Resource::TYPE_TOPIC, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of topics to be migrated.', + 'default' => 0, + 'example' => 10, + ]) + ->addRule(Resource::TYPE_SUBSCRIBER, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of subscribers to be migrated.', + 'default' => 0, + 'example' => 100, + ]) + ->addRule(Resource::TYPE_MESSAGE, [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Number of messages to be migrated.', + 'default' => 0, + 'example' => 50, + ]) ->addRule('size', [ 'type' => self::TYPE_INTEGER, 'description' => 'Size of files to be migrated in mb.', diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 606f9e8127..d5fe7753a4 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1694,4 +1694,842 @@ trait MigrationsBase 'x-appwrite-key' => $this->getProject()['apiKey'] ]); } + + /** + * Messaging + */ + public function testAppwriteMigrationMessagingProvider(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid', + 'apiKey' => 'my-apikey', + 'from' => 'migration@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $this->assertNotEmpty($provider['body']['$id']); + + $providerId = $provider['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertEquals([Resource::TYPE_PROVIDER], $result['resources']); + $this->assertArrayHasKey(Resource::TYPE_PROVIDER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_PROVIDER]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($providerId, $response['body']['$id']); + $this->assertEquals('Migration Sendgrid', $response['body']['name']); + $this->assertEquals('email', $response['body']['type']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingProviderSMTP(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/smtp', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration SMTP', + 'host' => 'smtp.test.com', + 'port' => 587, + 'from' => 'migration-smtp@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_PROVIDER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_PROVIDER]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($providerId, $response['body']['$id']); + $this->assertEquals('Migration SMTP', $response['body']['name']); + $this->assertEquals('email', $response['body']['type']); + $this->assertEquals('smtp', $response['body']['provider']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingProviderTwilio(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/twilio', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Twilio', + 'from' => '+15551234567', + 'accountSid' => 'test-account-sid', + 'authToken' => 'test-auth-token', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_PROVIDER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_PROVIDER]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_PROVIDER]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($providerId, $response['body']['$id']); + $this->assertEquals('Migration Twilio', $response['body']['name']); + $this->assertEquals('sms', $response['body']['type']); + $this->assertEquals('twilio', $response['body']['provider']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingTopic(): void + { + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Topic', + 'apiKey' => 'my-apikey', + 'from' => 'migration-topic@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $this->assertNotEmpty($topic['body']['$id']); + + $topicId = $topic['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_TOPIC, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_TOPIC]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_TOPIC]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($topicId, $response['body']['$id']); + $this->assertEquals('Migration Topic', $response['body']['name']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingSubscriber(): void + { + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-sub@test.com', + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $this->assertEquals(1, \count($user['body']['targets'])); + $targetId = $user['body']['targets'][0]['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Subscriber', + 'apiKey' => 'my-apikey', + 'from' => uniqid() . '-migration-sub@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Subscriber Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $subscriber = $this->client->call(Client::METHOD_POST, '/messaging/topics/' . $topicId . '/subscribers', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'subscriberId' => ID::unique(), + 'targetId' => $targetId, + ]); + + $this->assertEquals(201, $subscriber['headers']['status-code']); + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_SUBSCRIBER, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_SUBSCRIBER]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($topicId, $response['body']['$id']); + $this->assertGreaterThanOrEqual(1, $response['body']['emailTotal']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingMessage(): void + { + $this->getDestinationProject(true); + + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-msg@test.com', + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $this->assertEquals(1, \count($user['body']['targets'])); + $targetId = $user['body']['targets'][0]['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Message', + 'apiKey' => 'my-apikey', + 'from' => 'migration-msg@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Message Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $message = $this->client->call(Client::METHOD_POST, '/messaging/messages/email', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'messageId' => ID::unique(), + 'targets' => [$targetId], + 'topics' => [$topicId], + 'subject' => 'Migration Test Email', + 'content' => 'This is a migration test email', + 'draft' => true, + ]); + + $this->assertEquals(201, $message['headers']['status-code']); + $this->assertNotEmpty($message['body']['$id']); + + $messageId = $message['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + Resource::TYPE_MESSAGE, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_MESSAGE, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['error']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['pending']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_MESSAGE]['success']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['processing']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['warning']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($messageId, $response['body']['$id']); + $this->assertEquals('draft', $response['body']['status']); + $this->assertEquals('Migration Test Email', $response['body']['data']['subject']); + $this->assertEquals('This is a migration test email', $response['body']['data']['content']); + $this->assertContains($topicId, $response['body']['topics']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingSmsMessage(): void + { + $this->getDestinationProject(true); + + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-sms@test.com', + 'phone' => '+1' . str_pad((string) rand(200000000, 999999999), 10, '0', STR_PAD_LEFT), + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $this->assertGreaterThanOrEqual(1, \count($user['body']['targets'])); + + $smsTarget = null; + foreach ($user['body']['targets'] as $target) { + if ($target['providerType'] === 'sms') { + $smsTarget = $target; + break; + } + } + $this->assertNotNull($smsTarget); + $targetId = $smsTarget['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/twilio', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Twilio SMS Msg', + 'from' => '+15559876543', + 'accountSid' => 'test-account-sid', + 'authToken' => 'test-auth-token', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration SMS Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $message = $this->client->call(Client::METHOD_POST, '/messaging/messages/sms', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'messageId' => ID::unique(), + 'targets' => [$targetId], + 'topics' => [$topicId], + 'content' => 'Migration SMS test content', + 'draft' => true, + ]); + + $this->assertEquals(201, $message['headers']['status-code']); + $messageId = $message['body']['$id']; + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + Resource::TYPE_MESSAGE, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_MESSAGE, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_MESSAGE]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($messageId, $response['body']['$id']); + $this->assertEquals('draft', $response['body']['status']); + $this->assertEquals('Migration SMS test content', $response['body']['data']['content']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } + + public function testAppwriteMigrationMessagingScheduledMessage(): void + { + $this->getDestinationProject(true); + + $user = $this->client->call(Client::METHOD_POST, '/users', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'userId' => ID::unique(), + 'email' => uniqid() . '-migration-sched@test.com', + 'password' => 'password', + ]); + + $this->assertEquals(201, $user['headers']['status-code']); + $userId = $user['body']['$id']; + $targetId = $user['body']['targets'][0]['$id']; + + $provider = $this->client->call(Client::METHOD_POST, '/messaging/providers/sendgrid', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'providerId' => ID::unique(), + 'name' => 'Migration Sendgrid Scheduled', + 'apiKey' => 'my-apikey', + 'from' => 'migration-sched@test.com', + ]); + + $this->assertEquals(201, $provider['headers']['status-code']); + $providerId = $provider['body']['$id']; + + $topic = $this->client->call(Client::METHOD_POST, '/messaging/topics', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'topicId' => ID::unique(), + 'name' => 'Migration Scheduled Topic', + ]); + + $this->assertEquals(201, $topic['headers']['status-code']); + $topicId = $topic['body']['$id']; + + $subscriber = $this->client->call(Client::METHOD_POST, '/messaging/topics/' . $topicId . '/subscribers', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'subscriberId' => ID::unique(), + 'targetId' => $targetId, + ]); + + $this->assertEquals(201, $subscriber['headers']['status-code']); + + // Create a scheduled message with a future date using topics only + // Direct targets use source IDs which won't resolve in the destination via API + $futureDate = (new \DateTime('+1 year'))->format(\DateTime::ATOM); + $message = $this->client->call(Client::METHOD_POST, '/messaging/messages/email', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], [ + 'messageId' => ID::unique(), + 'topics' => [$topicId], + 'subject' => 'Migration Scheduled Email', + 'content' => 'This is a scheduled migration test email', + 'scheduledAt' => $futureDate, + ]); + + $this->assertEquals(201, $message['headers']['status-code']); + $messageId = $message['body']['$id']; + $this->assertEquals('scheduled', $message['body']['status']); + + $result = $this->performMigrationSync([ + 'resources' => [ + Resource::TYPE_USER, + Resource::TYPE_PROVIDER, + Resource::TYPE_TOPIC, + Resource::TYPE_SUBSCRIBER, + Resource::TYPE_MESSAGE, + ], + 'endpoint' => $this->webEndpoint, + 'projectId' => $this->getProject()['$id'], + 'apiKey' => $this->getProject()['apiKey'], + ]); + + $this->assertEquals('completed', $result['status']); + $this->assertArrayHasKey(Resource::TYPE_MESSAGE, $result['statusCounters']); + $this->assertEquals(0, $result['statusCounters'][Resource::TYPE_MESSAGE]['error']); + $this->assertGreaterThanOrEqual(1, $result['statusCounters'][Resource::TYPE_MESSAGE]['success']); + + $response = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($messageId, $response['body']['$id']); + $this->assertEquals('scheduled', $response['body']['status']); + $this->assertEquals('Migration Scheduled Email', $response['body']['data']['subject']); + $this->assertEquals( + (new \DateTime($futureDate))->getTimestamp(), + (new \DateTime($response['body']['scheduledAt']))->getTimestamp(), + ); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/messages/' . $messageId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/topics/' . $topicId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/messaging/providers/' . $providerId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]); + + $this->client->call(Client::METHOD_DELETE, '/users/' . $userId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getDestinationProject()['$id'], + 'x-appwrite-key' => $this->getDestinationProject()['apiKey'], + ]); + } } From f4b8992cdee45ab90729ad20a8170e6f374c3753 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 10 Mar 2026 12:19:18 +0000 Subject: [PATCH 13/40] Enable SMTP keep-alive to reuse connections across mail jobs Reduces job processing time by avoiding repeated TCP connect, TLS handshake, and SMTP AUTH on every email sent. Co-Authored-By: Claude Opus 4.6 --- app/init/registers.php | 1 + src/Appwrite/Platform/Workers/Mails.php | 1 + 2 files changed, 2 insertions(+) diff --git a/app/init/registers.php b/app/init/registers.php index 26a9329270..7b68c2af9a 100644 --- a/app/init/registers.php +++ b/app/init/registers.php @@ -420,6 +420,7 @@ $register->set('smtp', function () { $mail->Password = $password; $mail->SMTPSecure = System::getEnv('_APP_SMTP_SECURE', ''); $mail->SMTPAutoTLS = false; + $mail->SMTPKeepAlive = true; $mail->CharSet = 'UTF-8'; $mail->Timeout = 10; /* Connection timeout */ $mail->getSMTPInstance()->Timelimit = 30; /* Timeout for each individual SMTP command (e.g. HELO, EHLO, etc.) */ diff --git a/src/Appwrite/Platform/Workers/Mails.php b/src/Appwrite/Platform/Workers/Mails.php index 72f7cddd06..f144c58e1b 100644 --- a/src/Appwrite/Platform/Workers/Mails.php +++ b/src/Appwrite/Platform/Workers/Mails.php @@ -214,6 +214,7 @@ class Mails extends Action $mail->Password = $password; $mail->SMTPSecure = $smtp['secure']; $mail->SMTPAutoTLS = false; + $mail->SMTPKeepAlive = true; $mail->CharSet = 'UTF-8'; $mail->Timeout = 10; /* Connection timeout */ $mail->getSMTPInstance()->Timelimit = 30; /* Timeout for each individual SMTP command (e.g. HELO, EHLO, etc.) */ From 1be142e4becf9aaf1278853f9e8da9ed4791da2f Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Tue, 10 Mar 2026 12:33:30 +0000 Subject: [PATCH 14/40] test: use database fix branch for preserveDates datetime format --- composer.json | 2 +- composer.lock | 27 ++++++++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/composer.json b/composer.json index abe51e500b..c2d2749a74 100644 --- a/composer.json +++ b/composer.json @@ -58,7 +58,7 @@ "utopia-php/compression": "0.1.*", "utopia-php/config": "1.*", "utopia-php/console": "0.1.*", - "utopia-php/database": "5.*", + "utopia-php/database": "dev-fix-preserve-dates-datetime-format as 5.3.7", "utopia-php/detector": "0.2.*", "utopia-php/domains": "1.*", "utopia-php/emails": "0.6.*", diff --git a/composer.lock b/composer.lock index 738db8f287..3caf85b8cb 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b99693284208ff3d006260a089a4f7b9", + "content-hash": "09f5b42e8589d72026a0b3731fc4f2af", "packages": [ { "name": "adhocore/jwt", @@ -3850,16 +3850,16 @@ }, { "name": "utopia-php/database", - "version": "5.3.7", + "version": "dev-fix-preserve-dates-datetime-format", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "438cc82af2981cd41ad200dd9b0df5bf00f3046a" + "reference": "49aa3181862c1370c17c2156a3b3228dca48df1e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/438cc82af2981cd41ad200dd9b0df5bf00f3046a", - "reference": "438cc82af2981cd41ad200dd9b0df5bf00f3046a", + "url": "https://api.github.com/repos/utopia-php/database/zipball/49aa3181862c1370c17c2156a3b3228dca48df1e", + "reference": "49aa3181862c1370c17c2156a3b3228dca48df1e", "shasum": "" }, "require": { @@ -3902,9 +3902,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/5.3.7" + "source": "https://github.com/utopia-php/database/tree/fix-preserve-dates-datetime-format" }, - "time": "2026-03-09T04:28:56+00:00" + "time": "2026-03-10T12:28:50+00:00" }, { "name": "utopia-php/detector", @@ -9105,9 +9105,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-fix-preserve-dates-datetime-format", + "alias": "5.3.7", + "alias_normalized": "5.3.7.0" + } + ], "minimum-stability": "dev", - "stability-flags": [], + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": true, "prefer-lowest": false, "platform": { From 33cafdfc0e8e036ddc30d4cc4ba3a20590feda72 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 10 Mar 2026 22:17:26 +0000 Subject: [PATCH 15/40] Initial plan From 1113bffb78ebf8c4caaf0e54dabf463a1951efbb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 10 Mar 2026 22:19:06 +0000 Subject: [PATCH 16/40] fix: use message timestamp for receivedAt in StatsUsage worker Co-authored-by: stnguyen90 <1477010+stnguyen90@users.noreply.github.com> --- src/Appwrite/Platform/Workers/StatsUsage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/StatsUsage.php b/src/Appwrite/Platform/Workers/StatsUsage.php index 018c192647..801790f188 100644 --- a/src/Appwrite/Platform/Workers/StatsUsage.php +++ b/src/Appwrite/Platform/Workers/StatsUsage.php @@ -160,7 +160,7 @@ class StatsUsage extends Action } $this->stats[$projectId]['project'] = $project; - $this->stats[$projectId]['receivedAt'] = DateTime::now(); + $this->stats[$projectId]['receivedAt'] = DateTime::format(new \DateTime('@' . $message->getTimestamp())); foreach ($payload['metrics'] ?? [] as $metric) { $this->keys++; if (!isset($this->stats[$projectId]['keys'][$metric['key']])) { From 4bf3c72196498987bdc1aab54ba117622c370c9b Mon Sep 17 00:00:00 2001 From: eldadfux Date: Wed, 11 Mar 2026 06:25:08 +0100 Subject: [PATCH 17/40] fix: allow users to update phone number to empty without causing duplicate errors --- app/controllers/api/users.php | 13 +++--- tests/e2e/Services/Users/UsersBase.php | 56 +++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index d0e5e19a51..9d04018b10 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1560,12 +1560,15 @@ Http::patch('/v1/users/:userId/phone') $oldPhone = $user->getAttribute('phone'); + // Store null instead of empty string so unique constraint allows multiple users without phone + $phoneValue = $number !== '' ? $number : null; + $user - ->setAttribute('phone', $number) + ->setAttribute('phone', $phoneValue) ->setAttribute('phoneVerification', false) ; - if (\strlen($number) !== 0) { + if ($number !== '') { $target = $dbForProject->findOne('targets', [ Query::equal('identifier', [$number]), ]); @@ -1577,7 +1580,7 @@ Http::patch('/v1/users/:userId/phone') try { $user = $dbForProject->updateDocument('users', $user->getId(), new Document([ - 'phone' => $user->getAttribute('phone'), + 'phone' => $phoneValue, 'phoneVerification' => $user->getAttribute('phoneVerification'), ])); /** @@ -1586,14 +1589,14 @@ Http::patch('/v1/users/:userId/phone') $oldTarget = $user->find('identifier', $oldPhone, 'targets'); if ($oldTarget instanceof Document && !$oldTarget->isEmpty()) { - if (\strlen($number) !== 0) { + if ($number !== '') { $dbForProject->updateDocument('targets', $oldTarget->getId(), new Document(['identifier' => $number])); $oldTarget->setAttribute('identifier', $number); } else { $dbForProject->deleteDocument('targets', $oldTarget->getId()); } } else { - if (\strlen($number) !== 0) { + if ($number !== '') { $target = $dbForProject->createDocument('targets', new Document([ '$permissions' => [ Permission::read(Role::user($user->getId())), diff --git a/tests/e2e/Services/Users/UsersBase.php b/tests/e2e/Services/Users/UsersBase.php index 5c7b289722..866ee591a2 100644 --- a/tests/e2e/Services/Users/UsersBase.php +++ b/tests/e2e/Services/Users/UsersBase.php @@ -1596,7 +1596,7 @@ trait UsersBase ]); $this->assertEquals($user['headers']['status-code'], 200); - $this->assertEquals($user['body']['phone'], $updatedNumber); + $this->assertEmpty($user['body']['phone'] ?? ''); $user = $this->client->call(Client::METHOD_GET, '/users/' . $data['userId'], array_merge([ 'content-type' => 'application/json', @@ -1604,7 +1604,7 @@ trait UsersBase ], $this->getHeaders())); $this->assertEquals($user['headers']['status-code'], 200); - $this->assertEquals($user['body']['phone'], $updatedNumber); + $this->assertEmpty($user['body']['phone'] ?? ''); $updatedNumber = "+910000000000"; //dummy number $user = $this->client->call(Client::METHOD_PATCH, '/users/' . $data['userId'] . '/phone', array_merge([ @@ -1648,6 +1648,58 @@ trait UsersBase static::$userNumberUpdated = true; } + public function testUpdateTwoUsersPhoneToEmpty(): void + { + $projectId = $this->getProject()['$id']; + $headers = array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders()); + + // Create two users with distinct valid phone numbers + $user1 = $this->client->call(Client::METHOD_POST, '/users', $headers, [ + 'userId' => ID::unique(), + 'email' => 'user1-phone-empty-test@appwrite.io', + 'password' => 'password', + 'name' => 'User One', + 'phone' => '+16175551201', + ]); + $this->assertEquals(201, $user1['headers']['status-code']); + $this->assertEquals('+16175551201', $user1['body']['phone']); + + $user2 = $this->client->call(Client::METHOD_POST, '/users', $headers, [ + 'userId' => ID::unique(), + 'email' => 'user2-phone-empty-test@appwrite.io', + 'password' => 'password', + 'name' => 'User Two', + 'phone' => '+16175551202', + ]); + $this->assertEquals(201, $user2['headers']['status-code']); + $this->assertEquals('+16175551202', $user2['body']['phone']); + + // Update first user's phone to empty - must succeed + $response1 = $this->client->call(Client::METHOD_PATCH, '/users/' . $user1['body']['$id'] . '/phone', $headers, [ + 'number' => '', + ]); + $this->assertEquals(200, $response1['headers']['status-code'], 'First user phone should update to empty'); + $this->assertEmpty($response1['body']['phone'] ?? ''); + + // Update second user's phone to empty - must succeed (would fail with duplicate if empty was stored as '') + $response2 = $this->client->call(Client::METHOD_PATCH, '/users/' . $user2['body']['$id'] . '/phone', $headers, [ + 'number' => '', + ]); + $this->assertEquals(200, $response2['headers']['status-code'], 'Second user phone should update to empty without duplicate error'); + $this->assertEmpty($response2['body']['phone'] ?? ''); + + // Verify both users have empty phone via GET + $get1 = $this->client->call(Client::METHOD_GET, '/users/' . $user1['body']['$id'], $headers); + $get2 = $this->client->call(Client::METHOD_GET, '/users/' . $user2['body']['$id'], $headers); + $this->assertEquals(200, $get1['headers']['status-code']); + $this->assertEquals(200, $get2['headers']['status-code']); + $this->assertEmpty($get1['body']['phone'] ?? ''); + $this->assertEmpty($get2['body']['phone'] ?? ''); + } + public function testUpdateUserNumberSearch(): void { $data = $this->ensureUserNumberUpdated(); From 0ebda89adf98c475d3ea35157be4a4a60a621aa5 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Wed, 11 Mar 2026 05:56:44 +0000 Subject: [PATCH 18/40] update utopia-php/database to released 5.3.8 --- composer.json | 2 +- composer.lock | 29 ++++++++++------------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/composer.json b/composer.json index c2d2749a74..abe51e500b 100644 --- a/composer.json +++ b/composer.json @@ -58,7 +58,7 @@ "utopia-php/compression": "0.1.*", "utopia-php/config": "1.*", "utopia-php/console": "0.1.*", - "utopia-php/database": "dev-fix-preserve-dates-datetime-format as 5.3.7", + "utopia-php/database": "5.*", "utopia-php/detector": "0.2.*", "utopia-php/domains": "1.*", "utopia-php/emails": "0.6.*", diff --git a/composer.lock b/composer.lock index 3caf85b8cb..0e25d3bc5f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "09f5b42e8589d72026a0b3731fc4f2af", + "content-hash": "b99693284208ff3d006260a089a4f7b9", "packages": [ { "name": "adhocore/jwt", @@ -3850,16 +3850,16 @@ }, { "name": "utopia-php/database", - "version": "dev-fix-preserve-dates-datetime-format", + "version": "5.3.8", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "49aa3181862c1370c17c2156a3b3228dca48df1e" + "reference": "4920bb60afb98d4bd81f4d331765716ae1d40255" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/49aa3181862c1370c17c2156a3b3228dca48df1e", - "reference": "49aa3181862c1370c17c2156a3b3228dca48df1e", + "url": "https://api.github.com/repos/utopia-php/database/zipball/4920bb60afb98d4bd81f4d331765716ae1d40255", + "reference": "4920bb60afb98d4bd81f4d331765716ae1d40255", "shasum": "" }, "require": { @@ -3902,9 +3902,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/fix-preserve-dates-datetime-format" + "source": "https://github.com/utopia-php/database/tree/5.3.8" }, - "time": "2026-03-10T12:28:50+00:00" + "time": "2026-03-11T01:03:34+00:00" }, { "name": "utopia-php/detector", @@ -9105,18 +9105,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-fix-preserve-dates-datetime-format", - "alias": "5.3.7", - "alias_normalized": "5.3.7.0" - } - ], + "aliases": [], "minimum-stability": "dev", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": {}, "prefer-stable": true, "prefer-lowest": false, "platform": { @@ -9140,5 +9131,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.9.0" } From 7d15d7e1f57dc3bc52b0c8f6b9d41b4cab12b60e Mon Sep 17 00:00:00 2001 From: Hemachandar <132386067+hmacr@users.noreply.github.com> Date: Wed, 11 Mar 2026 16:15:50 +0530 Subject: [PATCH 19/40] Rename time-travel task (#11523) * Rename time-travel task * rename --- Dockerfile | 2 +- bin/{time-travel => task-time-travel} | 0 tests/e2e/Services/Functions/FunctionsConsoleClientTest.php | 2 +- tests/e2e/Services/Sites/SitesConsoleClientTest.php | 4 ++-- 4 files changed, 4 insertions(+), 4 deletions(-) rename bin/{time-travel => task-time-travel} (100%) diff --git a/Dockerfile b/Dockerfile index 266d4501d0..210c2bc3d9 100755 --- a/Dockerfile +++ b/Dockerfile @@ -70,7 +70,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/sdks && \ chmod +x /usr/local/bin/specs && \ chmod +x /usr/local/bin/ssl && \ - chmod +x /usr/local/bin/time-travel && \ + chmod +x /usr/local/bin/task-time-travel && \ chmod +x /usr/local/bin/screenshot && \ chmod +x /usr/local/bin/test && \ chmod +x /usr/local/bin/upgrade && \ diff --git a/bin/time-travel b/bin/task-time-travel similarity index 100% rename from bin/time-travel rename to bin/task-time-travel diff --git a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php index cc18d14a8e..06044d9984 100644 --- a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php @@ -686,7 +686,7 @@ class FunctionsConsoleClientTest extends Scope $stdout = ''; $stderr = ''; - $code = Console::execute("docker exec appwrite time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); + $code = Console::execute("docker exec appwrite task-time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); $this->assertSame(0, $code, "Time-travel command failed with code $code: $stderr ($stdout)"); $stdout = ''; diff --git a/tests/e2e/Services/Sites/SitesConsoleClientTest.php b/tests/e2e/Services/Sites/SitesConsoleClientTest.php index 2a94dded5f..2e0e1a892d 100644 --- a/tests/e2e/Services/Sites/SitesConsoleClientTest.php +++ b/tests/e2e/Services/Sites/SitesConsoleClientTest.php @@ -180,12 +180,12 @@ class SitesConsoleClientTest extends Scope $stdout = ''; $stderr = ''; - $code = Console::execute("docker exec appwrite-task-maintenance time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); + $code = Console::execute("docker exec appwrite task-time-travel --projectId={$this->getProject()['$id']} --resourceType=deployment --resourceId={$deploymentIdInactiveOld} --createdAt=2020-01-01T00:00:00Z", '', $stdout, $stderr); $this->assertSame(0, $code, "Time-travel command failed with code $code: $stderr ($stdout)"); $stdout = ''; $stderr = ''; - $code = Console::execute("docker exec appwrite-task-maintenance maintenance --type=trigger", '', $stdout, $stderr); + $code = Console::execute("docker exec appwrite maintenance --type=trigger", '', $stdout, $stderr); $this->assertSame(0, $code, "Maintenance command failed with code $code: $stderr ($stdout)"); $this->assertEventually(function () use ($siteId) { From e557a5fc6e47293a82d5257e3d2672e37c946e1b Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 19:41:49 +0000 Subject: [PATCH 20/40] Add Docker Hub login and parallel image pulls to CI Login to Docker Hub in all test jobs to avoid rate limits, add `docker compose pull --quiet` to parallelize image downloads before `docker compose up`, and replace sed-based .env overrides with native GitHub Actions env fields. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 74 +++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0cff6288e2..8e937b261a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,6 +53,12 @@ jobs: with: submodules: recursive + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -97,10 +103,17 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -142,10 +155,17 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -249,12 +269,19 @@ jobs: echo "_APP_DB_PORT=5432" >> $GITHUB_ENV fi + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 env: _APP_BROWSER_HOST: http://invalid-browser/v1 run: | docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -356,10 +383,17 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -434,11 +468,19 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 + env: + _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -499,11 +541,19 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 + env: + _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -562,11 +612,19 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 + env: + _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -636,11 +694,19 @@ jobs: path: /tmp/${{ env.IMAGE }}.tar fail-on-cache-miss: true + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Load and Start Appwrite timeout-minutes: 3 + env: + _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - sed -i 's/_APP_OPTIONS_ABUSE=disabled/_APP_OPTIONS_ABUSE=enabled/' .env + docker compose pull --quiet docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." From 4e1b710503ba1ee880842a394645b1ea57aba8ee Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 19:59:23 +0000 Subject: [PATCH 21/40] Fix Docker Hub login credentials to match repo config Use `vars.DOCKERHUB_USERNAME` and `secrets.DOCKERHUB_TOKEN` to match the existing publish and release workflows. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8e937b261a..55254ec35a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -56,8 +56,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -106,8 +106,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -158,8 +158,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -272,8 +272,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -386,8 +386,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -471,8 +471,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -544,8 +544,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -615,8 +615,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 @@ -697,8 +697,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v4 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite timeout-minutes: 3 From 0b416e44753a0a7b00c869ae43e69b0cc8145876 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:01:25 +0000 Subject: [PATCH 22/40] Skip locally-built images during docker compose pull Add --ignore-buildable flag so docker compose pull skips images with build directives (appwrite-dev) instead of trying to pull them from Docker Hub. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 55254ec35a..982e19f832 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -113,7 +113,7 @@ jobs: timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -165,7 +165,7 @@ jobs: timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -281,7 +281,7 @@ jobs: _APP_BROWSER_HOST: http://invalid-browser/v1 run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -393,7 +393,7 @@ jobs: timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -480,7 +480,7 @@ jobs: _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -553,7 +553,7 @@ jobs: _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -624,7 +624,7 @@ jobs: _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." @@ -706,7 +706,7 @@ jobs: _APP_OPTIONS_ABUSE: enabled run: | docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose pull --quiet + docker compose pull --quiet --ignore-buildable docker compose up -d until docker compose exec -T appwrite doctor > /dev/null 2>&1; do echo "Waiting for Appwrite to be ready..." From d22642590f76a3903ccca6a45be98305ea3a5f3b Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:07:13 +0000 Subject: [PATCH 23/40] Add build directive to all appwrite-dev services for --ignore-buildable Add x-build YAML anchor and apply it to all services using the appwrite-dev image so docker compose pull --ignore-buildable correctly skips them instead of trying to pull from Docker Hub. Co-Authored-By: Claude Opus 4.6 --- docker-compose.yml | 60 ++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4a38757737..c744a9c5a5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,15 @@ x-logging: &x-logging max-file: "5" max-size: "10m" +x-build: &x-build + build: + context: . + target: development + args: + DEBUG: false + TESTING: true + VERSION: dev + services: traefik: image: traefik:3.6 @@ -50,15 +59,8 @@ services: appwrite: container_name: appwrite - <<: *x-logging + <<: [*x-logging, *x-build] image: appwrite-dev - build: - context: . - target: development - args: - DEBUG: false - TESTING: true - VERSION: dev ports: - 9501:80 networks: @@ -260,7 +262,7 @@ services: appwrite-realtime: entrypoint: realtime - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-realtime image: appwrite-dev restart: unless-stopped @@ -312,7 +314,7 @@ services: appwrite-worker-audits: entrypoint: worker-audits - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-audits image: appwrite-dev networks: @@ -343,7 +345,7 @@ services: appwrite-worker-webhooks: entrypoint: worker-webhooks - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-webhooks image: appwrite-dev networks: @@ -378,7 +380,7 @@ services: appwrite-worker-deletes: entrypoint: worker-deletes - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-deletes image: appwrite-dev networks: @@ -443,7 +445,7 @@ services: appwrite-worker-databases: entrypoint: worker-databases - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-databases image: appwrite-dev networks: @@ -476,7 +478,7 @@ services: appwrite-worker-builds: entrypoint: worker-builds - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-builds image: appwrite-dev networks: @@ -551,7 +553,7 @@ services: appwrite-worker-screenshots: entrypoint: worker-screenshots - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-screenshots image: appwrite-dev networks: @@ -614,7 +616,7 @@ services: appwrite-worker-certificates: entrypoint: worker-certificates - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-certificates image: appwrite-dev networks: @@ -656,7 +658,7 @@ services: appwrite-worker-executions: entrypoint: worker-executions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-executions image: appwrite-dev networks: @@ -686,7 +688,7 @@ services: appwrite-worker-functions: entrypoint: worker-functions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-functions image: appwrite-dev networks: @@ -730,7 +732,7 @@ services: appwrite-worker-mails: entrypoint: worker-mails - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-mails image: appwrite-dev networks: @@ -772,7 +774,7 @@ services: appwrite-worker-messaging: entrypoint: worker-messaging - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-messaging restart: unless-stopped image: appwrite-dev @@ -829,7 +831,7 @@ services: appwrite-worker-migrations: entrypoint: worker-migrations - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-migrations restart: unless-stopped image: appwrite-dev @@ -874,7 +876,7 @@ services: appwrite-task-maintenance: entrypoint: maintenance - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-maintenance image: appwrite-dev networks: @@ -920,7 +922,7 @@ services: appwrite-task-interval: entrypoint: interval - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-interval image: appwrite-dev networks: @@ -961,7 +963,7 @@ services: appwrite-task-stats-resources: container_name: appwrite-task-stats-resources entrypoint: stats-resources - <<: *x-logging + <<: [*x-logging, *x-build] image: appwrite-dev networks: - appwrite @@ -993,7 +995,7 @@ services: appwrite-worker-stats-resources: entrypoint: worker-stats-resources - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-stats-resources image: appwrite-dev networks: @@ -1026,7 +1028,7 @@ services: appwrite-worker-stats-usage: entrypoint: worker-stats-usage - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-worker-stats-usage image: appwrite-dev networks: @@ -1059,7 +1061,7 @@ services: appwrite-task-scheduler-functions: entrypoint: schedule-functions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-scheduler-functions image: appwrite-dev networks: @@ -1089,7 +1091,7 @@ services: appwrite-task-scheduler-executions: entrypoint: schedule-executions - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-scheduler-executions image: appwrite-dev networks: @@ -1118,7 +1120,7 @@ services: appwrite-task-scheduler-messages: entrypoint: schedule-messages - <<: *x-logging + <<: [*x-logging, *x-build] container_name: appwrite-task-scheduler-messages image: appwrite-dev networks: From 6aac7ea6ad9f25ed2e1fc15c1ace0bc682b7b169 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:17:03 +0000 Subject: [PATCH 24/40] Update GitHub Actions to latest versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - actions/cache: v4 → v5 - docker/setup-buildx-action: v3 → v4 Fixes Node.js 20 deprecation warnings. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 982e19f832..3ccf652ccb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -60,7 +60,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Build Appwrite uses: docker/build-push-action@v6 @@ -79,7 +79,7 @@ jobs: VERSION=dev - name: Cache Docker Image - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -97,7 +97,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -149,7 +149,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -243,7 +243,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -377,7 +377,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -462,7 +462,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -535,7 +535,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -606,7 +606,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar @@ -688,7 +688,7 @@ jobs: uses: actions/checkout@v6 - name: Load Cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: key: ${{ env.CACHE_KEY }} path: /tmp/${{ env.IMAGE }}.tar From 3613a645d049c5c2705c7412837a91b3f24fd938 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:19:04 +0000 Subject: [PATCH 25/40] Remove php-retry action and run test commands directly Replace itznotabug/php-retry with native run steps and timeout-minutes. Also remove pull-requests: write permission that was only needed by php-retry to post PR comments. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 108 ++++++++---------------------------- 1 file changed, 22 insertions(+), 86 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3ccf652ccb..d252c72a42 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -90,7 +90,6 @@ jobs: needs: setup permissions: contents: read - pull-requests: write steps: - name: checkout @@ -124,18 +123,11 @@ jobs: run: docker compose exec -T appwrite vars - name: Run Unit Tests - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/unit - command: >- - docker compose exec - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/unit + timeout-minutes: 15 + run: >- + docker compose exec + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/unit e2e_general_test: name: E2E General Test @@ -143,7 +135,6 @@ jobs: needs: setup permissions: contents: read - pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -181,18 +172,11 @@ jobs: done - name: Run General Tests - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/General - command: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/General --debug + timeout-minutes: 15 + run: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/General --debug - name: Failure Logs if: failure() @@ -206,7 +190,6 @@ jobs: needs: setup permissions: contents: read - pull-requests: write strategy: fail-fast: false matrix: @@ -297,15 +280,8 @@ jobs: done - name: Run ${{ matrix.service }} tests with Project table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 20 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/${{ matrix.service }} - command: | + timeout-minutes: 20 + run: | echo "Using project tables" SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" @@ -339,7 +315,6 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read - pull-requests: write strategy: fail-fast: false matrix: @@ -409,15 +384,8 @@ jobs: done - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 20 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/${{ matrix.service }} - command: | + timeout-minutes: 20 + run: | if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then echo "Using shared tables V1" export _APP_DATABASE_SHARED_TABLES=database_db_main @@ -456,7 +424,6 @@ jobs: needs: setup permissions: contents: read - pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -488,15 +455,8 @@ jobs: done - name: Run Projects tests in dedicated table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Projects - command: | + timeout-minutes: 15 + run: | echo "Using project tables" export _APP_DATABASE_SHARED_TABLES= export _APP_DATABASE_SHARED_TABLES_V1= @@ -522,7 +482,6 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read - pull-requests: write strategy: fail-fast: false matrix: @@ -561,15 +520,8 @@ jobs: done - name: Run Projects tests in ${{ matrix.tables-mode }} table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Projects - command: | + timeout-minutes: 15 + run: | if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then echo "Using shared tables V1" export _APP_DATABASE_SHARED_TABLES=database_db_main @@ -600,7 +552,6 @@ jobs: needs: setup permissions: contents: read - pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -640,15 +591,8 @@ jobs: done - name: Run Site tests with browser connected in dedicated table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Sites - command: | + timeout-minutes: 15 + run: | echo "Keeping original value of _APP_BROWSER_HOST" echo "Using project tables" export _APP_DATABASE_SHARED_TABLES= @@ -675,7 +619,6 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read - pull-requests: write strategy: fail-fast: false matrix: @@ -722,15 +665,8 @@ jobs: done - name: Run Site tests with browser connected in ${{ matrix.tables-mode }} table mode - uses: itznotabug/php-retry@v3 - with: - max_attempts: 2 - retry_wait_seconds: 300 - timeout_minutes: 15 - job_id: ${{ job.check_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - test_dir: tests/e2e/Services/Sites - command: | + timeout-minutes: 15 + run: | echo "Keeping original value of _APP_BROWSER_HOST" if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then echo "Using shared tables V1" From dc9a1c03d18bc6f3ef5f1452f9ab89ef00e9d0cb Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:20:23 +0000 Subject: [PATCH 26/40] Replace shell exports with GitHub Actions env fields Use step-level env: fields and GHA expressions for conditional values instead of shell export statements and if/elif blocks. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 103 ++++++++++++++---------------------- 1 file changed, 39 insertions(+), 64 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d252c72a42..e78a23575c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -385,17 +385,10 @@ jobs: - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode timeout-minutes: 20 + env: + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | - if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then - echo "Using shared tables V1" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1=database_db_main - elif [ "${{ matrix.tables-mode }}" == "Shared V2" ]; then - echo "Using shared tables V2" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1= - fi - SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" # Services that rely on sequential test method execution (shared static state) @@ -456,16 +449,15 @@ jobs: - name: Run Projects tests in dedicated table mode timeout-minutes: 15 - run: | - echo "Using project tables" - export _APP_DATABASE_SHARED_TABLES= - export _APP_DATABASE_SHARED_TABLES_V1= - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled + env: + _APP_DATABASE_SHARED_TABLES: "" + _APP_DATABASE_SHARED_TABLES_V1: "" + run: >- + docker compose exec -T + -e _APP_DATABASE_SHARED_TABLES + -e _APP_DATABASE_SHARED_TABLES_V1 + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled - name: Failure Logs if: failure() @@ -521,22 +513,15 @@ jobs: - name: Run Projects tests in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 - run: | - if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then - echo "Using shared tables V1" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1=database_db_main - elif [ "${{ matrix.tables-mode }}" == "Shared V2" ]; then - echo "Using shared tables V2" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1= - fi - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled + env: + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} + run: >- + docker compose exec -T + -e _APP_DATABASE_SHARED_TABLES + -e _APP_DATABASE_SHARED_TABLES_V1 + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled - name: Failure Logs if: failure() @@ -592,17 +577,15 @@ jobs: - name: Run Site tests with browser connected in dedicated table mode timeout-minutes: 15 - run: | - echo "Keeping original value of _APP_BROWSER_HOST" - echo "Using project tables" - export _APP_DATABASE_SHARED_TABLES= - export _APP_DATABASE_SHARED_TABLES_V1= - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots + env: + _APP_DATABASE_SHARED_TABLES: "" + _APP_DATABASE_SHARED_TABLES_V1: "" + run: >- + docker compose exec -T + -e _APP_DATABASE_SHARED_TABLES + -e _APP_DATABASE_SHARED_TABLES_V1 + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots - name: Failure Logs if: failure() @@ -666,23 +649,15 @@ jobs: - name: Run Site tests with browser connected in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 - run: | - echo "Keeping original value of _APP_BROWSER_HOST" - if [ "${{ matrix.tables-mode }}" == "Shared V1" ]; then - echo "Using shared tables V1" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1=database_db_main - elif [ "${{ matrix.tables-mode }}" == "Shared V2" ]; then - echo "Using shared tables V2" - export _APP_DATABASE_SHARED_TABLES=database_db_main - export _APP_DATABASE_SHARED_TABLES_V1= - fi - - docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ - appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots + env: + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} + run: >- + docker compose exec -T + -e _APP_DATABASE_SHARED_TABLES + -e _APP_DATABASE_SHARED_TABLES_V1 + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots - name: Failure Logs if: failure() From d30df31b826f24962f27089feefc99fbdb503818 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:25:47 +0000 Subject: [PATCH 27/40] Move compose-time env vars to docker compose up step _APP_DATABASE_SHARED_TABLES and _APP_DATABASE_SHARED_TABLES_V1 are read by the server at boot time via System::getEnv(), not by the test runner. Passing them via docker compose exec -e had no effect on the already-running Swoole server. Move them to the Load and Start Appwrite step so they're set at docker compose up time. Keep _APP_E2E_RESPONSE_FORMAT on exec since it's read by the test runner process (tests/e2e/Scopes/Scope.php). Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 46 +++++++++++-------------------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e78a23575c..201a47c8ba 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -262,6 +262,8 @@ jobs: timeout-minutes: 3 env: _APP_BROWSER_HOST: http://invalid-browser/v1 + _APP_DATABASE_SHARED_TABLES: "" + _APP_DATABASE_SHARED_TABLES_V1: "" run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -282,7 +284,6 @@ jobs: - name: Run ${{ matrix.service }} tests with Project table mode timeout-minutes: 20 run: | - echo "Using project tables" SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" # Services that rely on sequential test method execution (shared static state) @@ -291,14 +292,7 @@ jobs: Databases|Functions|Realtime) FUNCTIONAL_FLAG="" ;; esac - echo "Running with paratest (parallel) for: ${{ matrix.service }} ${FUNCTIONAL_FLAG:+(functional)}" docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES="" \ - -e _APP_DATABASE_SHARED_TABLES_V1="" \ - -e _APP_DB_ADAPTER="${{ env._APP_DB_ADAPTER }}" \ - -e _APP_DB_HOST="${{ env._APP_DB_HOST }}" \ - -e _APP_DB_PORT="${{ env._APP_DB_PORT }}" \ - -e _APP_DB_SCHEMA=appwrite \ -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ appwrite vendor/bin/paratest --processes $(nproc) $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --exclude-group ciIgnore --log-junit tests/e2e/Services/${{ matrix.service }}/junit.xml @@ -366,6 +360,9 @@ jobs: - name: Load and Start Appwrite timeout-minutes: 3 + env: + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -385,9 +382,6 @@ jobs: - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode timeout-minutes: 20 - env: - _APP_DATABASE_SHARED_TABLES: database_db_main - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" @@ -398,8 +392,6 @@ jobs: esac docker compose exec -T \ - -e _APP_DATABASE_SHARED_TABLES \ - -e _APP_DATABASE_SHARED_TABLES_V1 \ -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" \ appwrite vendor/bin/paratest --processes $(nproc) $FUNCTIONAL_FLAG "$SERVICE_PATH" --exclude-group abuseEnabled --exclude-group screenshots --exclude-group ciIgnore --log-junit tests/e2e/Services/${{ matrix.service }}/junit.xml @@ -438,6 +430,8 @@ jobs: timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled + _APP_DATABASE_SHARED_TABLES: "" + _APP_DATABASE_SHARED_TABLES_V1: "" run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -449,13 +443,8 @@ jobs: - name: Run Projects tests in dedicated table mode timeout-minutes: 15 - env: - _APP_DATABASE_SHARED_TABLES: "" - _APP_DATABASE_SHARED_TABLES_V1: "" run: >- docker compose exec -T - -e _APP_DATABASE_SHARED_TABLES - -e _APP_DATABASE_SHARED_TABLES_V1 -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled @@ -502,6 +491,8 @@ jobs: timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -513,13 +504,8 @@ jobs: - name: Run Projects tests in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 - env: - _APP_DATABASE_SHARED_TABLES: database_db_main - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: >- docker compose exec -T - -e _APP_DATABASE_SHARED_TABLES - -e _APP_DATABASE_SHARED_TABLES_V1 -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled @@ -558,6 +544,8 @@ jobs: timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled + _APP_DATABASE_SHARED_TABLES: "" + _APP_DATABASE_SHARED_TABLES_V1: "" run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -577,13 +565,8 @@ jobs: - name: Run Site tests with browser connected in dedicated table mode timeout-minutes: 15 - env: - _APP_DATABASE_SHARED_TABLES: "" - _APP_DATABASE_SHARED_TABLES_V1: "" run: >- docker compose exec -T - -e _APP_DATABASE_SHARED_TABLES - -e _APP_DATABASE_SHARED_TABLES_V1 -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots @@ -630,6 +613,8 @@ jobs: timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled + _APP_DATABASE_SHARED_TABLES: database_db_main + _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -649,13 +634,8 @@ jobs: - name: Run Site tests with browser connected in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 - env: - _APP_DATABASE_SHARED_TABLES: database_db_main - _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: >- docker compose exec -T - -e _APP_DATABASE_SHARED_TABLES - -e _APP_DATABASE_SHARED_TABLES_V1 -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots From 534dc55f17780a49b7d800316f7bbcb67ad5a9e7 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:35:42 +0000 Subject: [PATCH 28/40] Remove unnecessary abuse enabled from screenshot tests The screenshot tests have no abuse-related code. Abuse was only enabled on these jobs as a side effect of the original sed command applying to all jobs below e2e_service_test. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 201a47c8ba..721c25cadf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -543,7 +543,6 @@ jobs: - name: Load and Start Appwrite timeout-minutes: 3 env: - _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: "" _APP_DATABASE_SHARED_TABLES_V1: "" run: | @@ -612,7 +611,6 @@ jobs: - name: Load and Start Appwrite timeout-minutes: 3 env: - _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | From ecca0d80363fcd88030989d22c6a6729241f15e0 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 20:48:39 +0000 Subject: [PATCH 29/40] Increase Load and Start Appwrite timeout from 3 to 10 minutes The 3-minute timeout was too tight with the added docker compose pull step for downloading third-party images. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 721c25cadf..5c20ca094d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -109,7 +109,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -153,7 +153,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -259,7 +259,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_BROWSER_HOST: http://invalid-browser/v1 _APP_DATABASE_SHARED_TABLES: "" @@ -359,7 +359,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} @@ -427,7 +427,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: "" @@ -488,7 +488,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: database_db_main @@ -541,7 +541,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_DATABASE_SHARED_TABLES: "" _APP_DATABASE_SHARED_TABLES_V1: "" @@ -609,7 +609,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 10 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} From 0bbcc3f570455e241776460669a5683d74e3676a Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 21:06:44 +0000 Subject: [PATCH 30/40] Move dev tools to docker-compose.override.yml Move adminer, redis-insight, mongo-express, and graphql-explorer to an override file. CI sets COMPOSE_FILE=docker-compose.yml explicitly so these are excluded from test runs, reducing the number of images to pull from 14 to 10. Local docker compose auto-loads both files so dev workflow is unchanged. Co-Authored-By: Claude Opus 4.6 --- docker-compose.override.yml | 144 ++++++++++++++++++++++++++++++++++++ docker-compose.yml | 142 +---------------------------------- 2 files changed, 147 insertions(+), 139 deletions(-) create mode 100644 docker-compose.override.yml diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000000..29182f01d2 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,144 @@ +# Dev tools for local development only. +# This file is automatically loaded by `docker compose` alongside docker-compose.yml. +# CI sets COMPOSE_FILE=docker-compose.yml explicitly, so these services are excluded from test runs. + +services: + appwrite-mongo-express: + profiles: ["mongodb"] + image: mongo-express + container_name: appwrite-mongo-express + networks: + - appwrite + ports: + - "8082:8081" + environment: + ME_CONFIG_MONGODB_URL: "mongodb://root:${_APP_DB_ROOT_PASS}@appwrite-mongodb:27017/?replicaSet=rs0&directConnection=true" + ME_CONFIG_BASICAUTH_USERNAME: ${_APP_DB_USER} + ME_CONFIG_BASICAUTH_PASSWORD: ${_APP_DB_PASS} + depends_on: + - mongodb + + adminer: + image: adminer + container_name: appwrite-adminer + restart: always + ports: + - 9506:8080 + networks: + - appwrite + - gateway + environment: + - ADMINER_DESIGN=pepa-linha + - ADMINER_DEFAULT_SERVER=mariadb + - ADMINER_DEFAULT_USERNAME=root + - ADMINER_DEFAULT_PASSWORD=rootsecretpassword + - ADMINER_DEFAULT_DB=appwrite + configs: + - source: adminer-index.php + target: /var/www/html/index.php + mode: 0755 + labels: + - "traefik.enable=true" + - "traefik.constraint-label-stack=appwrite" + - "traefik.docker.network=gateway" + - "traefik.http.services.appwrite_adminer.loadbalancer.server.port=8080" + - "traefik.http.routers.appwrite_adminer_http.entrypoints=appwrite_web" + - "traefik.http.routers.appwrite_adminer_http.rule=Host(`mysql.localhost`)" + - "traefik.http.routers.appwrite_adminer_http.service=appwrite_adminer" + - "traefik.http.routers.appwrite_adminer_https.entrypoints=appwrite_websecure" + - "traefik.http.routers.appwrite_adminer_https.rule=Host(`mysql.localhost`)" + - "traefik.http.routers.appwrite_adminer_https.service=appwrite_adminer" + - "traefik.http.routers.appwrite_adminer_https.tls=true" + + redis-insight: + image: redis/redisinsight:latest + restart: unless-stopped + networks: + - appwrite + - gateway + environment: + - RI_PRE_SETUP_DATABASES_PATH=/mnt/connections.json + configs: + - source: redisinsight-connections.json + target: /mnt/connections.json + mode: 0755 + labels: + - "traefik.enable=true" + - "traefik.constraint-label-stack=appwrite" + - "traefik.docker.network=gateway" + - "traefik.http.services.appwrite_redisinsight.loadbalancer.server.port=5540" + - "traefik.http.routers.appwrite_redisinsight_http.entrypoints=appwrite_web" + - "traefik.http.routers.appwrite_redisinsight_http.rule=Host(`redis.localhost`)" + - "traefik.http.routers.appwrite_redisinsight_http.service=appwrite_redisinsight" + - "traefik.http.routers.appwrite_redisinsight_https.entrypoints=appwrite_websecure" + - "traefik.http.routers.appwrite_redisinsight_https.rule=Host(`redis.localhost`)" + - "traefik.http.routers.appwrite_redisinsight_https.service=appwrite_redisinsight" + - "traefik.http.routers.appwrite_redisinsight_https.tls=true" + ports: + - "8081:5540" + + graphql-explorer: + container_name: appwrite-graphql-explorer + image: appwrite/altair:0.3.0 + restart: unless-stopped + networks: + - appwrite + ports: + - "9509:3000" + environment: + - SERVER_URL=http://localhost/v1/graphql + +configs: + redisinsight-connections.json: + content: | + [ + { + "compressor": "NONE", + "id": "104dc90a-21ef-4d5e-8912-b30baabb152f", + "host": "redis", + "port": 6379, + "name": "redis:6379", + "db": 0, + "username": "default", + "password": null, + "connectionType": "STANDALONE", + "nameFromProvider": null, + "provider": "REDIS", + "lastConnection": "2025-10-16T09:22:02.591Z", + "modules": [ + { + "name": "ReJSON", + "version": 20808, + "semanticVersion": "2.8.8" + }, + { + "name": "search", + "version": 21015, + "semanticVersion": "2.10.15" + } + ], + "tls": false, + "tlsServername": null, + "verifyServerCert": null, + "caCert": null, + "clientCert": null, + "ssh": false, + "sshOptions": null, + "forceStandalone": false, + "tags": [] + } + ] + + adminer-index.php: + content: | + $$_ENV['ADMINER_DEFAULT_SERVER'], + 'driver' => 'server', /* seems to autodetect the driver from server settings */ + 'username' => $$_ENV['ADMINER_DEFAULT_USERNAME'], + 'password' => $$_ENV['ADMINER_DEFAULT_PASSWORD'], + 'db' => $$_ENV['ADMINER_DEFAULT_DB'], + ]; + } + include './adminer.php'; diff --git a/docker-compose.yml b/docker-compose.yml index c744a9c5a5..0013fdfa64 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1277,20 +1277,7 @@ services: retries: 10 start_period: 30s - appwrite-mongo-express: - profiles: ["mongodb"] - image: mongo-express - container_name: appwrite-mongo-express - networks: - - appwrite - ports: - - "8082:8081" - environment: - ME_CONFIG_MONGODB_URL: "mongodb://root:${_APP_DB_ROOT_PASS}@appwrite-mongodb:27017/?replicaSet=rs0&directConnection=true" - ME_CONFIG_BASICAUTH_USERNAME: ${_APP_DB_USER} - ME_CONFIG_BASICAUTH_PASSWORD: ${_APP_DB_PASS} - depends_on: - - mongodb + postgresql: profiles: ["postgresql"] @@ -1398,78 +1385,8 @@ services: networks: - appwrite - adminer: - image: adminer - container_name: appwrite-adminer - <<: *x-logging - restart: always - ports: - - 9506:8080 - networks: - - appwrite - - gateway - environment: - - ADMINER_DESIGN=pepa-linha - - ADMINER_DEFAULT_SERVER=mariadb - - ADMINER_DEFAULT_USERNAME=root - - ADMINER_DEFAULT_PASSWORD=rootsecretpassword - - ADMINER_DEFAULT_DB=appwrite - configs: - - source: adminer-index.php - target: /var/www/html/index.php - mode: 0755 - labels: - - "traefik.enable=true" - - "traefik.constraint-label-stack=appwrite" - - "traefik.docker.network=gateway" - - "traefik.http.services.appwrite_adminer.loadbalancer.server.port=8080" - - "traefik.http.routers.appwrite_adminer_http.entrypoints=appwrite_web" - - "traefik.http.routers.appwrite_adminer_http.rule=Host(`mysql.localhost`)" - - "traefik.http.routers.appwrite_adminer_http.service=appwrite_adminer" - - "traefik.http.routers.appwrite_adminer_https.entrypoints=appwrite_websecure" - - "traefik.http.routers.appwrite_adminer_https.rule=Host(`mysql.localhost`)" - - "traefik.http.routers.appwrite_adminer_https.service=appwrite_adminer" - - "traefik.http.routers.appwrite_adminer_https.tls=true" - - redis-insight: - image: redis/redisinsight:latest - restart: unless-stopped - networks: - - appwrite - - gateway - environment: - - RI_PRE_SETUP_DATABASES_PATH=/mnt/connections.json - configs: - - source: redisinsight-connections.json - target: /mnt/connections.json - mode: 0755 - labels: - - "traefik.enable=true" - - "traefik.constraint-label-stack=appwrite" - - "traefik.docker.network=gateway" - - "traefik.http.services.appwrite_redisinsight.loadbalancer.server.port=5540" - - "traefik.http.routers.appwrite_redisinsight_http.entrypoints=appwrite_web" - - "traefik.http.routers.appwrite_redisinsight_http.rule=Host(`redis.localhost`)" - - "traefik.http.routers.appwrite_redisinsight_http.service=appwrite_redisinsight" - - "traefik.http.routers.appwrite_redisinsight_https.entrypoints=appwrite_websecure" - - "traefik.http.routers.appwrite_redisinsight_https.rule=Host(`redis.localhost`)" - - "traefik.http.routers.appwrite_redisinsight_https.service=appwrite_redisinsight" - - "traefik.http.routers.appwrite_redisinsight_https.tls=true" - ports: - - "8081:5540" - - graphql-explorer: - container_name: appwrite-graphql-explorer - image: appwrite/altair:0.3.0 - restart: unless-stopped - networks: - - appwrite - ports: - - "9509:3000" - environment: - - SERVER_URL=http://localhost/v1/graphql - - # Dev Tools End ------------------------------------------------------------------------------------------ + # Dev tools (adminer, redis-insight, mongo-express, graphql-explorer) + # are defined in docker-compose.override.yml networks: gateway: @@ -1482,60 +1399,7 @@ networks: runtimes: name: runtimes -configs: - redisinsight-connections.json: - content: | - [ - { - "compressor": "NONE", - "id": "104dc90a-21ef-4d5e-8912-b30baabb152f", - "host": "redis", - "port": 6379, - "name": "redis:6379", - "db": 0, - "username": "default", - "password": null, - "connectionType": "STANDALONE", - "nameFromProvider": null, - "provider": "REDIS", - "lastConnection": "2025-10-16T09:22:02.591Z", - "modules": [ - { - "name": "ReJSON", - "version": 20808, - "semanticVersion": "2.8.8" - }, - { - "name": "search", - "version": 21015, - "semanticVersion": "2.10.15" - } - ], - "tls": false, - "tlsServername": null, - "verifyServerCert": null, - "caCert": null, - "clientCert": null, - "ssh": false, - "sshOptions": null, - "forceStandalone": false, - "tags": [] - } - ] - adminer-index.php: - content: | - $$_ENV['ADMINER_DEFAULT_SERVER'], - 'driver' => 'server', /* seems to autodetect the driver from server settings */ - 'username' => $$_ENV['ADMINER_DEFAULT_USERNAME'], - 'password' => $$_ENV['ADMINER_DEFAULT_PASSWORD'], - 'db' => $$_ENV['ADMINER_DEFAULT_DB'], - ]; - } - include './adminer.php'; volumes: appwrite-mariadb: From 33ce469ab0206b161b60983fabca86d666fdc839 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 21:44:15 +0000 Subject: [PATCH 31/40] Add Docker healthcheck and use --wait instead of polling loop Replace the manual shell polling loop (until doctor > /dev/null) with a proper Docker healthcheck on the appwrite service and `docker compose up --wait`, which blocks until healthchecks pass. Also reverts the timeout back to 3 minutes now that image pulls are cached. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 64 ++++++++++--------------------------- docker-compose.yml | 5 +++ 2 files changed, 21 insertions(+), 48 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5c20ca094d..cebcd2a62c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -109,15 +109,11 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Environment Variables run: docker compose exec -T appwrite vars @@ -153,15 +149,11 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Wait for Open Runtimes timeout-minutes: 3 @@ -259,7 +251,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_BROWSER_HOST: http://invalid-browser/v1 _APP_DATABASE_SHARED_TABLES: "" @@ -267,11 +259,7 @@ jobs: run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Wait for Open Runtimes timeout-minutes: 3 @@ -359,18 +347,14 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Wait for Open Runtimes timeout-minutes: 3 @@ -427,7 +411,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: "" @@ -435,11 +419,7 @@ jobs: run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Run Projects tests in dedicated table mode timeout-minutes: 15 @@ -488,7 +468,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: database_db_main @@ -496,11 +476,7 @@ jobs: run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Run Projects tests in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 @@ -541,18 +517,14 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_DATABASE_SHARED_TABLES: "" _APP_DATABASE_SHARED_TABLES_V1: "" run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Wait for Open Runtimes timeout-minutes: 3 @@ -609,18 +581,14 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 10 + timeout-minutes: 3 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable - docker compose up -d - until docker compose exec -T appwrite doctor > /dev/null 2>&1; do - echo "Waiting for Appwrite to be ready..." - sleep 2 - done + docker compose up -d --quiet-pull --wait - name: Wait for Open Runtimes timeout-minutes: 3 diff --git a/docker-compose.yml b/docker-compose.yml index 0013fdfa64..583b558e60 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,6 +61,11 @@ services: container_name: appwrite <<: [*x-logging, *x-build] image: appwrite-dev + healthcheck: + test: ["CMD", "doctor"] + interval: 5s + timeout: 5s + retries: 12 ports: - 9501:80 networks: From 38798c899396ee48b3151185ecaa632495c3f939 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 21:58:30 +0000 Subject: [PATCH 32/40] Add healthchecks for MariaDB, PostgreSQL, and Redis Add proper healthchecks to infrastructure services and use condition: service_healthy for redis in appwrite's depends_on so it waits for Redis to be ready before starting. Co-Authored-By: Claude Opus 4.6 --- docker-compose.yml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 583b558e60..c0b0560a7f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,10 +108,10 @@ services: - ./dev:/usr/src/code/dev depends_on: - - ${_APP_DB_HOST:-mongodb} - - redis - - coredns - # - clamav + redis: + condition: service_healthy + coredns: + condition: service_started entrypoint: - php - -e @@ -1244,6 +1244,11 @@ services: - MYSQL_PASSWORD=${_APP_DB_PASS} - MARIADB_AUTO_UPGRADE=1 command: "mysqld --innodb-flush-method=fsync" + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 5s + timeout: 5s + retries: 12 mongodb: profiles: ["mongodb"] @@ -1303,6 +1308,11 @@ services: - POSTGRES_USER=${_APP_DB_USER} - POSTGRES_PASSWORD=${_APP_DB_PASS} command: "postgres" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${_APP_DB_USER}"] + interval: 5s + timeout: 5s + retries: 12 redis: image: redis:7.4.7-alpine @@ -1319,6 +1329,11 @@ services: - appwrite volumes: - appwrite-redis:/data:rw + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 12 coredns: # DNS server for testing purposes (Proxy APIs) image: coredns/coredns:1.12.4 From fd28ad8a6681c9cf8d678cc80c4c29bfc9e5b00d Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 21:59:33 +0000 Subject: [PATCH 33/40] Remove --debug flag from test commands for quieter CI output Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cebcd2a62c..39c9aee9ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -168,8 +168,7 @@ jobs: run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/General --debug - + appwrite test /usr/src/code/tests/e2e/General - name: Failure Logs if: failure() run: | From 16929bc42074ef739151d4021aa9d92ee91082f7 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 22:00:00 +0000 Subject: [PATCH 34/40] Remove remaining --debug flags from test commands Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 39c9aee9ca..879b72327d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -425,7 +425,7 @@ jobs: run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled + appwrite test /usr/src/code/tests/e2e/Services/Projects --group=abuseEnabled - name: Failure Logs if: failure() @@ -482,7 +482,7 @@ jobs: run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Projects --debug --group=abuseEnabled + appwrite test /usr/src/code/tests/e2e/Services/Projects --group=abuseEnabled - name: Failure Logs if: failure() @@ -538,7 +538,7 @@ jobs: run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots + appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots - name: Failure Logs if: failure() @@ -602,7 +602,7 @@ jobs: run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Sites --debug --group=screenshots + appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots - name: Failure Logs if: failure() From 4cbe50193a163f3e9e44e97c520a46a200515e5f Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 22:24:52 +0000 Subject: [PATCH 35/40] Increase Load and Start Appwrite timeout to 5 minutes The docker compose pull step alone can take over 2.5 minutes on CI (e.g. openruntimes-executor, traefik), leaving no time for docker compose up --wait within 3 minutes. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 879b72327d..73c136bb93 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -109,7 +109,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable @@ -149,7 +149,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose pull --quiet --ignore-buildable From 7dfe44cb363a8a4350e4339fbc504f794143959f Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 22:44:42 +0000 Subject: [PATCH 36/40] Run abuse-enabled tests across entire test suite, not just Projects The abuseEnabled jobs previously only ran tests in Services/Projects, missing the Account abuse test and any future abuseEnabled tests in other services. Also rename jobs to "Abuse" for consistency. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 73c136bb93..c13370f3f0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -387,7 +387,7 @@ jobs: docker compose logs openruntimes-executor e2e_abuse_enabled: - name: E2E Service Test (Abuse enabled) + name: E2E Service Test (Abuse) runs-on: ubuntu-latest needs: setup permissions: @@ -420,12 +420,12 @@ jobs: docker compose pull --quiet --ignore-buildable docker compose up -d --quiet-pull --wait - - name: Run Projects tests in dedicated table mode + - name: Run abuse-enabled tests in dedicated table mode timeout-minutes: 15 run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Projects --group=abuseEnabled + appwrite test /usr/src/code/tests/e2e --group=abuseEnabled - name: Failure Logs if: failure() @@ -436,7 +436,7 @@ jobs: docker compose logs openruntimes-executor e2e_abuse_enabled_shared_mode: - name: E2E Shared Mode Service Test (Abuse enabled) + name: E2E Shared Mode Service Test (Abuse) runs-on: ubuntu-latest needs: [ setup, check_database_changes ] if: needs.check_database_changes.outputs.database_changed == 'true' @@ -477,12 +477,12 @@ jobs: docker compose pull --quiet --ignore-buildable docker compose up -d --quiet-pull --wait - - name: Run Projects tests in ${{ matrix.tables-mode }} table mode + - name: Run abuse-enabled tests in ${{ matrix.tables-mode }} table mode timeout-minutes: 15 run: >- docker compose exec -T -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Projects --group=abuseEnabled + appwrite test /usr/src/code/tests/e2e --group=abuseEnabled - name: Failure Logs if: failure() From 89419344c2b3f045d5ce8c048eb1dd2a7782c4de Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 23:20:51 +0000 Subject: [PATCH 37/40] Restore MariaDB defaults, fix remaining timeouts, filter listener span logs - Change .env defaults back from MongoDB to MariaDB - Bump all remaining "Load and Start Appwrite" timeouts from 3 to 5 minutes - Filter listener.* span logs to only export on error Co-Authored-By: Claude Opus 4.6 --- .env | 8 ++++---- .github/workflows/tests.yml | 12 ++++++------ app/init/span.php | 7 ++++++- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.env b/.env index 0df9cb42f4..3c8bef5b68 100644 --- a/.env +++ b/.env @@ -39,10 +39,10 @@ _APP_REDIS_HOST=redis _APP_REDIS_PORT=6379 _APP_REDIS_PASS= _APP_REDIS_USER= -COMPOSE_PROFILES=mongodb -_APP_DB_ADAPTER=mongodb -_APP_DB_HOST=mongodb -_APP_DB_PORT=27017 +COMPOSE_PROFILES=mariadb +_APP_DB_ADAPTER=mariadb +_APP_DB_HOST=mariadb +_APP_DB_PORT=3306 _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c13370f3f0..9f2556f3e5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -250,7 +250,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_BROWSER_HOST: http://invalid-browser/v1 _APP_DATABASE_SHARED_TABLES: "" @@ -346,7 +346,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} @@ -410,7 +410,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: "" @@ -467,7 +467,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_OPTIONS_ABUSE: enabled _APP_DATABASE_SHARED_TABLES: database_db_main @@ -516,7 +516,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_DATABASE_SHARED_TABLES: "" _APP_DATABASE_SHARED_TABLES_V1: "" @@ -580,7 +580,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Load and Start Appwrite - timeout-minutes: 3 + timeout-minutes: 5 env: _APP_DATABASE_SHARED_TABLES: database_db_main _APP_DATABASE_SHARED_TABLES_V1: ${{ matrix.tables-mode == 'Shared V1' && 'database_db_main' || '' }} diff --git a/app/init/span.php b/app/init/span.php index 76f37f5300..8afa01b2df 100644 --- a/app/init/span.php +++ b/app/init/span.php @@ -5,4 +5,9 @@ use Utopia\Span\Span; use Utopia\Span\Storage; Span::setStorage(new Storage\Coroutine()); -Span::addExporter(new Exporter\Pretty()); +Span::addExporter(new Exporter\Pretty(), function (Span $span): bool { + if (\str_starts_with($span->getAction(), 'listener.')) { + return $span->getError() !== null; + } + return true; +}); From 8cb36835cc4eaa92345bae1b2413e7aa89b11d41 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 23:41:26 +0000 Subject: [PATCH 38/40] Restore php-retry action for flaky test resilience Re-add itznotabug/php-retry@v3 wrapping all test steps with max_attempts: 2 and retry_wait_seconds: 300. Also restore pull-requests: write permission needed by the action. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 135 ++++++++++++++++++++++++++---------- 1 file changed, 100 insertions(+), 35 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9f2556f3e5..20d3c71a9a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -90,6 +90,7 @@ jobs: needs: setup permissions: contents: read + pull-requests: write steps: - name: checkout @@ -119,11 +120,18 @@ jobs: run: docker compose exec -T appwrite vars - name: Run Unit Tests - timeout-minutes: 15 - run: >- - docker compose exec - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/unit + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/unit + command: >- + docker compose exec + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/unit e2e_general_test: name: E2E General Test @@ -131,6 +139,7 @@ jobs: needs: setup permissions: contents: read + pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -164,11 +173,19 @@ jobs: done - name: Run General Tests - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/General + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/General + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/General + - name: Failure Logs if: failure() run: | @@ -181,6 +198,7 @@ jobs: needs: setup permissions: contents: read + pull-requests: write strategy: fail-fast: false matrix: @@ -228,7 +246,7 @@ jobs: run: | DB_ADAPTER_LOWER=$(echo "${{ matrix.db_adapter }}" | tr 'A-Z' 'a-z') echo "COMPOSE_PROFILES=${DB_ADAPTER_LOWER}" >> $GITHUB_ENV - + if [ "${{ matrix.db_adapter }}" = "MARIADB" ]; then echo "_APP_DB_ADAPTER=mariadb" >> $GITHUB_ENV echo "_APP_DB_HOST=mariadb" >> $GITHUB_ENV @@ -269,8 +287,15 @@ jobs: done - name: Run ${{ matrix.service }} tests with Project table mode - timeout-minutes: 20 - run: | + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 20 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/Services/${{ matrix.service }} + command: | SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" # Services that rely on sequential test method execution (shared static state) @@ -296,6 +321,7 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read + pull-requests: write strategy: fail-fast: false matrix: @@ -364,8 +390,15 @@ jobs: done - name: Run ${{ matrix.service }} tests with ${{ matrix.tables-mode }} table mode - timeout-minutes: 20 - run: | + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 20 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/Services/${{ matrix.service }} + command: | SERVICE_PATH="/usr/src/code/tests/e2e/Services/${{ matrix.service }}" # Services that rely on sequential test method execution (shared static state) @@ -392,6 +425,7 @@ jobs: needs: setup permissions: contents: read + pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -421,11 +455,18 @@ jobs: docker compose up -d --quiet-pull --wait - name: Run abuse-enabled tests in dedicated table mode - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e --group=abuseEnabled + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e --group=abuseEnabled - name: Failure Logs if: failure() @@ -442,6 +483,7 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read + pull-requests: write strategy: fail-fast: false matrix: @@ -478,11 +520,18 @@ jobs: docker compose up -d --quiet-pull --wait - name: Run abuse-enabled tests in ${{ matrix.tables-mode }} table mode - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e --group=abuseEnabled + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e --group=abuseEnabled - name: Failure Logs if: failure() @@ -498,6 +547,7 @@ jobs: needs: setup permissions: contents: read + pull-requests: write steps: - name: checkout uses: actions/checkout@v6 @@ -534,11 +584,18 @@ jobs: done - name: Run Site tests with browser connected in dedicated table mode - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/Services/Sites + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots - name: Failure Logs if: failure() @@ -555,6 +612,7 @@ jobs: if: needs.check_database_changes.outputs.database_changed == 'true' permissions: contents: read + pull-requests: write strategy: fail-fast: false matrix: @@ -598,11 +656,18 @@ jobs: done - name: Run Site tests with browser connected in ${{ matrix.tables-mode }} table mode - timeout-minutes: 15 - run: >- - docker compose exec -T - -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" - appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots + uses: itznotabug/php-retry@v3 + with: + max_attempts: 2 + retry_wait_seconds: 300 + timeout_minutes: 15 + job_id: ${{ job.check_run_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + test_dir: tests/e2e/Services/Sites + command: >- + docker compose exec -T + -e _APP_E2E_RESPONSE_FORMAT="${{ github.event.inputs.response_format }}" + appwrite test /usr/src/code/tests/e2e/Services/Sites --group=screenshots - name: Failure Logs if: failure() From 9f4ba3a4a285042b8e79327be96d81a0ac51db09 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 23:47:03 +0000 Subject: [PATCH 39/40] Reduce php-retry wait time from 300s to 60s Co-Authored-By: Claude Opus 4.6 --- .github/workflows/tests.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 20d3c71a9a..1b112eebbd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -123,7 +123,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -176,7 +176,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -290,7 +290,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 20 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -393,7 +393,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 20 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -458,7 +458,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -523,7 +523,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -587,7 +587,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} @@ -659,7 +659,7 @@ jobs: uses: itznotabug/php-retry@v3 with: max_attempts: 2 - retry_wait_seconds: 300 + retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} From f09b9a994eaefed1ec506db792facc823d707858 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 11 Mar 2026 23:59:31 +0000 Subject: [PATCH 40/40] Revert .env defaults back to MongoDB Co-Authored-By: Claude Opus 4.6 --- .env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 3c8bef5b68..0df9cb42f4 100644 --- a/.env +++ b/.env @@ -39,10 +39,10 @@ _APP_REDIS_HOST=redis _APP_REDIS_PORT=6379 _APP_REDIS_PASS= _APP_REDIS_USER= -COMPOSE_PROFILES=mariadb -_APP_DB_ADAPTER=mariadb -_APP_DB_HOST=mariadb -_APP_DB_PORT=3306 +COMPOSE_PROFILES=mongodb +_APP_DB_ADAPTER=mongodb +_APP_DB_HOST=mongodb +_APP_DB_PORT=27017 _APP_DB_SCHEMA=appwrite _APP_DB_USER=user _APP_DB_PASS=password