From 618423df3eeefd7dfd8949a2868505c35b0e5f21 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 2 Jul 2026 08:56:05 +0530 Subject: [PATCH 1/5] chore(ci): optimize runner costs with spot instances and leaner matrix - Run all RunsOn jobs on spot instances (price-capacity-optimized) - Limit push-to-main runs to the MongoDB/dedicated matrix; keep the full database/mode sweep for workflow_dispatch and PRs that change the utopia-php/database version - Reduce runner volumes from 120GB to 60GB (~2x measured peak usage) --- .github/workflows/ci.yml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2eb2b4580e..88d995ab18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,8 +188,8 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const runner4 = `runs-on=${context.runId}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel+ecr-cache`; - const runner8 = `runs-on=${context.runId}/runner=8cpu-linux-x64/volume=120g/spot=false/extras=otel+ecr-cache`; + const runner4 = `runs-on=${context.runId}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache`; + const runner8 = `runs-on=${context.runId}/runner=8cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache`; const services = [ { name: 'Account', runner: runner4, functional: true, processes: null, timeout: 15 }, @@ -242,7 +242,10 @@ jobs: return getDbVersion(decode(base.content)) !== getDbVersion(decode(head.content)); }; - if (!pr || await isDatabaseVersionChanged()) { + const fullMatrix = context.eventName === 'workflow_dispatch' + || (pr && await isDatabaseVersionChanged()); + + if (fullMatrix) { core.setOutput('databases', JSON.stringify(['MariaDB', 'PostgreSQL', 'MongoDB'])); core.setOutput('modes', JSON.stringify(['dedicated', 'shared'])); return; @@ -253,7 +256,7 @@ jobs: build: name: Build - runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel+s3-cache+ecr-cache + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+s3-cache+ecr-cache steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -285,7 +288,7 @@ jobs: unit: name: Tests / Unit - runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel+ecr-cache + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache needs: build permissions: contents: read @@ -348,7 +351,7 @@ jobs: e2e_general: name: Tests / E2E / General - runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel+ecr-cache + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache needs: build permissions: contents: read @@ -542,7 +545,7 @@ jobs: e2e_abuse: name: Tests / E2E / Abuse (${{ matrix.mode }}) - runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel+ecr-cache + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache needs: [build, matrix] permissions: contents: read @@ -619,7 +622,7 @@ jobs: e2e_screenshots: name: Tests / E2E / Screenshots (${{ matrix.mode }}) - runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel+ecr-cache + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache needs: [build, matrix] permissions: contents: read @@ -702,7 +705,7 @@ jobs: benchmark: name: Benchmark if: github.event_name == 'pull_request' || github.event_name == 'push' - runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel+ecr-cache + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache needs: build permissions: actions: read From 1ee03ff5ad4bc6e42792d69a6179cc6b8aa01772 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 2 Jul 2026 09:30:09 +0530 Subject: [PATCH 2/5] chore(ci): add job-level timeouts to RunsOn jobs Step-level timeouts don't bound the php-retry attempts (a Sites job ran 28 minutes); a job-level timeout-minutes is enforced by GitHub itself. 15 minutes for test jobs, 20 for benchmark, 30 for build (cold cache). --- .github/workflows/ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88d995ab18..2ab8a2a248 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -257,6 +257,7 @@ jobs: build: name: Build runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+s3-cache+ecr-cache + timeout-minutes: 30 steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -289,6 +290,7 @@ jobs: unit: name: Tests / Unit runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache + timeout-minutes: 15 needs: build permissions: contents: read @@ -352,6 +354,7 @@ jobs: e2e_general: name: Tests / E2E / General runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache + timeout-minutes: 15 needs: build permissions: contents: read @@ -426,6 +429,7 @@ jobs: e2e_service: name: Tests / E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service.name }} runs-on: ${{ matrix.service.runner }} + timeout-minutes: 15 needs: [build, matrix] permissions: contents: read @@ -546,6 +550,7 @@ jobs: e2e_abuse: name: Tests / E2E / Abuse (${{ matrix.mode }}) runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache + timeout-minutes: 15 needs: [build, matrix] permissions: contents: read @@ -623,6 +628,7 @@ jobs: e2e_screenshots: name: Tests / E2E / Screenshots (${{ matrix.mode }}) runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache + timeout-minutes: 15 needs: [build, matrix] permissions: contents: read @@ -706,6 +712,7 @@ jobs: name: Benchmark if: github.event_name == 'pull_request' || github.event_name == 'push' runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache + timeout-minutes: 20 needs: build permissions: actions: read From 3a1373ebbb56e25cc67567de247c332b9df68f92 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 2 Jul 2026 09:51:12 +0530 Subject: [PATCH 3/5] chore(ci): let per-attempt timeouts fire before the job timeout A job-level timeout reports the job as cancelled and skips the Failure Logs step. Attempt timeouts (6 min for matrix services, 5 min elsewhere) fail the step first, so hangs surface as real failures with compose logs; the 15-minute job timeout remains as a backstop. --- .github/workflows/ci.yml | 62 ++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ab8a2a248..3861e1f9d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,33 +192,33 @@ jobs: const runner8 = `runs-on=${context.runId}/runner=8cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache`; const services = [ - { name: 'Account', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Avatars', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Console', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Databases', runner: runner8, functional: false, processes: 3, timeout: 15 }, - { name: 'TablesDB', runner: runner8, functional: false, processes: 3, timeout: 15 }, - { name: 'Functions', runner: runner4, functional: false, processes: null, timeout: 15 }, - { name: 'FunctionsSchedule', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'GraphQL', runner: runner4, functional: false, processes: null, timeout: 15 }, - { name: 'Health', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Advisor', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Locale', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Projects', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Realtime', runner: runner4, functional: false, processes: null, timeout: 15 }, - { name: 'Sites', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Proxy', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Storage', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Tokens', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Teams', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Users', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'ProjectWebhooks', runner: runner4, functional: false, processes: null, timeout: 15 }, - { name: 'Webhooks', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'VCS', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Messaging', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Notifications', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Migrations', runner: runner4, functional: true, processes: 1, timeout: 15 }, - { name: 'Project', runner: runner4, functional: true, processes: null, timeout: 15 }, - { name: 'Presences', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Account', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Avatars', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Console', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Databases', runner: runner8, functional: false, processes: 3, timeout: 6 }, + { name: 'TablesDB', runner: runner8, functional: false, processes: 3, timeout: 6 }, + { name: 'Functions', runner: runner4, functional: false, processes: null, timeout: 6 }, + { name: 'FunctionsSchedule', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'GraphQL', runner: runner4, functional: false, processes: null, timeout: 6 }, + { name: 'Health', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Advisor', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Locale', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Projects', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Realtime', runner: runner4, functional: false, processes: null, timeout: 6 }, + { name: 'Sites', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Proxy', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Storage', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Tokens', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Teams', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Users', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'ProjectWebhooks', runner: runner4, functional: false, processes: null, timeout: 6 }, + { name: 'Webhooks', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'VCS', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Messaging', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Notifications', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Migrations', runner: runner4, functional: true, processes: 1, timeout: 6 }, + { name: 'Project', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Presences', runner: runner4, functional: true, processes: null, timeout: 6 }, ]; core.setOutput('services', JSON.stringify(services)); @@ -342,7 +342,7 @@ jobs: with: max_attempts: ${{ github.event_name == 'pull_request' && 2 || 1 }} retry_wait_seconds: 60 - timeout_minutes: 15 + timeout_minutes: 5 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} test_dir: tests/unit @@ -411,7 +411,7 @@ jobs: with: max_attempts: ${{ github.event_name == 'pull_request' && 2 || 1 }} retry_wait_seconds: 60 - timeout_minutes: 15 + timeout_minutes: 5 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} test_dir: tests/e2e/General @@ -610,7 +610,7 @@ jobs: with: max_attempts: ${{ github.event_name == 'pull_request' && 2 || 1 }} retry_wait_seconds: 60 - timeout_minutes: 15 + timeout_minutes: 5 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} test_dir: tests/e2e @@ -693,7 +693,7 @@ jobs: with: max_attempts: ${{ github.event_name == 'pull_request' && 2 || 1 }} retry_wait_seconds: 60 - timeout_minutes: 15 + timeout_minutes: 5 job_id: ${{ job.check_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} test_dir: tests/e2e/Services/Sites From 6ff130d6f64e817caeb0f5e1956be75f218bed63 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 2 Jul 2026 09:57:41 +0530 Subject: [PATCH 4/5] (ci): timestamp failure logs Compose logs without timestamps can't show when a worker processed a job relative to a test's polling window, which blocked debugging the intermittent TablesDB schema-polling hangs. --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3861e1f9d3..84ed743952 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -424,7 +424,7 @@ jobs: if: failure() run: | echo "=== Appwrite Logs ===" - docker compose logs + docker compose logs --timestamps e2e_service: name: Tests / E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service.name }} @@ -545,7 +545,7 @@ jobs: if: failure() run: | echo "=== Appwrite Logs ===" - docker compose logs + docker compose logs --timestamps e2e_abuse: name: Tests / E2E / Abuse (${{ matrix.mode }}) @@ -623,7 +623,7 @@ jobs: if: failure() run: | echo "=== Appwrite Logs ===" - docker compose logs + docker compose logs --timestamps e2e_screenshots: name: Tests / E2E / Screenshots (${{ matrix.mode }}) @@ -706,7 +706,7 @@ jobs: if: failure() run: | echo "=== Appwrite Logs ===" - docker compose logs + docker compose logs --timestamps benchmark: name: Benchmark From e48d42e3fc6d927f433a40036df1d187d0876865 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Thu, 2 Jul 2026 10:02:52 +0530 Subject: [PATCH 5/5] (ci): give matrix test attempts 8 minutes of headroom Project's test run alone takes ~6.5 minutes, so a 6-minute attempt cap would kill legitimate runs. 8 minutes per attempt with a 20-minute job timeout keeps a full retry (8+1+8=17) inside the job budget. --- .github/workflows/ci.yml | 56 ++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84ed743952..1655cd3ece 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,33 +192,33 @@ jobs: const runner8 = `runs-on=${context.runId}/runner=8cpu-linux-x64/volume=60g/spot=price-capacity-optimized/extras=otel+ecr-cache`; const services = [ - { name: 'Account', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Avatars', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Console', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Databases', runner: runner8, functional: false, processes: 3, timeout: 6 }, - { name: 'TablesDB', runner: runner8, functional: false, processes: 3, timeout: 6 }, - { name: 'Functions', runner: runner4, functional: false, processes: null, timeout: 6 }, - { name: 'FunctionsSchedule', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'GraphQL', runner: runner4, functional: false, processes: null, timeout: 6 }, - { name: 'Health', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Advisor', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Locale', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Projects', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Realtime', runner: runner4, functional: false, processes: null, timeout: 6 }, - { name: 'Sites', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Proxy', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Storage', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Tokens', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Teams', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Users', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'ProjectWebhooks', runner: runner4, functional: false, processes: null, timeout: 6 }, - { name: 'Webhooks', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'VCS', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Messaging', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Notifications', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Migrations', runner: runner4, functional: true, processes: 1, timeout: 6 }, - { name: 'Project', runner: runner4, functional: true, processes: null, timeout: 6 }, - { name: 'Presences', runner: runner4, functional: true, processes: null, timeout: 6 }, + { name: 'Account', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Avatars', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Console', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Databases', runner: runner8, functional: false, processes: 3, timeout: 8 }, + { name: 'TablesDB', runner: runner8, functional: false, processes: 3, timeout: 8 }, + { name: 'Functions', runner: runner4, functional: false, processes: null, timeout: 8 }, + { name: 'FunctionsSchedule', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'GraphQL', runner: runner4, functional: false, processes: null, timeout: 8 }, + { name: 'Health', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Advisor', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Locale', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Projects', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Realtime', runner: runner4, functional: false, processes: null, timeout: 8 }, + { name: 'Sites', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Proxy', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Storage', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Tokens', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Teams', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Users', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'ProjectWebhooks', runner: runner4, functional: false, processes: null, timeout: 8 }, + { name: 'Webhooks', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'VCS', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Messaging', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Notifications', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Migrations', runner: runner4, functional: true, processes: 1, timeout: 8 }, + { name: 'Project', runner: runner4, functional: true, processes: null, timeout: 8 }, + { name: 'Presences', runner: runner4, functional: true, processes: null, timeout: 8 }, ]; core.setOutput('services', JSON.stringify(services)); @@ -429,7 +429,7 @@ jobs: e2e_service: name: Tests / E2E / ${{ matrix.database }} (${{ matrix.mode }}) / ${{ matrix.service.name }} runs-on: ${{ matrix.service.runner }} - timeout-minutes: 15 + timeout-minutes: 20 needs: [build, matrix] permissions: contents: read