From 2120f2f41e1b996996602a089dc7dd8e0af479a3 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Sat, 20 Jun 2026 20:53:26 +0530 Subject: [PATCH] Improve CI image handoff and merged-run behavior (#12629) --- .github/workflows/ci.yml | 169 ++++++++++++++------------------------- 1 file changed, 62 insertions(+), 107 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2bd7ea4a7..488d635828 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,12 @@ name: CI concurrency: - group: ci-${{ github.ref }} - cancel-in-progress: true + group: ci-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: COMPOSE_FILE: docker-compose.yml IMAGE: appwrite-dev - REGISTRY_IMAGE: ghcr.io/${{ github.repository }}/appwrite-dev K6_VERSION: '0.53.0' on: @@ -239,36 +238,36 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const runner4 = `runs-on=${context.runId}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel`; - const runner8 = `runs-on=${context.runId}/runner=8cpu-linux-x64/volume=120g/spot=false/extras=otel`; + const runner4 = `runs-on=${context.runId}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel+ecr-cache`; + const runner8 = `runs-on=${context.runId}/runner=8cpu-linux-x64/volume=120g/spot=false/extras=otel+ecr-cache`; const services = [ - { name: 'Account', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'Avatars', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'Console', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'Databases', runner: runner8, functional: false, processes: 3, timeout: 30 }, - { name: 'TablesDB', runner: runner8, functional: false, processes: 3, timeout: 30 }, - { name: 'Functions', runner: runner4, functional: false, processes: null, timeout: 20 }, - { name: 'FunctionsSchedule', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'GraphQL', runner: runner4, functional: false, processes: null, timeout: 20 }, - { name: 'Health', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'Advisor', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'Locale', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'Projects', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'Realtime', runner: runner4, functional: false, processes: null, timeout: 20 }, - { name: 'Sites', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'Proxy', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'Storage', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'Tokens', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'Teams', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'Users', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'ProjectWebhooks', runner: runner4, functional: false, processes: null, timeout: 20 }, - { name: 'Webhooks', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'VCS', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'Messaging', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'Migrations', runner: runner4, functional: true, processes: 1, timeout: 20 }, - { name: 'Project', runner: runner4, functional: true, processes: null, timeout: 20 }, - { name: 'Presences', runner: runner4, functional: true, processes: null, timeout: 20 }, + { name: 'Account', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Avatars', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Console', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Databases', runner: runner8, functional: false, processes: 3, timeout: 15 }, + { name: 'TablesDB', runner: runner8, functional: false, processes: 3, timeout: 15 }, + { name: 'Functions', runner: runner4, functional: false, processes: null, timeout: 15 }, + { name: 'FunctionsSchedule', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'GraphQL', runner: runner4, functional: false, processes: null, timeout: 15 }, + { name: 'Health', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Advisor', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Locale', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Projects', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Realtime', runner: runner4, functional: false, processes: null, timeout: 15 }, + { name: 'Sites', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Proxy', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Storage', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Tokens', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Teams', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Users', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'ProjectWebhooks', runner: runner4, functional: false, processes: null, timeout: 15 }, + { name: 'Webhooks', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'VCS', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Messaging', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Migrations', runner: runner4, functional: true, processes: 1, timeout: 15 }, + { name: 'Project', runner: runner4, functional: true, processes: null, timeout: 15 }, + { name: 'Presences', runner: runner4, functional: true, processes: null, timeout: 15 }, ]; core.setOutput('services', JSON.stringify(services)); @@ -303,7 +302,7 @@ jobs: build: name: Build - runs-on: ubuntu-latest + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel+s3-cache+ecr-cache steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -316,13 +315,6 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GHCR - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 @@ -331,9 +323,9 @@ jobs: with: context: . push: true - tags: ${{ env.REGISTRY_IMAGE }}:${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=max + tags: ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} + cache-from: type=s3,blobs_prefix=cache/${{ github.repository }}/,manifests_prefix=cache/${{ github.repository }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }} + cache-to: type=s3,blobs_prefix=cache/${{ github.repository }}/,manifests_prefix=cache/${{ github.repository }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max target: development build-args: | DEBUG=false @@ -342,7 +334,7 @@ jobs: unit: name: Tests / Unit - runs-on: ubuntu-latest + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel+ecr-cache needs: build permissions: contents: read @@ -358,17 +350,10 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GHCR - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Pull Docker Image run: | - docker pull ${{ env.REGISTRY_IMAGE }}:${{ github.sha }} - docker tag ${{ env.REGISTRY_IMAGE }}:${{ github.sha }} ${{ env.IMAGE }} + docker pull ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} + docker tag ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} ${{ env.IMAGE }} - name: Load and Start Appwrite timeout-minutes: 5 @@ -380,9 +365,10 @@ jobs: run: docker compose exec -T appwrite vars - name: Run Unit Tests + timeout-minutes: 15 uses: itznotabug/php-retry@d6bef45a8bff490babfb613e33b00d133e4062f0 # v3 with: - max_attempts: 2 + max_attempts: ${{ github.event_name == 'pull_request' && 2 || 1 }} retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} @@ -395,7 +381,7 @@ jobs: e2e_general: name: Tests / E2E / General - runs-on: ubuntu-latest + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel+ecr-cache needs: build permissions: contents: read @@ -411,17 +397,10 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GHCR - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Pull Docker Image run: | - docker pull ${{ env.REGISTRY_IMAGE }}:${{ github.sha }} - docker tag ${{ env.REGISTRY_IMAGE }}:${{ github.sha }} ${{ env.IMAGE }} + docker pull ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} + docker tag ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} ${{ env.IMAGE }} - name: Load and Start Appwrite timeout-minutes: 5 @@ -438,9 +417,10 @@ jobs: done - name: Run General Tests + timeout-minutes: 15 uses: itznotabug/php-retry@d6bef45a8bff490babfb613e33b00d133e4062f0 # v3 with: - max_attempts: 2 + max_attempts: ${{ github.event_name == 'pull_request' && 2 || 1 }} retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} @@ -502,17 +482,10 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GHCR - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Pull Docker Image run: | - docker pull ${{ env.REGISTRY_IMAGE }}:${{ github.sha }} - docker tag ${{ env.REGISTRY_IMAGE }}:${{ github.sha }} ${{ env.IMAGE }} + docker pull ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} + docker tag ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} ${{ env.IMAGE }} - name: Load and Start Appwrite timeout-minutes: 5 @@ -534,9 +507,10 @@ jobs: done - name: Run tests + timeout-minutes: 15 uses: itznotabug/php-retry@d6bef45a8bff490babfb613e33b00d133e4062f0 # v3 with: - max_attempts: 2 + max_attempts: ${{ github.event_name == 'pull_request' && 2 || 1 }} retry_wait_seconds: 60 timeout_minutes: ${{ matrix.service.timeout }} job_id: ${{ job.check_run_id }} @@ -569,7 +543,7 @@ jobs: e2e_abuse: name: Tests / E2E / Abuse (${{ matrix.mode }}) - runs-on: ubuntu-latest + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel+ecr-cache needs: [build, matrix] permissions: contents: read @@ -591,17 +565,10 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GHCR - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Pull Docker Image run: | - docker pull ${{ env.REGISTRY_IMAGE }}:${{ github.sha }} - docker tag ${{ env.REGISTRY_IMAGE }}:${{ github.sha }} ${{ env.IMAGE }} + docker pull ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} + docker tag ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} ${{ env.IMAGE }} - name: Load and Start Appwrite timeout-minutes: 5 @@ -615,9 +582,10 @@ jobs: docker compose up -d --quiet-pull --wait - name: Run tests + timeout-minutes: 15 uses: itznotabug/php-retry@d6bef45a8bff490babfb613e33b00d133e4062f0 # v3 with: - max_attempts: 2 + max_attempts: ${{ github.event_name == 'pull_request' && 2 || 1 }} retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} @@ -636,7 +604,7 @@ jobs: e2e_screenshots: name: Tests / E2E / Screenshots (${{ matrix.mode }}) - runs-on: ubuntu-latest + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel+ecr-cache needs: [build, matrix] permissions: contents: read @@ -656,17 +624,10 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GHCR - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Pull Docker Image run: | - docker pull ${{ env.REGISTRY_IMAGE }}:${{ github.sha }} - docker tag ${{ env.REGISTRY_IMAGE }}:${{ github.sha }} ${{ env.IMAGE }} + docker pull ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} + docker tag ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} ${{ env.IMAGE }} - name: Load and Start Appwrite timeout-minutes: 5 @@ -687,9 +648,10 @@ jobs: done - name: Run tests + timeout-minutes: 15 uses: itznotabug/php-retry@d6bef45a8bff490babfb613e33b00d133e4062f0 # v3 with: - max_attempts: 2 + max_attempts: ${{ github.event_name == 'pull_request' && 2 || 1 }} retry_wait_seconds: 60 timeout_minutes: 15 job_id: ${{ job.check_run_id }} @@ -709,7 +671,7 @@ jobs: benchmark: name: Benchmark if: github.event_name == 'pull_request' || github.event_name == 'push' - runs-on: ubuntu-latest + runs-on: runs-on=${{ github.run_id }}/runner=4cpu-linux-x64/volume=120g/spot=false/extras=otel+ecr-cache needs: build permissions: actions: read @@ -729,18 +691,11 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GHCR - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Pull Appwrite image run: | - docker pull ${{ env.REGISTRY_IMAGE }}:${{ github.sha }} - docker tag ${{ env.REGISTRY_IMAGE }}:${{ github.sha }} ${{ env.IMAGE }} - docker tag ${{ env.REGISTRY_IMAGE }}:${{ github.sha }} ${{ env.IMAGE }}:after + docker pull ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} + docker tag ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} ${{ env.IMAGE }} + docker tag ${{ env.RUNS_ON_ECR_CACHE }}:${{ env.IMAGE }}-${{ github.sha }} ${{ env.IMAGE }}:after - name: Setup k6 uses: grafana/setup-k6-action@db07bd9765aac508ef18982e52ab937fe633a065 # v1.2.1