Merge pull request #11963 from appwrite/chore/http-benchmark-comparison

2026-05-26 13:51:13 +00:00 · 2026-04-23 09:48:37 +05:30
parent 49f8d6e89c 3d66078fe9
commit a8e6b1b683
4 changed files with 1113 additions and 83 deletions
@@ -0,0 +1,349 @@
+const fs = require('fs');
+
+const marker = '<!-- appwrite-benchmark-results -->';
+const serviceLabels = ['Account', 'TablesDB', 'Storage', 'Functions'];
+
+module.exports = async ({ github, context, core }) => {
+    const body = buildComment(core);
+    fs.writeFileSync('benchmark-comment.txt', body);
+
+    const pullRequest = context.payload.pull_request;
+    if (!pullRequest || pullRequest.head.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) {
+        return;
+    }
+
+    const comments = await github.paginate(github.rest.issues.listComments, {
+        owner: context.repo.owner,
+        repo: context.repo.repo,
+        issue_number: pullRequest.number,
+        per_page: 100,
+    });
+
+    const existing = comments.find((comment) => {
+        return comment.user?.type === 'Bot' && comment.body?.includes(marker);
+    }) || comments.find((comment) => {
+        return comment.user?.type === 'Bot' && comment.body?.includes('Benchmark results');
+    });
+
+    if (existing) {
+        await github.rest.issues.updateComment({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            comment_id: existing.id,
+            body,
+        });
+        return;
+    }
+
+    await github.rest.issues.createComment({
+        owner: context.repo.owner,
+        repo: context.repo.repo,
+        issue_number: pullRequest.number,
+        body,
+    });
+};
+
+function buildComment(core) {
+    const before = readSummary('benchmark-before-summary.json', core);
+    const after = readSummary('benchmark-after-summary.json', core);
+    const beforeSamples = readSamples('benchmark-before-samples.json', core);
+    const afterSamples = readSamples('benchmark-after-samples.json', core);
+    const baseRef = markdownText(process.env.BENCHMARK_BASE_REF || 'base');
+    const headRef = markdownText(process.env.BENCHMARK_HEAD_REF || 'head');
+    const rows = benchmarkRows(before, after, beforeSamples, afterSamples);
+    const topWaits = topSamples(afterSamples, 'appwrite_api_waiting', 3);
+    const lines = [
+        marker,
+        '## :sparkles: Benchmark results',
+        '',
+        `Comparing ${baseRef} (before) to ${headRef} (after).`,
+        '',
+    ];
+
+    if (before === null) {
+        lines.push('> Before benchmark did not complete; showing current branch metrics only.', '');
+    }
+    if (after === null) {
+        lines.push('> Current branch benchmark did not complete; showing available metrics only.', '');
+    }
+
+    lines.push(
+        '**Before**',
+        '',
+        metricTable(rows, 'before'),
+        '',
+        '**After**',
+        '',
+        metricTable(rows, 'after'),
+        '',
+        '**Delta**',
+        '',
+        '| Scenario | P95 delta (ms) |',
+        '| --- | ---: |',
+        ...rows.map(deltaRow),
+        '',
+        '<details>',
+        '<summary><strong>Top API waits</strong></summary>',
+        '',
+        '<br>',
+        '',
+        '| API request | Max wait (ms) |',
+        '| --- | ---: |',
+        ...topWaitRows(topWaits),
+        '',
+        '</details>',
+    );
+
+    return `${lines.join('\n')}\n`;
+}
+
+function readSummary(path, core) {
+    if (!fs.existsSync(path)) {
+        return null;
+    }
+
+    try {
+        return JSON.parse(fs.readFileSync(path, 'utf8'));
+    } catch (error) {
+        core?.warning(`Invalid benchmark summary ${path}: ${error.message}`);
+        return null;
+    }
+}
+
+function readSamples(path, core) {
+    if (!fs.existsSync(path)) {
+        return [];
+    }
+
+    const contents = fs.readFileSync(path, 'utf8').trim();
+    if (contents === '') {
+        return [];
+    }
+
+    return contents
+        .split('\n')
+        .filter(Boolean)
+        .flatMap((line) => {
+            try {
+                return [JSON.parse(line)];
+            } catch (error) {
+                core?.warning(`Invalid benchmark sample in ${path}: ${error.message}`);
+                return [];
+            }
+        });
+}
+
+function benchmarkRows(before, after, beforeSamples, afterSamples) {
+    const beforeServices = serviceStats(beforeSamples);
+    const afterServices = serviceStats(afterSamples);
+    return [
+        {
+            label: 'API total',
+            before: apiSampleStats(beforeSamples) || summaryStats(before, 'appwrite_api_duration'),
+            after: apiSampleStats(afterSamples) || summaryStats(after, 'appwrite_api_duration'),
+        },
+        ...serviceLabels.map((label) => ({
+            label,
+            before: beforeServices.get(label) || null,
+            after: afterServices.get(label) || null,
+        })),
+    ];
+}
+
+function summaryStats(summary, durationMetric, iterationsMetric = null, rpsMetric = null) {
+    const values = metricValues(summary, durationMetric);
+    if (!values) {
+        return null;
+    }
+
+    return {
+        p50: values.med ?? null,
+        p95: values['p(95)'] ?? null,
+        iterations: iterationsMetric ? metricValue(summary, iterationsMetric, 'count') : values.count ?? null,
+        rps: rpsMetric ? metricValue(summary, rpsMetric, 'rate') : null,
+    };
+}
+
+function serviceStats(samples) {
+    const apiSamples = samples.filter((sample) => {
+        return sample.metric === 'appwrite_api_duration' && typeof sample.data?.value === 'number';
+    });
+    const groups = new Map();
+
+    for (const sample of apiSamples) {
+        const service = serviceFromName(sample.data.tags?.name || '');
+        if (!service) {
+            continue;
+        }
+
+        const serviceSamples = groups.get(service) || [];
+        serviceSamples.push(sample);
+        groups.set(service, serviceSamples);
+    }
+
+    return new Map([...groups.entries()].map(([service, serviceSamples]) => {
+        const values = serviceSamples.map((sample) => sample.data.value);
+        const durationSeconds = sampleWindowSeconds(serviceSamples);
+        return [service, {
+            p50: percentile(values, 50),
+            p95: percentile(values, 95),
+            iterations: values.length,
+            rps: durationSeconds ? values.length / durationSeconds : null,
+        }];
+    }));
+}
+
+function apiSampleStats(samples) {
+    const apiSamples = samples.filter((sample) => {
+        return sample.metric === 'appwrite_api_duration' && typeof sample.data?.value === 'number';
+    });
+    const values = apiSamples.map((sample) => sample.data.value);
+    if (values.length === 0) {
+        return null;
+    }
+
+    const durationSeconds = sampleWindowSeconds(apiSamples);
+    return {
+        p50: percentile(values, 50),
+        p95: percentile(values, 95),
+        iterations: values.length,
+        rps: durationSeconds ? values.length / durationSeconds : null,
+    };
+}
+
+function serviceFromName(name) {
+    if (name.startsWith('account.')) {
+        return 'Account';
+    }
+    if (name.startsWith('tablesdb.')) {
+        return 'TablesDB';
+    }
+    if (name.startsWith('storage.') || name.startsWith('tokens.')) {
+        return 'Storage';
+    }
+    if (name.startsWith('functions.')) {
+        return 'Functions';
+    }
+    return null;
+}
+
+function sampleWindowSeconds(samples) {
+    const times = samples
+        .map((sample) => Date.parse(sample.data?.time))
+        .filter((value) => !Number.isNaN(value));
+    if (times.length < 2) {
+        return null;
+    }
+
+    return Math.max((Math.max(...times) - Math.min(...times)) / 1000, 1);
+}
+
+function percentile(values, percentileValue) {
+    if (values.length === 0) {
+        return null;
+    }
+
+    const sorted = [...values].sort((left, right) => left - right);
+    const index = Math.ceil((percentileValue / 100) * sorted.length) - 1;
+    return sorted[Math.max(0, Math.min(index, sorted.length - 1))];
+}
+
+function metricValues(data, metric) {
+    return data?.metrics?.[metric]?.values ?? null;
+}
+
+function metricValue(data, metric, stat) {
+    return metricValues(data, metric)?.[stat] ?? null;
+}
+
+function metricTable(rows, side) {
+    return [
+        '| Scenario | P50 (ms) | P95 (ms) | Requests | RPS |',
+        '| --- | ---: | ---: | ---: | ---: |',
+        ...rows.map((row) => metricRow(row, side)),
+    ].join('\n');
+}
+
+function metricRow(row, side) {
+    const values = row[side];
+    return `| ${row.label} | ${formatMs(values?.p50)} | ${formatMs(values?.p95)} | ${formatCount(values?.iterations)} | ${formatRate(values?.rps)} |`;
+}
+
+function deltaRow(row) {
+    return `| ${row.label} | ${formatDelta(row.before?.p95, row.after?.p95)} |`;
+}
+
+function topSamples(samples, metric, limit) {
+    const byName = samples.reduce((result, sample) => {
+        if (sample.metric !== metric || typeof sample.data?.value !== 'number') {
+            return result;
+        }
+
+        const name = sample.data.tags?.name || 'unknown';
+        const current = result.get(name);
+        if (!current || sample.data.value > current.value) {
+            result.set(name, { name, value: sample.data.value });
+        }
+
+        return result;
+    }, new Map());
+
+    return [...byName.values()]
+        .sort((left, right) => right.value - left.value)
+        .slice(0, limit);
+}
+
+function topWaitRows(samples) {
+    if (samples.length === 0) {
+        return ['| n/a | n/a |'];
+    }
+
+    return samples.map((sample) => {
+        return `| ${markdownText(sample.name).replace(/\|/g, '\\|')} | ${formatMs(sample.value)} |`;
+    });
+}
+
+function markdownText(value) {
+    return String(value || '').replace(/[\r\n]/g, ' ').replace(/[&<>"']/g, (char) => {
+        return ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#039;' })[char];
+    });
+}
+
+function formatMs(value) {
+    return formatNumber(value, 2);
+}
+
+function formatRate(value) {
+    return formatNumber(value, 2);
+}
+
+function formatCount(value) {
+    if (value === null || value === undefined || Number.isNaN(value)) {
+        return 'n/a';
+    }
+
+    return `${Math.round(value)}`;
+}
+
+function formatDelta(before, after) {
+    if (before === null || before === undefined || after === null || after === undefined || Number.isNaN(before) || Number.isNaN(after)) {
+        return 'n/a';
+    }
+
+    const difference = Number((after - before).toFixed(2));
+    return `${difference > 0 ? '+' : ''}${trimNumber(difference)}`;
+}
+
+function formatNumber(value, decimals) {
+    if (value === null || value === undefined || Number.isNaN(value)) {
+        return 'n/a';
+    }
+
+    return trimNumber(Number(value).toFixed(decimals));
+}
+
+function trimNumber(value) {
+    const text = String(value);
+    const trimmed = text.includes('.') ? text.replace(/\.?0+$/, '') : text;
+    return trimmed === '' ? '0' : trimmed;
+}
@@ -7,6 +7,7 @@ concurrency:
 env:
  COMPOSE_FILE: docker-compose.yml
  IMAGE: appwrite-dev
+  K6_VERSION: '0.53.0'

 on:
  pull_request:
@@ -547,6 +548,8 @@ jobs:
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
+        with:
+          fetch-depth: 1

      - name: Download Docker Image
        uses: actions/download-artifact@v7
@@ -660,13 +663,19 @@ jobs:

  benchmark:
    name: Benchmark
+    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    needs: build
    permissions:
+      actions: read
+      contents: read
+      issues: write
      pull-requests: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
+        with:
+          fetch-depth: 1

      - name: Download Docker Image
        uses: actions/download-artifact@v7
@@ -680,80 +689,145 @@ jobs:
          username: ${{ vars.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

-      - name: Load and Start Appwrite
+      - name: Load Appwrite image
        run: |
-          sed -i 's/traefik/localhost/g' .env
          docker load --input /tmp/${{ env.IMAGE }}.tar
-          docker compose up -d
-          sleep 10
+          docker tag ${{ env.IMAGE }} ${{ env.IMAGE }}:after

-      - name: Install Oha
+      - name: Setup k6
+        uses: grafana/setup-k6-action@ffe7d7290dfa715e48c2ccc924d068444c94bde2
+        with:
+          k6-version: ${{ env.K6_VERSION }}
+
+      - name: Prepare benchmark before
+        id: benchmark_before_prepare
+        continue-on-error: true
        run: |
-          echo "deb [signed-by=/usr/share/keyrings/azlux-archive-keyring.gpg] http://packages.azlux.fr/debian/ stable main" | sudo tee /etc/apt/sources.list.d/azlux.list
-          sudo wget -O /usr/share/keyrings/azlux-archive-keyring.gpg https://azlux.fr/repo.gpg
-          sudo apt update
-          sudo apt install oha
-          oha --version
+          git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }}
+          git worktree add --detach /tmp/appwrite-benchmark-before ${{ github.event.pull_request.base.sha }}
+          docker build \
+            --cache-from ${{ env.IMAGE }}:after \
+            --target development \
+            --build-arg DEBUG=false \
+            --build-arg TESTING=true \
+            --build-arg VERSION=dev \
+            --tag ${{ env.IMAGE }}:before \
+            /tmp/appwrite-benchmark-before

-      - name: Benchmark PR
-        run: 'oha -z 180s http://localhost/v1/health/version --output-format json > benchmark.json'
-
-      - name: Cleaning
-        run: docker compose down -v
-
-      - name: Installing latest version
+      - name: Start before Appwrite
+        id: benchmark_before_start
+        if: steps.benchmark_before_prepare.outcome == 'success'
+        continue-on-error: true
+        working-directory: /tmp/appwrite-benchmark-before
+        env:
+          _APP_DOMAIN: localhost
+          _APP_CONSOLE_DOMAIN: localhost
+          _APP_DOMAIN_FUNCTIONS: functions.localhost
+          _APP_OPTIONS_ABUSE: disabled
        run: |
-          rm .env
-          LATEST_TAG=$(curl -fsSL -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/appwrite/appwrite/releases/latest | jq -r .tag_name)
-          echo "Latest release tag: $LATEST_TAG"
-          curl -fsSL "https://raw.githubusercontent.com/appwrite/appwrite/${LATEST_TAG}/docker-compose.yml" -o docker-compose.yml
-          curl -fsSL "https://raw.githubusercontent.com/appwrite/appwrite/${LATEST_TAG}/.env" -o .env
-          sed -i 's/_APP_OPTIONS_ABUSE=enabled/_APP_OPTIONS_ABUSE=disabled/g' .env
-          docker compose up -d
-          sleep 10
+          docker tag ${{ env.IMAGE }}:before ${{ env.IMAGE }}
+          docker compose up -d --wait --no-build

-      - name: Benchmark Latest
-        run: oha -z 180s http://localhost/v1/health/version --output-format json > benchmark-latest.json
+      - name: Prepare benchmark files
+        run: rm -f benchmark-before-summary.json benchmark-after-summary.json benchmark-before-samples.json benchmark-after-samples.json

-      - name: Prepare comment
+      - name: Benchmark before
+        if: steps.benchmark_before_start.outcome == 'success'
+        continue-on-error: true
+        uses: grafana/run-k6-action@a15e2072ede004e8d46141e33d7f7dad8ad08d9d
+        env:
+          APPWRITE_ENDPOINT: 'http://localhost/v1'
+          APPWRITE_BENCHMARK_ITERATIONS: '5'
+          APPWRITE_BENCHMARK_VUS: '1'
+          APPWRITE_WORKER_TIMEOUT_MS: '120000'
+          APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-before-summary.json'
+        with:
+          path: tests/benchmarks/http.js
+          flags: --quiet --out json=benchmark-before-samples.json
+          cloud-comment-on-pr: false
+          debug: true
+
+      - name: Stop before Appwrite
+        if: always()
        run: |
-          echo '## :sparkles: Benchmark results' > benchmark.txt
-          echo ' ' >> benchmark.txt
-          echo "- Requests per second: $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt
-          echo "- Requests with 200 status code: $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt
-          echo "- P99 latency: $(jq -r '.latencyPercentiles.p99' benchmark.json )" >> benchmark.txt
-          echo " " >> benchmark.txt
-          echo " " >> benchmark.txt
-          echo "## :zap:  Benchmark Comparison" >> benchmark.txt
-          echo " " >> benchmark.txt
-          echo "| Metric | This PR | Latest version | "  >> benchmark.txt
-          echo "| --- | --- | --- | "  >> benchmark.txt
-          echo "| RPS | $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.summary.requestsPerSec|tonumber|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | "  >> benchmark.txt
-          echo "| 200 | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | "  >> benchmark.txt
-          echo "| P99 | $(jq -r '.latencyPercentiles.p99' benchmark.json ) | $(jq -r '.latencyPercentiles.p99' benchmark-latest.json ) | "  >> benchmark.txt
+          if [ -d /tmp/appwrite-benchmark-before ]; then
+            cd /tmp/appwrite-benchmark-before
+            docker compose down -v || true
+          fi
+
+      - name: Wait for benchmark ports
+        if: always()
+        run: |
+          for port in 80 443 8080 9503; do
+            for attempt in $(seq 1 30); do
+              if ! ss -ltn | awk '{print $4}' | grep -Eq "[:.]${port}$"; then
+                break
+              fi
+              sleep 1
+            done
+
+            if ss -ltn | awk '{print $4}' | grep -Eq "[:.]${port}$"; then
+              echo "Port ${port} is still in use after stopping the before stack"
+              ss -ltn
+              exit 1
+            fi
+          done
+
+      - name: Start after Appwrite
+        env:
+          _APP_DOMAIN: localhost
+          _APP_CONSOLE_DOMAIN: localhost
+          _APP_DOMAIN_FUNCTIONS: functions.localhost
+          _APP_OPTIONS_ABUSE: disabled
+        run: |
+          docker tag ${{ env.IMAGE }}:after ${{ env.IMAGE }}
+          docker compose up -d --wait --no-build
+
+      - name: Benchmark after
+        id: benchmark_after
+        continue-on-error: true
+        uses: grafana/run-k6-action@a15e2072ede004e8d46141e33d7f7dad8ad08d9d
+        env:
+          APPWRITE_ENDPOINT: 'http://localhost/v1'
+          APPWRITE_BENCHMARK_ITERATIONS: '5'
+          APPWRITE_BENCHMARK_VUS: '1'
+          APPWRITE_WORKER_TIMEOUT_MS: '120000'
+          APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH: '../../benchmark-before-summary.json'
+          APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-after-summary.json'
+        with:
+          path: tests/benchmarks/http.js
+          flags: --quiet --out json=benchmark-after-samples.json
+          cloud-comment-on-pr: false
+          debug: true
+
+      - name: Stop after Appwrite
+        if: always()
+        run: docker compose down -v || true
+
+      - name: Comment on PR
+        if: always()
+        uses: actions/github-script@v8
+        env:
+          BENCHMARK_BASE_REF: ${{ github.event.pull_request.base.ref }}
+          BENCHMARK_HEAD_REF: ${{ github.event.pull_request.head.ref }}
+        with:
+          script: |
+            const comment = require('./.github/workflows/benchmark-comment.js');
+            await comment({ github, context, core });

      - name: Save results
        uses: actions/upload-artifact@v7
        if: ${{ !cancelled() }}
        with:
-          name: benchmark.json
-          path: benchmark.json
+          name: benchmark-results
+          path: |
+            benchmark-comment.txt
+            benchmark-before-summary.json
+            benchmark-after-summary.json
+            benchmark-before-samples.json
+            benchmark-after-samples.json
          retention-days: 7

-      - name: Find Comment
-        if: github.event.pull_request.head.repo.full_name == github.repository
-        uses: peter-evans/find-comment@v3
-        id: fc
-        with:
-          issue-number: ${{ github.event.pull_request.number }}
-          comment-author: 'github-actions[bot]'
-          body-includes: Benchmark results
-
-      - name: Comment on PR
-        if: github.event.pull_request.head.repo.full_name == github.repository
-        uses: peter-evans/create-or-update-comment@v4
-        with:
-          comment-id: ${{ steps.fc.outputs.comment-id }}
-          issue-number: ${{ github.event.pull_request.number }}
-          body-path: benchmark.txt
-          edit-mode: replace
+      - name: Fail benchmark
+        if: always() && steps.benchmark_after.outcome != 'success'
+        run: exit 1