From 6b35a4ad89b6d97b3967c2785b8bafd181348205 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 26 Jun 2026 15:12:31 +0100 Subject: [PATCH 1/7] Run benchmark at 50 VUs and clarify the PR comment Bump the CI benchmark to 50 VUs (with 250 shared iterations so every VU does work) for a meaningful concurrency/RPS measurement. Rework the PR comment around a headline table (RPS, P50, P95 with direction-aware percent change) and move the per-scenario breakdown and top API waits into a collapsed details section. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/benchmark-comment.js | 91 ++++++++++++++++---------- .github/workflows/ci.yml | 8 +-- 2 files changed, 60 insertions(+), 39 deletions(-) diff --git a/.github/workflows/benchmark-comment.js b/.github/workflows/benchmark-comment.js index f25116c4f2..0de20aac13 100644 --- a/.github/workflows/benchmark-comment.js +++ b/.github/workflows/benchmark-comment.js @@ -51,42 +51,37 @@ function buildComment(core) { const baseRef = markdownText(process.env.BENCHMARK_BASE_REF || 'base'); const headRef = markdownText(process.env.BENCHMARK_HEAD_REF || 'head'); const rows = benchmarkRows(before, after, beforeSamples, afterSamples); - const topWaits = topSamples(afterSamples, 'appwrite_api_waiting', 3); + const total = rows[0]; + const topWaits = topSamples(afterSamples, 'appwrite_api_waiting', 5); const lines = [ marker, '## :sparkles: Benchmark results', '', - `Comparing ${baseRef} (before) to ${headRef} (after).`, + `Comparing \`${baseRef}\` (before) → \`${headRef}\` (after).`, '', ]; if (before === null) { - lines.push('> Before benchmark did not complete; showing current branch metrics only.', ''); + lines.push('> :warning: Before benchmark did not complete — change column unavailable.', ''); } if (after === null) { - lines.push('> Current branch benchmark did not complete; showing available metrics only.', ''); + lines.push('> :warning: Current branch benchmark did not complete — showing available metrics only.', ''); } lines.push( - '**Before**', - '', - metricTable(rows, 'before'), - '', - '**After**', - '', - metricTable(rows, 'after'), - '', - '**Delta**', - '', - '| Scenario | P95 delta (ms) |', - '| --- | ---: |', - ...rows.map(deltaRow), + ...headlineTable(total), '', '
', - 'Top API waits', + 'Per-scenario breakdown & investigation details', '', '
', '', + '_Metrics below reflect the current branch (after). Δ P95 compares against the base._', + '', + detailTable(rows), + '', + '**Top API waits (after)**', + '', '| API request | Max wait (ms) |', '| --- | ---: |', ...topWaitRows(topWaits), @@ -97,6 +92,44 @@ function buildComment(core) { return `${lines.join('\n')}\n`; } +function headlineTable(total) { + const { before, after } = total; + return [ + '| Metric | Before | After | Change |', + '| --- | ---: | ---: | :--- |', + `| :rocket: Requests/sec | ${formatRate(before?.rps)} | ${formatRate(after?.rps)} | ${changeCell(before?.rps, after?.rps, false)} |`, + `| :stopwatch: Latency P50 | ${formatMsUnit(before?.p50)} | ${formatMsUnit(after?.p50)} | ${changeCell(before?.p50, after?.p50, true)} |`, + `| :stopwatch: Latency P95 | ${formatMsUnit(before?.p95)} | ${formatMsUnit(after?.p95)} | ${changeCell(before?.p95, after?.p95, true)} |`, + ]; +} + +function changeCell(before, after, lowerIsBetter) { + if (before === null || before === undefined || after === null || after === undefined || Number.isNaN(before) || Number.isNaN(after) || before === 0) { + return 'n/a'; + } + + const percent = Number((((after - before) / before) * 100).toFixed(1)); + if (percent === 0) { + return ':white_circle: 0%'; + } + + const improved = lowerIsBetter ? percent < 0 : percent > 0; + return `${improved ? ':green_circle:' : ':red_circle:'} ${percent > 0 ? '+' : ''}${trimNumber(percent)}%`; +} + +function detailTable(rows) { + return [ + '| Scenario | P50 (ms) | P95 (ms) | Requests | RPS | Δ P95 (ms) |', + '| --- | ---: | ---: | ---: | ---: | ---: |', + ...rows.map(detailRow), + ].join('\n'); +} + +function detailRow(row) { + const values = row.after; + return `| ${row.label} | ${formatMs(values?.p50)} | ${formatMs(values?.p95)} | ${formatCount(values?.iterations)} | ${formatRate(values?.rps)} | ${formatDelta(row.before?.p95, row.after?.p95)} |`; +} + function readSummary(path, core) { if (!fs.existsSync(path)) { return null; @@ -256,23 +289,6 @@ function metricValue(data, metric, stat) { return metricValues(data, metric)?.[stat] ?? null; } -function metricTable(rows, side) { - return [ - '| Scenario | P50 (ms) | P95 (ms) | Requests | RPS |', - '| --- | ---: | ---: | ---: | ---: |', - ...rows.map((row) => metricRow(row, side)), - ].join('\n'); -} - -function metricRow(row, side) { - const values = row[side]; - return `| ${row.label} | ${formatMs(values?.p50)} | ${formatMs(values?.p95)} | ${formatCount(values?.iterations)} | ${formatRate(values?.rps)} |`; -} - -function deltaRow(row) { - return `| ${row.label} | ${formatDelta(row.before?.p95, row.after?.p95)} |`; -} - function topSamples(samples, metric, limit) { const byName = samples.reduce((result, sample) => { if (sample.metric !== metric || typeof sample.data?.value !== 'number') { @@ -313,6 +329,11 @@ function formatMs(value) { return formatNumber(value, 2); } +function formatMsUnit(value) { + const formatted = formatNumber(value, 2); + return formatted === 'n/a' ? formatted : `${formatted} ms`; +} + function formatRate(value) { return formatNumber(value, 2); } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7bc5c91e5..13eb44a6f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -686,8 +686,8 @@ jobs: uses: grafana/run-k6-action@de51a7390bdf0ac85a3bef493691bd71d4c7c158 # v1.4.0 env: APPWRITE_ENDPOINT: 'http://localhost:9520/v1' - APPWRITE_BENCHMARK_ITERATIONS: '5' - APPWRITE_BENCHMARK_VUS: '1' + APPWRITE_BENCHMARK_ITERATIONS: '250' + APPWRITE_BENCHMARK_VUS: '50' APPWRITE_WORKER_TIMEOUT_MS: '120000' APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-before-summary.json' with: @@ -747,8 +747,8 @@ jobs: uses: grafana/run-k6-action@de51a7390bdf0ac85a3bef493691bd71d4c7c158 # v1.4.0 env: APPWRITE_ENDPOINT: 'http://localhost:9520/v1' - APPWRITE_BENCHMARK_ITERATIONS: '5' - APPWRITE_BENCHMARK_VUS: '1' + APPWRITE_BENCHMARK_ITERATIONS: '250' + APPWRITE_BENCHMARK_VUS: '50' APPWRITE_WORKER_TIMEOUT_MS: '120000' APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH: '../../benchmark-before-summary.json' APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-after-summary.json' From 180bb589318d074bd6f747ecbf077c560a07d0f2 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 26 Jun 2026 15:17:29 +0100 Subject: [PATCH 2/7] Use constant-vus executor: 50 VUs for 2 minutes Switch the benchmark scenario from shared-iterations to constant-vus so it runs a fixed 50 VUs for a fixed 2m window (configurable via APPWRITE_BENCHMARK_DURATION) instead of draining a shared iteration pool. Drops the now-unused iterations/maxDuration knobs. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 4 ++-- tests/benchmarks/http.js | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13eb44a6f5..d753b1cc0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -686,7 +686,7 @@ jobs: uses: grafana/run-k6-action@de51a7390bdf0ac85a3bef493691bd71d4c7c158 # v1.4.0 env: APPWRITE_ENDPOINT: 'http://localhost:9520/v1' - APPWRITE_BENCHMARK_ITERATIONS: '250' + APPWRITE_BENCHMARK_DURATION: '2m' APPWRITE_BENCHMARK_VUS: '50' APPWRITE_WORKER_TIMEOUT_MS: '120000' APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-before-summary.json' @@ -747,7 +747,7 @@ jobs: uses: grafana/run-k6-action@de51a7390bdf0ac85a3bef493691bd71d4c7c158 # v1.4.0 env: APPWRITE_ENDPOINT: 'http://localhost:9520/v1' - APPWRITE_BENCHMARK_ITERATIONS: '250' + APPWRITE_BENCHMARK_DURATION: '2m' APPWRITE_BENCHMARK_VUS: '50' APPWRITE_WORKER_TIMEOUT_MS: '120000' APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH: '../../benchmark-before-summary.json' diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 1e357f7dc9..5930b6efe2 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -18,8 +18,8 @@ const REGION = __ENV.APPWRITE_REGION || 'default'; const REDIRECT_URL = __ENV.APPWRITE_BENCHMARK_REDIRECT_URL || 'http://localhost'; const PASSWORD = __ENV.APPWRITE_BENCHMARK_PASSWORD || 'Password123!'; const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 120000); -const ITERATIONS = Number(__ENV.APPWRITE_BENCHMARK_ITERATIONS || 1); const VUS = Number(__ENV.APPWRITE_BENCHMARK_VUS || 1); +const DURATION = __ENV.APPWRITE_BENCHMARK_DURATION || '2m'; const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || '/tmp/appwrite-k6-summary.json'; const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || ''; const PREVIOUS_SUMMARY = PREVIOUS_SUMMARY_PATH ? loadPreviousSummary(PREVIOUS_SUMMARY_PATH) : null; @@ -32,11 +32,10 @@ export const flowFailures = new Counter('appwrite_benchmark_flow_failures'); export const options = { scenarios: { curated_flows: { - executor: 'shared-iterations', + executor: 'constant-vus', exec: 'curatedFlows', vus: VUS, - iterations: ITERATIONS, - maxDuration: __ENV.APPWRITE_BENCHMARK_MAX_DURATION || '30m', + duration: DURATION, }, }, thresholds: { From 2face3c2922c8ca853885c55c1648a15cc3bcb10 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 26 Jun 2026 15:36:08 +0100 Subject: [PATCH 3/7] Format benchmark numbers as inline code with thousands separators Wrap every numeric cell in inline code and group thousands (e.g. `3,891.22 ms`, `+1,680.87`) so the comment is easier to scan; n/a stays plain. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/benchmark-comment.js | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/benchmark-comment.js b/.github/workflows/benchmark-comment.js index 0de20aac13..75f0658439 100644 --- a/.github/workflows/benchmark-comment.js +++ b/.github/workflows/benchmark-comment.js @@ -110,11 +110,11 @@ function changeCell(before, after, lowerIsBetter) { const percent = Number((((after - before) / before) * 100).toFixed(1)); if (percent === 0) { - return ':white_circle: 0%'; + return `:white_circle: ${code('0%')}`; } const improved = lowerIsBetter ? percent < 0 : percent > 0; - return `${improved ? ':green_circle:' : ':red_circle:'} ${percent > 0 ? '+' : ''}${trimNumber(percent)}%`; + return `${improved ? ':green_circle:' : ':red_circle:'} ${code(`${percent > 0 ? '+' : ''}${trimNumber(percent)}%`)}`; } function detailTable(rows) { @@ -326,16 +326,16 @@ function markdownText(value) { } function formatMs(value) { - return formatNumber(value, 2); + return code(formatNumber(value, 2)); } function formatMsUnit(value) { const formatted = formatNumber(value, 2); - return formatted === 'n/a' ? formatted : `${formatted} ms`; + return formatted === 'n/a' ? formatted : code(`${formatted} ms`); } function formatRate(value) { - return formatNumber(value, 2); + return code(formatNumber(value, 2)); } function formatCount(value) { @@ -343,7 +343,7 @@ function formatCount(value) { return 'n/a'; } - return `${Math.round(value)}`; + return code(Math.round(value).toLocaleString('en-US')); } function formatDelta(before, after) { @@ -352,7 +352,8 @@ function formatDelta(before, after) { } const difference = Number((after - before).toFixed(2)); - return `${difference > 0 ? '+' : ''}${trimNumber(difference)}`; + const sign = difference > 0 ? '+' : difference < 0 ? '-' : ''; + return code(`${sign}${formatNumber(Math.abs(difference), 2)}`); } function formatNumber(value, decimals) { @@ -360,7 +361,14 @@ function formatNumber(value, decimals) { return 'n/a'; } - return trimNumber(Number(value).toFixed(decimals)); + return trimNumber(Number(value).toLocaleString('en-US', { + minimumFractionDigits: decimals, + maximumFractionDigits: decimals, + })); +} + +function code(text) { + return text === 'n/a' ? text : `\`${text}\``; } function trimNumber(value) { From 11397dcba023828e669d68a97dffbebd90692404 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 26 Jun 2026 15:44:11 +0100 Subject: [PATCH 4/7] Drop absolute p95 latency threshold from benchmark The p(95)<2000ms gate was calibrated for a single VU. At 50 concurrent VUs the CI runner saturates and P95 always exceeds 2s, so the gate failed every run. Keep http_req_failed and flow_failures as the correctness gates; latency is now judged via the before/after comparison in the PR comment. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/benchmarks/http.js | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 5930b6efe2..80218225fd 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -40,7 +40,6 @@ export const options = { }, thresholds: { http_req_failed: ['rate<0.05'], - appwrite_api_duration: ['p(95)<2000'], appwrite_benchmark_flow_failures: ['count<1'], }, }; From 101ffff8e2c7adf234b348dbbc2e6865dbeefce2 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 26 Jun 2026 15:46:48 +0100 Subject: [PATCH 5/7] Only color benchmark change when delta exceeds 5% MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Treat changes within ±5% as noise and show a neutral white circle; only swings beyond 5% get a green/red indicator. The percent value is still always shown. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/benchmark-comment.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark-comment.js b/.github/workflows/benchmark-comment.js index 75f0658439..bdb0055c60 100644 --- a/.github/workflows/benchmark-comment.js +++ b/.github/workflows/benchmark-comment.js @@ -109,12 +109,14 @@ function changeCell(before, after, lowerIsBetter) { } const percent = Number((((after - before) / before) * 100).toFixed(1)); - if (percent === 0) { - return `:white_circle: ${code('0%')}`; + const label = code(`${percent > 0 ? '+' : ''}${trimNumber(percent)}%`); + + if (Math.abs(percent) <= 5) { + return `:white_circle: ${label}`; } const improved = lowerIsBetter ? percent < 0 : percent > 0; - return `${improved ? ':green_circle:' : ':red_circle:'} ${code(`${percent > 0 ? '+' : ''}${trimNumber(percent)}%`)}`; + return `${improved ? ':green_circle:' : ':red_circle:'} ${label}`; } function detailTable(rows) { From 0050bf47b2d6ac9100c8f284cd874b0860cb5766 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 26 Jun 2026 15:47:17 +0100 Subject: [PATCH 6/7] Run benchmark at 20 VUs Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d753b1cc0b..832d89aaf8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -687,7 +687,7 @@ jobs: env: APPWRITE_ENDPOINT: 'http://localhost:9520/v1' APPWRITE_BENCHMARK_DURATION: '2m' - APPWRITE_BENCHMARK_VUS: '50' + APPWRITE_BENCHMARK_VUS: '20' APPWRITE_WORKER_TIMEOUT_MS: '120000' APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-before-summary.json' with: @@ -748,7 +748,7 @@ jobs: env: APPWRITE_ENDPOINT: 'http://localhost:9520/v1' APPWRITE_BENCHMARK_DURATION: '2m' - APPWRITE_BENCHMARK_VUS: '50' + APPWRITE_BENCHMARK_VUS: '20' APPWRITE_WORKER_TIMEOUT_MS: '120000' APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH: '../../benchmark-before-summary.json' APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-after-summary.json' From 5782f4046a4e81e0bbbdf87c3934d6628264aa69 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 26 Jun 2026 15:55:05 +0100 Subject: [PATCH 7/7] Run benchmark for 1 minute instead of 2 Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 4 ++-- tests/benchmarks/http.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 832d89aaf8..19493e8443 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -686,7 +686,7 @@ jobs: uses: grafana/run-k6-action@de51a7390bdf0ac85a3bef493691bd71d4c7c158 # v1.4.0 env: APPWRITE_ENDPOINT: 'http://localhost:9520/v1' - APPWRITE_BENCHMARK_DURATION: '2m' + APPWRITE_BENCHMARK_DURATION: '1m' APPWRITE_BENCHMARK_VUS: '20' APPWRITE_WORKER_TIMEOUT_MS: '120000' APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-before-summary.json' @@ -747,7 +747,7 @@ jobs: uses: grafana/run-k6-action@de51a7390bdf0ac85a3bef493691bd71d4c7c158 # v1.4.0 env: APPWRITE_ENDPOINT: 'http://localhost:9520/v1' - APPWRITE_BENCHMARK_DURATION: '2m' + APPWRITE_BENCHMARK_DURATION: '1m' APPWRITE_BENCHMARK_VUS: '20' APPWRITE_WORKER_TIMEOUT_MS: '120000' APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH: '../../benchmark-before-summary.json' diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 80218225fd..e809372a4f 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -19,7 +19,7 @@ const REDIRECT_URL = __ENV.APPWRITE_BENCHMARK_REDIRECT_URL || 'http://localhost' const PASSWORD = __ENV.APPWRITE_BENCHMARK_PASSWORD || 'Password123!'; const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 120000); const VUS = Number(__ENV.APPWRITE_BENCHMARK_VUS || 1); -const DURATION = __ENV.APPWRITE_BENCHMARK_DURATION || '2m'; +const DURATION = __ENV.APPWRITE_BENCHMARK_DURATION || '1m'; const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || '/tmp/appwrite-k6-summary.json'; const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || ''; const PREVIOUS_SUMMARY = PREVIOUS_SUMMARY_PATH ? loadPreviousSummary(PREVIOUS_SUMMARY_PATH) : null;