diff --git a/.github/workflows/benchmark-comment.js b/.github/workflows/benchmark-comment.js index f25116c4f2..bdb0055c60 100644 --- a/.github/workflows/benchmark-comment.js +++ b/.github/workflows/benchmark-comment.js @@ -51,42 +51,37 @@ function buildComment(core) { const baseRef = markdownText(process.env.BENCHMARK_BASE_REF || 'base'); const headRef = markdownText(process.env.BENCHMARK_HEAD_REF || 'head'); const rows = benchmarkRows(before, after, beforeSamples, afterSamples); - const topWaits = topSamples(afterSamples, 'appwrite_api_waiting', 3); + const total = rows[0]; + const topWaits = topSamples(afterSamples, 'appwrite_api_waiting', 5); const lines = [ marker, '## :sparkles: Benchmark results', '', - `Comparing ${baseRef} (before) to ${headRef} (after).`, + `Comparing \`${baseRef}\` (before) → \`${headRef}\` (after).`, '', ]; if (before === null) { - lines.push('> Before benchmark did not complete; showing current branch metrics only.', ''); + lines.push('> :warning: Before benchmark did not complete — change column unavailable.', ''); } if (after === null) { - lines.push('> Current branch benchmark did not complete; showing available metrics only.', ''); + lines.push('> :warning: Current branch benchmark did not complete — showing available metrics only.', ''); } lines.push( - '**Before**', - '', - metricTable(rows, 'before'), - '', - '**After**', - '', - metricTable(rows, 'after'), - '', - '**Delta**', - '', - '| Scenario | P95 delta (ms) |', - '| --- | ---: |', - ...rows.map(deltaRow), + ...headlineTable(total), '', '
', - 'Top API waits', + 'Per-scenario breakdown & investigation details', '', '
', '', + '_Metrics below reflect the current branch (after). Δ P95 compares against the base._', + '', + detailTable(rows), + '', + '**Top API waits (after)**', + '', '| API request | Max wait (ms) |', '| --- | ---: |', ...topWaitRows(topWaits), @@ -97,6 +92,46 @@ function buildComment(core) { return `${lines.join('\n')}\n`; } +function headlineTable(total) { + const { before, after } = total; + return [ + '| Metric | Before | After | Change |', + '| --- | ---: | ---: | :--- |', + `| :rocket: Requests/sec | ${formatRate(before?.rps)} | ${formatRate(after?.rps)} | ${changeCell(before?.rps, after?.rps, false)} |`, + `| :stopwatch: Latency P50 | ${formatMsUnit(before?.p50)} | ${formatMsUnit(after?.p50)} | ${changeCell(before?.p50, after?.p50, true)} |`, + `| :stopwatch: Latency P95 | ${formatMsUnit(before?.p95)} | ${formatMsUnit(after?.p95)} | ${changeCell(before?.p95, after?.p95, true)} |`, + ]; +} + +function changeCell(before, after, lowerIsBetter) { + if (before === null || before === undefined || after === null || after === undefined || Number.isNaN(before) || Number.isNaN(after) || before === 0) { + return 'n/a'; + } + + const percent = Number((((after - before) / before) * 100).toFixed(1)); + const label = code(`${percent > 0 ? '+' : ''}${trimNumber(percent)}%`); + + if (Math.abs(percent) <= 5) { + return `:white_circle: ${label}`; + } + + const improved = lowerIsBetter ? percent < 0 : percent > 0; + return `${improved ? ':green_circle:' : ':red_circle:'} ${label}`; +} + +function detailTable(rows) { + return [ + '| Scenario | P50 (ms) | P95 (ms) | Requests | RPS | Δ P95 (ms) |', + '| --- | ---: | ---: | ---: | ---: | ---: |', + ...rows.map(detailRow), + ].join('\n'); +} + +function detailRow(row) { + const values = row.after; + return `| ${row.label} | ${formatMs(values?.p50)} | ${formatMs(values?.p95)} | ${formatCount(values?.iterations)} | ${formatRate(values?.rps)} | ${formatDelta(row.before?.p95, row.after?.p95)} |`; +} + function readSummary(path, core) { if (!fs.existsSync(path)) { return null; @@ -256,23 +291,6 @@ function metricValue(data, metric, stat) { return metricValues(data, metric)?.[stat] ?? null; } -function metricTable(rows, side) { - return [ - '| Scenario | P50 (ms) | P95 (ms) | Requests | RPS |', - '| --- | ---: | ---: | ---: | ---: |', - ...rows.map((row) => metricRow(row, side)), - ].join('\n'); -} - -function metricRow(row, side) { - const values = row[side]; - return `| ${row.label} | ${formatMs(values?.p50)} | ${formatMs(values?.p95)} | ${formatCount(values?.iterations)} | ${formatRate(values?.rps)} |`; -} - -function deltaRow(row) { - return `| ${row.label} | ${formatDelta(row.before?.p95, row.after?.p95)} |`; -} - function topSamples(samples, metric, limit) { const byName = samples.reduce((result, sample) => { if (sample.metric !== metric || typeof sample.data?.value !== 'number') { @@ -310,11 +328,16 @@ function markdownText(value) { } function formatMs(value) { - return formatNumber(value, 2); + return code(formatNumber(value, 2)); +} + +function formatMsUnit(value) { + const formatted = formatNumber(value, 2); + return formatted === 'n/a' ? formatted : code(`${formatted} ms`); } function formatRate(value) { - return formatNumber(value, 2); + return code(formatNumber(value, 2)); } function formatCount(value) { @@ -322,7 +345,7 @@ function formatCount(value) { return 'n/a'; } - return `${Math.round(value)}`; + return code(Math.round(value).toLocaleString('en-US')); } function formatDelta(before, after) { @@ -331,7 +354,8 @@ function formatDelta(before, after) { } const difference = Number((after - before).toFixed(2)); - return `${difference > 0 ? '+' : ''}${trimNumber(difference)}`; + const sign = difference > 0 ? '+' : difference < 0 ? '-' : ''; + return code(`${sign}${formatNumber(Math.abs(difference), 2)}`); } function formatNumber(value, decimals) { @@ -339,7 +363,14 @@ function formatNumber(value, decimals) { return 'n/a'; } - return trimNumber(Number(value).toFixed(decimals)); + return trimNumber(Number(value).toLocaleString('en-US', { + minimumFractionDigits: decimals, + maximumFractionDigits: decimals, + })); +} + +function code(text) { + return text === 'n/a' ? text : `\`${text}\``; } function trimNumber(value) { diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7bc5c91e5..19493e8443 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -686,8 +686,8 @@ jobs: uses: grafana/run-k6-action@de51a7390bdf0ac85a3bef493691bd71d4c7c158 # v1.4.0 env: APPWRITE_ENDPOINT: 'http://localhost:9520/v1' - APPWRITE_BENCHMARK_ITERATIONS: '5' - APPWRITE_BENCHMARK_VUS: '1' + APPWRITE_BENCHMARK_DURATION: '1m' + APPWRITE_BENCHMARK_VUS: '20' APPWRITE_WORKER_TIMEOUT_MS: '120000' APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-before-summary.json' with: @@ -747,8 +747,8 @@ jobs: uses: grafana/run-k6-action@de51a7390bdf0ac85a3bef493691bd71d4c7c158 # v1.4.0 env: APPWRITE_ENDPOINT: 'http://localhost:9520/v1' - APPWRITE_BENCHMARK_ITERATIONS: '5' - APPWRITE_BENCHMARK_VUS: '1' + APPWRITE_BENCHMARK_DURATION: '1m' + APPWRITE_BENCHMARK_VUS: '20' APPWRITE_WORKER_TIMEOUT_MS: '120000' APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH: '../../benchmark-before-summary.json' APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-after-summary.json' diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 1e357f7dc9..e809372a4f 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -18,8 +18,8 @@ const REGION = __ENV.APPWRITE_REGION || 'default'; const REDIRECT_URL = __ENV.APPWRITE_BENCHMARK_REDIRECT_URL || 'http://localhost'; const PASSWORD = __ENV.APPWRITE_BENCHMARK_PASSWORD || 'Password123!'; const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 120000); -const ITERATIONS = Number(__ENV.APPWRITE_BENCHMARK_ITERATIONS || 1); const VUS = Number(__ENV.APPWRITE_BENCHMARK_VUS || 1); +const DURATION = __ENV.APPWRITE_BENCHMARK_DURATION || '1m'; const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || '/tmp/appwrite-k6-summary.json'; const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || ''; const PREVIOUS_SUMMARY = PREVIOUS_SUMMARY_PATH ? loadPreviousSummary(PREVIOUS_SUMMARY_PATH) : null; @@ -32,16 +32,14 @@ export const flowFailures = new Counter('appwrite_benchmark_flow_failures'); export const options = { scenarios: { curated_flows: { - executor: 'shared-iterations', + executor: 'constant-vus', exec: 'curatedFlows', vus: VUS, - iterations: ITERATIONS, - maxDuration: __ENV.APPWRITE_BENCHMARK_MAX_DURATION || '30m', + duration: DURATION, }, }, thresholds: { http_req_failed: ['rate<0.05'], - appwrite_api_duration: ['p(95)<2000'], appwrite_benchmark_flow_failures: ['count<1'], }, };