Merge pull request #12697 from appwrite/feat/improve-benchmark

Run benchmark at 50 VUs and clarify the PR comment
This commit is contained in:
Luke B. Silver authored and GitHub committed 2026-06-26 15:55:31 +01:00
commit a81a9de9fc
3 files changed
+78 -49

No files matched your search

+71 -40
View File
@@ -51,42 +51,37 @@ function buildComment(core) {
const baseRef = markdownText(process.env.BENCHMARK_BASE_REF || 'base');
const headRef = markdownText(process.env.BENCHMARK_HEAD_REF || 'head');
const rows = benchmarkRows(before, after, beforeSamples, afterSamples);
const topWaits = topSamples(afterSamples, 'appwrite_api_waiting', 3);
const total = rows[0];
const topWaits = topSamples(afterSamples, 'appwrite_api_waiting', 5);
const lines = [
marker,
'## :sparkles: Benchmark results',
'',
`Comparing ${baseRef} (before) to ${headRef} (after).`,
`Comparing \`${baseRef}\` (before) → \`${headRef}\` (after).`,
'',
];
if (before === null) {
lines.push('> Before benchmark did not complete; showing current branch metrics only.', '');
lines.push('> :warning: Before benchmark did not complete — change column unavailable.', '');
}
if (after === null) {
lines.push('> Current branch benchmark did not complete; showing available metrics only.', '');
lines.push('> :warning: Current branch benchmark did not complete — showing available metrics only.', '');
}
lines.push(
'**Before**',
'',
metricTable(rows, 'before'),
'',
'**After**',
'',
metricTable(rows, 'after'),
'',
'**Delta**',
'',
'| Scenario | P95 delta (ms) |',
'| --- | ---: |',
...rows.map(deltaRow),
...headlineTable(total),
'',
'<details>',
'<summary><strong>Top API waits</strong></summary>',
'<summary><strong>Per-scenario breakdown & investigation details</strong></summary>',
'',
'<br>',
'',
'_Metrics below reflect the current branch (after). Δ P95 compares against the base._',
'',
detailTable(rows),
'',
'**Top API waits (after)**',
'',
'| API request | Max wait (ms) |',
'| --- | ---: |',
...topWaitRows(topWaits),
@@ -97,6 +92,46 @@ function buildComment(core) {
return `${lines.join('\n')}\n`;
}
function headlineTable(total) {
const { before, after } = total;
return [
'| Metric | Before | After | Change |',
'| --- | ---: | ---: | :--- |',
`| :rocket: Requests/sec | ${formatRate(before?.rps)} | ${formatRate(after?.rps)} | ${changeCell(before?.rps, after?.rps, false)} |`,
`| :stopwatch: Latency P50 | ${formatMsUnit(before?.p50)} | ${formatMsUnit(after?.p50)} | ${changeCell(before?.p50, after?.p50, true)} |`,
`| :stopwatch: Latency P95 | ${formatMsUnit(before?.p95)} | ${formatMsUnit(after?.p95)} | ${changeCell(before?.p95, after?.p95, true)} |`,
];
}
function changeCell(before, after, lowerIsBetter) {
if (before === null || before === undefined || after === null || after === undefined || Number.isNaN(before) || Number.isNaN(after) || before === 0) {
return 'n/a';
}
const percent = Number((((after - before) / before) * 100).toFixed(1));
const label = code(`${percent > 0 ? '+' : ''}${trimNumber(percent)}%`);
if (Math.abs(percent) <= 5) {
return `:white_circle: ${label}`;
}
const improved = lowerIsBetter ? percent < 0 : percent > 0;
return `${improved ? ':green_circle:' : ':red_circle:'} ${label}`;
}
function detailTable(rows) {
return [
'| Scenario | P50 (ms) | P95 (ms) | Requests | RPS | Δ P95 (ms) |',
'| --- | ---: | ---: | ---: | ---: | ---: |',
...rows.map(detailRow),
].join('\n');
}
function detailRow(row) {
const values = row.after;
return `| ${row.label} | ${formatMs(values?.p50)} | ${formatMs(values?.p95)} | ${formatCount(values?.iterations)} | ${formatRate(values?.rps)} | ${formatDelta(row.before?.p95, row.after?.p95)} |`;
}
function readSummary(path, core) {
if (!fs.existsSync(path)) {
return null;
@@ -256,23 +291,6 @@ function metricValue(data, metric, stat) {
return metricValues(data, metric)?.[stat] ?? null;
}
function metricTable(rows, side) {
return [
'| Scenario | P50 (ms) | P95 (ms) | Requests | RPS |',
'| --- | ---: | ---: | ---: | ---: |',
...rows.map((row) => metricRow(row, side)),
].join('\n');
}
function metricRow(row, side) {
const values = row[side];
return `| ${row.label} | ${formatMs(values?.p50)} | ${formatMs(values?.p95)} | ${formatCount(values?.iterations)} | ${formatRate(values?.rps)} |`;
}
function deltaRow(row) {
return `| ${row.label} | ${formatDelta(row.before?.p95, row.after?.p95)} |`;
}
function topSamples(samples, metric, limit) {
const byName = samples.reduce((result, sample) => {
if (sample.metric !== metric || typeof sample.data?.value !== 'number') {
@@ -310,11 +328,16 @@ function markdownText(value) {
}
function formatMs(value) {
return formatNumber(value, 2);
return code(formatNumber(value, 2));
}
function formatMsUnit(value) {
const formatted = formatNumber(value, 2);
return formatted === 'n/a' ? formatted : code(`${formatted} ms`);
}
function formatRate(value) {
return formatNumber(value, 2);
return code(formatNumber(value, 2));
}
function formatCount(value) {
@@ -322,7 +345,7 @@ function formatCount(value) {
return 'n/a';
}
return `${Math.round(value)}`;
return code(Math.round(value).toLocaleString('en-US'));
}
function formatDelta(before, after) {
@@ -331,7 +354,8 @@ function formatDelta(before, after) {
}
const difference = Number((after - before).toFixed(2));
return `${difference > 0 ? '+' : ''}${trimNumber(difference)}`;
const sign = difference > 0 ? '+' : difference < 0 ? '-' : '';
return code(`${sign}${formatNumber(Math.abs(difference), 2)}`);
}
function formatNumber(value, decimals) {
@@ -339,7 +363,14 @@ function formatNumber(value, decimals) {
return 'n/a';
}
return trimNumber(Number(value).toFixed(decimals));
return trimNumber(Number(value).toLocaleString('en-US', {
minimumFractionDigits: decimals,
maximumFractionDigits: decimals,
}));
}
function code(text) {
return text === 'n/a' ? text : `\`${text}\``;
}
function trimNumber(value) {
+4 -4
View File
@@ -686,8 +686,8 @@ jobs:
uses: grafana/run-k6-action@de51a7390bdf0ac85a3bef493691bd71d4c7c158 # v1.4.0
env:
APPWRITE_ENDPOINT: 'http://localhost:9520/v1'
APPWRITE_BENCHMARK_ITERATIONS: '5'
APPWRITE_BENCHMARK_VUS: '1'
APPWRITE_BENCHMARK_DURATION: '1m'
APPWRITE_BENCHMARK_VUS: '20'
APPWRITE_WORKER_TIMEOUT_MS: '120000'
APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-before-summary.json'
with:
@@ -747,8 +747,8 @@ jobs:
uses: grafana/run-k6-action@de51a7390bdf0ac85a3bef493691bd71d4c7c158 # v1.4.0
env:
APPWRITE_ENDPOINT: 'http://localhost:9520/v1'
APPWRITE_BENCHMARK_ITERATIONS: '5'
APPWRITE_BENCHMARK_VUS: '1'
APPWRITE_BENCHMARK_DURATION: '1m'
APPWRITE_BENCHMARK_VUS: '20'
APPWRITE_WORKER_TIMEOUT_MS: '120000'
APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH: '../../benchmark-before-summary.json'
APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-after-summary.json'
+3 -5
View File
@@ -18,8 +18,8 @@ const REGION = __ENV.APPWRITE_REGION || 'default';
const REDIRECT_URL = __ENV.APPWRITE_BENCHMARK_REDIRECT_URL || 'http://localhost';
const PASSWORD = __ENV.APPWRITE_BENCHMARK_PASSWORD || 'Password123!';
const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 120000);
const ITERATIONS = Number(__ENV.APPWRITE_BENCHMARK_ITERATIONS || 1);
const VUS = Number(__ENV.APPWRITE_BENCHMARK_VUS || 1);
const DURATION = __ENV.APPWRITE_BENCHMARK_DURATION || '1m';
const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || '/tmp/appwrite-k6-summary.json';
const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || '';
const PREVIOUS_SUMMARY = PREVIOUS_SUMMARY_PATH ? loadPreviousSummary(PREVIOUS_SUMMARY_PATH) : null;
@@ -32,16 +32,14 @@ export const flowFailures = new Counter('appwrite_benchmark_flow_failures');
export const options = {
scenarios: {
curated_flows: {
executor: 'shared-iterations',
executor: 'constant-vus',
exec: 'curatedFlows',
vus: VUS,
iterations: ITERATIONS,
maxDuration: __ENV.APPWRITE_BENCHMARK_MAX_DURATION || '30m',
duration: DURATION,
},
},
thresholds: {
http_req_failed: ['rate<0.05'],
appwrite_api_duration: ['p(95)<2000'],
appwrite_benchmark_flow_failures: ['count<1'],
},
};