mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Merge pull request #12697 from appwrite/feat/improve-benchmark
Run benchmark at 50 VUs and clarify the PR comment
This commit is contained in:
3 files changed
+78
-49
No files matched your search
@@ -51,42 +51,37 @@ function buildComment(core) {
|
||||
const baseRef = markdownText(process.env.BENCHMARK_BASE_REF || 'base');
|
||||
const headRef = markdownText(process.env.BENCHMARK_HEAD_REF || 'head');
|
||||
const rows = benchmarkRows(before, after, beforeSamples, afterSamples);
|
||||
const topWaits = topSamples(afterSamples, 'appwrite_api_waiting', 3);
|
||||
const total = rows[0];
|
||||
const topWaits = topSamples(afterSamples, 'appwrite_api_waiting', 5);
|
||||
const lines = [
|
||||
marker,
|
||||
'## :sparkles: Benchmark results',
|
||||
'',
|
||||
`Comparing ${baseRef} (before) to ${headRef} (after).`,
|
||||
`Comparing \`${baseRef}\` (before) → \`${headRef}\` (after).`,
|
||||
'',
|
||||
];
|
||||
|
||||
if (before === null) {
|
||||
lines.push('> Before benchmark did not complete; showing current branch metrics only.', '');
|
||||
lines.push('> :warning: Before benchmark did not complete — change column unavailable.', '');
|
||||
}
|
||||
if (after === null) {
|
||||
lines.push('> Current branch benchmark did not complete; showing available metrics only.', '');
|
||||
lines.push('> :warning: Current branch benchmark did not complete — showing available metrics only.', '');
|
||||
}
|
||||
|
||||
lines.push(
|
||||
'**Before**',
|
||||
'',
|
||||
metricTable(rows, 'before'),
|
||||
'',
|
||||
'**After**',
|
||||
'',
|
||||
metricTable(rows, 'after'),
|
||||
'',
|
||||
'**Delta**',
|
||||
'',
|
||||
'| Scenario | P95 delta (ms) |',
|
||||
'| --- | ---: |',
|
||||
...rows.map(deltaRow),
|
||||
...headlineTable(total),
|
||||
'',
|
||||
'<details>',
|
||||
'<summary><strong>Top API waits</strong></summary>',
|
||||
'<summary><strong>Per-scenario breakdown & investigation details</strong></summary>',
|
||||
'',
|
||||
'<br>',
|
||||
'',
|
||||
'_Metrics below reflect the current branch (after). Δ P95 compares against the base._',
|
||||
'',
|
||||
detailTable(rows),
|
||||
'',
|
||||
'**Top API waits (after)**',
|
||||
'',
|
||||
'| API request | Max wait (ms) |',
|
||||
'| --- | ---: |',
|
||||
...topWaitRows(topWaits),
|
||||
@@ -97,6 +92,46 @@ function buildComment(core) {
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
function headlineTable(total) {
|
||||
const { before, after } = total;
|
||||
return [
|
||||
'| Metric | Before | After | Change |',
|
||||
'| --- | ---: | ---: | :--- |',
|
||||
`| :rocket: Requests/sec | ${formatRate(before?.rps)} | ${formatRate(after?.rps)} | ${changeCell(before?.rps, after?.rps, false)} |`,
|
||||
`| :stopwatch: Latency P50 | ${formatMsUnit(before?.p50)} | ${formatMsUnit(after?.p50)} | ${changeCell(before?.p50, after?.p50, true)} |`,
|
||||
`| :stopwatch: Latency P95 | ${formatMsUnit(before?.p95)} | ${formatMsUnit(after?.p95)} | ${changeCell(before?.p95, after?.p95, true)} |`,
|
||||
];
|
||||
}
|
||||
|
||||
function changeCell(before, after, lowerIsBetter) {
|
||||
if (before === null || before === undefined || after === null || after === undefined || Number.isNaN(before) || Number.isNaN(after) || before === 0) {
|
||||
return 'n/a';
|
||||
}
|
||||
|
||||
const percent = Number((((after - before) / before) * 100).toFixed(1));
|
||||
const label = code(`${percent > 0 ? '+' : ''}${trimNumber(percent)}%`);
|
||||
|
||||
if (Math.abs(percent) <= 5) {
|
||||
return `:white_circle: ${label}`;
|
||||
}
|
||||
|
||||
const improved = lowerIsBetter ? percent < 0 : percent > 0;
|
||||
return `${improved ? ':green_circle:' : ':red_circle:'} ${label}`;
|
||||
}
|
||||
|
||||
function detailTable(rows) {
|
||||
return [
|
||||
'| Scenario | P50 (ms) | P95 (ms) | Requests | RPS | Δ P95 (ms) |',
|
||||
'| --- | ---: | ---: | ---: | ---: | ---: |',
|
||||
...rows.map(detailRow),
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function detailRow(row) {
|
||||
const values = row.after;
|
||||
return `| ${row.label} | ${formatMs(values?.p50)} | ${formatMs(values?.p95)} | ${formatCount(values?.iterations)} | ${formatRate(values?.rps)} | ${formatDelta(row.before?.p95, row.after?.p95)} |`;
|
||||
}
|
||||
|
||||
function readSummary(path, core) {
|
||||
if (!fs.existsSync(path)) {
|
||||
return null;
|
||||
@@ -256,23 +291,6 @@ function metricValue(data, metric, stat) {
|
||||
return metricValues(data, metric)?.[stat] ?? null;
|
||||
}
|
||||
|
||||
function metricTable(rows, side) {
|
||||
return [
|
||||
'| Scenario | P50 (ms) | P95 (ms) | Requests | RPS |',
|
||||
'| --- | ---: | ---: | ---: | ---: |',
|
||||
...rows.map((row) => metricRow(row, side)),
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function metricRow(row, side) {
|
||||
const values = row[side];
|
||||
return `| ${row.label} | ${formatMs(values?.p50)} | ${formatMs(values?.p95)} | ${formatCount(values?.iterations)} | ${formatRate(values?.rps)} |`;
|
||||
}
|
||||
|
||||
function deltaRow(row) {
|
||||
return `| ${row.label} | ${formatDelta(row.before?.p95, row.after?.p95)} |`;
|
||||
}
|
||||
|
||||
function topSamples(samples, metric, limit) {
|
||||
const byName = samples.reduce((result, sample) => {
|
||||
if (sample.metric !== metric || typeof sample.data?.value !== 'number') {
|
||||
@@ -310,11 +328,16 @@ function markdownText(value) {
|
||||
}
|
||||
|
||||
function formatMs(value) {
|
||||
return formatNumber(value, 2);
|
||||
return code(formatNumber(value, 2));
|
||||
}
|
||||
|
||||
function formatMsUnit(value) {
|
||||
const formatted = formatNumber(value, 2);
|
||||
return formatted === 'n/a' ? formatted : code(`${formatted} ms`);
|
||||
}
|
||||
|
||||
function formatRate(value) {
|
||||
return formatNumber(value, 2);
|
||||
return code(formatNumber(value, 2));
|
||||
}
|
||||
|
||||
function formatCount(value) {
|
||||
@@ -322,7 +345,7 @@ function formatCount(value) {
|
||||
return 'n/a';
|
||||
}
|
||||
|
||||
return `${Math.round(value)}`;
|
||||
return code(Math.round(value).toLocaleString('en-US'));
|
||||
}
|
||||
|
||||
function formatDelta(before, after) {
|
||||
@@ -331,7 +354,8 @@ function formatDelta(before, after) {
|
||||
}
|
||||
|
||||
const difference = Number((after - before).toFixed(2));
|
||||
return `${difference > 0 ? '+' : ''}${trimNumber(difference)}`;
|
||||
const sign = difference > 0 ? '+' : difference < 0 ? '-' : '';
|
||||
return code(`${sign}${formatNumber(Math.abs(difference), 2)}`);
|
||||
}
|
||||
|
||||
function formatNumber(value, decimals) {
|
||||
@@ -339,7 +363,14 @@ function formatNumber(value, decimals) {
|
||||
return 'n/a';
|
||||
}
|
||||
|
||||
return trimNumber(Number(value).toFixed(decimals));
|
||||
return trimNumber(Number(value).toLocaleString('en-US', {
|
||||
minimumFractionDigits: decimals,
|
||||
maximumFractionDigits: decimals,
|
||||
}));
|
||||
}
|
||||
|
||||
function code(text) {
|
||||
return text === 'n/a' ? text : `\`${text}\``;
|
||||
}
|
||||
|
||||
function trimNumber(value) {
|
||||
|
||||
@@ -686,8 +686,8 @@ jobs:
|
||||
uses: grafana/run-k6-action@de51a7390bdf0ac85a3bef493691bd71d4c7c158 # v1.4.0
|
||||
env:
|
||||
APPWRITE_ENDPOINT: 'http://localhost:9520/v1'
|
||||
APPWRITE_BENCHMARK_ITERATIONS: '5'
|
||||
APPWRITE_BENCHMARK_VUS: '1'
|
||||
APPWRITE_BENCHMARK_DURATION: '1m'
|
||||
APPWRITE_BENCHMARK_VUS: '20'
|
||||
APPWRITE_WORKER_TIMEOUT_MS: '120000'
|
||||
APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-before-summary.json'
|
||||
with:
|
||||
@@ -747,8 +747,8 @@ jobs:
|
||||
uses: grafana/run-k6-action@de51a7390bdf0ac85a3bef493691bd71d4c7c158 # v1.4.0
|
||||
env:
|
||||
APPWRITE_ENDPOINT: 'http://localhost:9520/v1'
|
||||
APPWRITE_BENCHMARK_ITERATIONS: '5'
|
||||
APPWRITE_BENCHMARK_VUS: '1'
|
||||
APPWRITE_BENCHMARK_DURATION: '1m'
|
||||
APPWRITE_BENCHMARK_VUS: '20'
|
||||
APPWRITE_WORKER_TIMEOUT_MS: '120000'
|
||||
APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH: '../../benchmark-before-summary.json'
|
||||
APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-after-summary.json'
|
||||
|
||||
@@ -18,8 +18,8 @@ const REGION = __ENV.APPWRITE_REGION || 'default';
|
||||
const REDIRECT_URL = __ENV.APPWRITE_BENCHMARK_REDIRECT_URL || 'http://localhost';
|
||||
const PASSWORD = __ENV.APPWRITE_BENCHMARK_PASSWORD || 'Password123!';
|
||||
const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 120000);
|
||||
const ITERATIONS = Number(__ENV.APPWRITE_BENCHMARK_ITERATIONS || 1);
|
||||
const VUS = Number(__ENV.APPWRITE_BENCHMARK_VUS || 1);
|
||||
const DURATION = __ENV.APPWRITE_BENCHMARK_DURATION || '1m';
|
||||
const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || '/tmp/appwrite-k6-summary.json';
|
||||
const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || '';
|
||||
const PREVIOUS_SUMMARY = PREVIOUS_SUMMARY_PATH ? loadPreviousSummary(PREVIOUS_SUMMARY_PATH) : null;
|
||||
@@ -32,16 +32,14 @@ export const flowFailures = new Counter('appwrite_benchmark_flow_failures');
|
||||
export const options = {
|
||||
scenarios: {
|
||||
curated_flows: {
|
||||
executor: 'shared-iterations',
|
||||
executor: 'constant-vus',
|
||||
exec: 'curatedFlows',
|
||||
vus: VUS,
|
||||
iterations: ITERATIONS,
|
||||
maxDuration: __ENV.APPWRITE_BENCHMARK_MAX_DURATION || '30m',
|
||||
duration: DURATION,
|
||||
},
|
||||
},
|
||||
thresholds: {
|
||||
http_req_failed: ['rate<0.05'],
|
||||
appwrite_api_duration: ['p(95)<2000'],
|
||||
appwrite_benchmark_flow_failures: ['count<1'],
|
||||
},
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user