diff --git a/.github/workflows/benchmark-comment.js b/.github/workflows/benchmark-comment.js index 4a9c920365..fa0fea87f4 100644 --- a/.github/workflows/benchmark-comment.js +++ b/.github/workflows/benchmark-comment.js @@ -1,7 +1,7 @@ const fs = require('fs'); const marker = ''; -const serviceLabels = ['Account', 'TablesDB', 'Storage', 'Functions', 'Sites', 'Health']; +const serviceLabels = ['Account', 'TablesDB', 'Storage', 'Functions']; module.exports = async ({ github, context, core }) => { const body = buildComment(core); @@ -51,7 +51,7 @@ function buildComment(core) { const baseRef = markdownText(process.env.BENCHMARK_BASE_REF || 'base'); const headRef = markdownText(process.env.BENCHMARK_HEAD_REF || 'head'); const rows = benchmarkRows(before, after, beforeSamples, afterSamples); - const topWaits = topSamples(afterSamples, 'appwrite_http_waiting', 3); + const topWaits = topSamples(afterSamples, 'appwrite_api_waiting', 3); const lines = [ marker, '## :sparkles: Benchmark results', @@ -68,22 +68,26 @@ function buildComment(core) { } lines.push( - '| Scenario | Before P50 (ms) | Before P95 (ms) | After P50 (ms) | After P95 (ms) | Delta P95 (ms) | After iterations | After RPS |', - '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', - ...rows.map(comparisonRow), + '**Before**', + '', + metricTable(rows, 'before'), + '', + '**After**', + '', + metricTable(rows, 'after'), + '', + '**Delta**', + '', + '| Scenario | P95 delta (ms) |', + '| --- | ---: |', + ...rows.map(deltaRow), '', '
', - 'Current run details', + 'Top API waits', '', '
', '', - '| Scenario | P50 (ms) | P95 (ms) | Iterations | RPS |', - '| --- | ---: | ---: | ---: | ---: |', - ...rows.map(detailRow), - '', - '**Top 3 request waits**', - '', - '| Request | Max wait (ms) |', + '| API request | Max wait (ms) |', '| --- | ---: |', ...topWaitRows(topWaits), '', @@ -133,11 +137,6 @@ function benchmarkRows(before, after, beforeSamples, afterSamples) { const beforeServices = serviceStats(beforeSamples); const afterServices = serviceStats(afterSamples); return [ - { - label: 'Load test', - before: summaryStats(before, 'appwrite_http_duration', 'iterations', 'http_reqs'), - after: summaryStats(after, 'appwrite_http_duration', 'iterations', 'http_reqs'), - }, { label: 'API total', before: apiSampleStats(beforeSamples) || summaryStats(before, 'appwrite_api_duration'), @@ -148,16 +147,6 @@ function benchmarkRows(before, after, beforeSamples, afterSamples) { before: beforeServices.get(label) || null, after: afterServices.get(label) || null, })), - { - label: 'TablesDB schema', - before: summaryStats(before, 'appwrite_worker_tables_duration', 'appwrite_worker_tables_samples', 'appwrite_worker_tables_samples'), - after: summaryStats(after, 'appwrite_worker_tables_duration', 'appwrite_worker_tables_samples', 'appwrite_worker_tables_samples'), - }, - { - label: 'Mail delivery', - before: summaryStats(before, 'appwrite_worker_mails_duration', 'appwrite_worker_mails_samples', 'appwrite_worker_mails_samples'), - after: summaryStats(after, 'appwrite_worker_mails_duration', 'appwrite_worker_mails_samples', 'appwrite_worker_mails_samples'), - }, ]; } @@ -205,14 +194,15 @@ function serviceStats(samples) { } function apiSampleStats(samples) { - const values = samples - .filter((sample) => sample.metric === 'appwrite_api_duration' && typeof sample.data?.value === 'number') - .map((sample) => sample.data.value); + const apiSamples = samples.filter((sample) => { + return sample.metric === 'appwrite_api_duration' && typeof sample.data?.value === 'number'; + }); + const values = apiSamples.map((sample) => sample.data.value); if (values.length === 0) { return null; } - const durationSeconds = sampleWindowSeconds(samples); + const durationSeconds = sampleWindowSeconds(apiSamples); return { p50: percentile(values, 50), p95: percentile(values, 95), @@ -234,12 +224,6 @@ function serviceFromName(name) { if (name.startsWith('functions.')) { return 'Functions'; } - if (name.startsWith('sites.')) { - return 'Sites'; - } - if (name.startsWith('health.')) { - return 'Health'; - } return null; } @@ -272,12 +256,21 @@ function metricValue(data, metric, stat) { return metricValues(data, metric)?.[stat] ?? null; } -function comparisonRow(row) { - return `| ${row.label} | ${formatMs(row.before?.p50)} | ${formatMs(row.before?.p95)} | ${formatMs(row.after?.p50)} | ${formatMs(row.after?.p95)} | ${formatDelta(row.before?.p95, row.after?.p95)} | ${formatCount(row.after?.iterations)} | ${formatRate(row.after?.rps)} |`; +function metricTable(rows, side) { + return [ + '| Scenario | P50 (ms) | P95 (ms) | Iterations | RPS |', + '| --- | ---: | ---: | ---: | ---: |', + ...rows.map((row) => metricRow(row, side)), + ].join('\n'); } -function detailRow(row) { - return `| ${row.label} | ${formatMs(row.after?.p50)} | ${formatMs(row.after?.p95)} | ${formatCount(row.after?.iterations)} | ${formatRate(row.after?.rps)} |`; +function metricRow(row, side) { + const values = row[side]; + return `| ${row.label} | ${formatMs(values?.p50)} | ${formatMs(values?.p95)} | ${formatCount(values?.iterations)} | ${formatRate(values?.rps)} |`; +} + +function deltaRow(row) { + return `| ${row.label} | ${formatDelta(row.before?.p95, row.after?.p95)} |`; } function topSamples(samples, metric, limit) { diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8e5cb38fb..97ca5546d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -712,7 +712,6 @@ jobs: _APP_DOMAIN: localhost _APP_CONSOLE_DOMAIN: localhost _APP_DOMAIN_FUNCTIONS: functions.localhost - _APP_DOMAIN_SITES: sites.localhost run: | docker tag ${{ env.IMAGE }}:before ${{ env.IMAGE }} docker compose up -d --wait --no-build @@ -726,7 +725,6 @@ jobs: uses: grafana/run-k6-action@v1 env: APPWRITE_ENDPOINT: 'http://localhost/v1' - APPWRITE_MAILDEV_ENDPOINT: 'http://localhost:9503/email' APPWRITE_BENCHMARK_ITERATIONS: '1' APPWRITE_BENCHMARK_VUS: '1' APPWRITE_WORKER_TIMEOUT_MS: '120000' @@ -768,7 +766,6 @@ jobs: _APP_DOMAIN: localhost _APP_CONSOLE_DOMAIN: localhost _APP_DOMAIN_FUNCTIONS: functions.localhost - _APP_DOMAIN_SITES: sites.localhost run: | docker tag ${{ env.IMAGE }}:after ${{ env.IMAGE }} docker compose up -d --wait --no-build @@ -779,7 +776,6 @@ jobs: uses: grafana/run-k6-action@v1 env: APPWRITE_ENDPOINT: 'http://localhost/v1' - APPWRITE_MAILDEV_ENDPOINT: 'http://localhost:9503/email' APPWRITE_BENCHMARK_ITERATIONS: '1' APPWRITE_BENCHMARK_VUS: '1' APPWRITE_WORKER_TIMEOUT_MS: '120000' diff --git a/tests/benchmarks/http-local.sh b/tests/benchmarks/http-local.sh index acb8a07058..734c825fda 100755 --- a/tests/benchmarks/http-local.sh +++ b/tests/benchmarks/http-local.sh @@ -6,7 +6,6 @@ export K6_WEB_DASHBOARD_HOST="${K6_WEB_DASHBOARD_HOST:-127.0.0.1}" export K6_WEB_DASHBOARD_PORT="${K6_WEB_DASHBOARD_PORT:-5665}" export K6_WEB_DASHBOARD_EXPORT="${K6_WEB_DASHBOARD_EXPORT:-/tmp/appwrite-k6-report.html}" export APPWRITE_ENDPOINT="${APPWRITE_ENDPOINT:-http://localhost/v1}" -export APPWRITE_MAILDEV_ENDPOINT="${APPWRITE_MAILDEV_ENDPOINT:-http://localhost:9503/email}" export APPWRITE_WORKER_TIMEOUT_MS="${APPWRITE_WORKER_TIMEOUT_MS:-120000}" export APPWRITE_BENCHMARK_SUMMARY_PATH="${APPWRITE_BENCHMARK_SUMMARY_PATH:-/tmp/appwrite-k6-summary.json}" diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 3592d0248e..ccab88c011 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -12,12 +12,10 @@ import encoding from 'k6/encoding'; import { Counter, Trend } from 'k6/metrics'; const ENDPOINT = (__ENV.APPWRITE_ENDPOINT || 'http://localhost/v1').replace(/\/+$/, ''); -const MAILDEV_ENDPOINT = __ENV.APPWRITE_MAILDEV_ENDPOINT || 'http://localhost:9503/email'; const CONSOLE_PROJECT = __ENV.APPWRITE_CONSOLE_PROJECT || 'console'; const REGION = __ENV.APPWRITE_REGION || 'default'; const REDIRECT_URL = __ENV.APPWRITE_BENCHMARK_REDIRECT_URL || 'http://localhost'; const PASSWORD = __ENV.APPWRITE_BENCHMARK_PASSWORD || 'Password123!'; -const MAIL_TIMEOUT_MS = Number(__ENV.APPWRITE_MAIL_TIMEOUT_MS || 20000); const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 120000); const ITERATIONS = Number(__ENV.APPWRITE_BENCHMARK_ITERATIONS || 1); const VUS = Number(__ENV.APPWRITE_BENCHMARK_VUS || 1); @@ -25,13 +23,9 @@ const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || '/tmp/appwrite-k6- const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || SUMMARY_PATH; const PREVIOUS_SUMMARY = loadPreviousSummary(); -export const httpDuration = new Trend('appwrite_http_duration', true); export const httpWaiting = new Trend('appwrite_http_waiting', true); export const apiDuration = new Trend('appwrite_api_duration', true); -export const tablesWorkerDuration = new Trend('appwrite_worker_tables_duration', true); -export const mailsWorkerDuration = new Trend('appwrite_worker_mails_duration', true); -export const tablesWorkerSamples = new Counter('appwrite_worker_tables_samples'); -export const mailsWorkerSamples = new Counter('appwrite_worker_mails_samples'); +export const apiWaiting = new Trend('appwrite_api_waiting', true); export const flowFailures = new Counter('appwrite_benchmark_flow_failures'); export const options = { @@ -79,15 +73,12 @@ const API_SCOPES = [ 'buckets.write', 'functions.read', 'functions.write', - 'sites.read', - 'sites.write', 'log.read', 'log.write', 'execution.read', 'execution.write', 'locale.read', 'avatars.read', - 'health.read', 'rules.read', 'rules.write', 'migrations.read', @@ -179,41 +170,60 @@ export function setup() { hostname: hostnameFromUrl(REDIRECT_URL), }, apiHeaders, [201, 409], 'setup.project.platforms.web.create'); - const smtp = rawRequest('PATCH', `/projects/${projectId}/smtp`, { - enabled: true, - senderName: 'Benchmark', - senderEmail: 'benchmark@appwrite.io', - replyTo: 'benchmark@appwrite.io', - host: __ENV.APPWRITE_SMTP_HOST || 'maildev', - port: Number(__ENV.APPWRITE_SMTP_PORT || 1025), - username: __ENV.APPWRITE_SMTP_USERNAME || 'user', - password: __ENV.APPWRITE_SMTP_PASSWORD || 'password', - ...(String(__ENV.APPWRITE_SMTP_SECURE || '') !== '' ? { secure: __ENV.APPWRITE_SMTP_SECURE } : {}), - }, consoleSessionHeaders, 'setup.projects.smtp.update'); - - if (smtp.status !== 200) { - console.warn(`Custom SMTP was not enabled (${smtp.status}). Mail worker timings may be unavailable.`); - } + const tablesDb = setupTablesDb(apiHeaders); return { runId, teamId, projectId, + databaseId: tablesDb.databaseId, + tableId: tablesDb.tableId, consoleSessionHeaders, apiHeaders, platformStatus: platform.status, }; } +function setupTablesDb(apiHeaders) { + const databaseId = unique('tdb'); + const tableId = unique('tbl'); + + setupApi('POST', '/tablesdb', { databaseId, name: 'Benchmark TablesDB' }, apiHeaders, [201], 'setup.tablesdb.create'); + setupApi('POST', `/tablesdb/${databaseId}/tables`, { + tableId, + name: 'Benchmark Table', + permissions: BASE_PERMISSIONS, + rowSecurity: false, + }, apiHeaders, [201], 'setup.tablesdb.tables.create'); + + const columns = [ + ['string', 'title', { size: 128 }], + ['integer', 'quantity', { min: 0, max: 100000 }], + ['email', 'email', {}], + ['boolean', 'active', {}], + ]; + + for (const [type, key, extra] of columns) { + setupApi('POST', `/tablesdb/${databaseId}/tables/${tableId}/columns/${type}`, { + key, + required: false, + array: false, + ...extra, + }, apiHeaders, [202], `setup.tablesdb.columns.${type}.create`); + waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/columns/${key}`, apiHeaders, 'available', WORKER_TIMEOUT_MS, `setup.tablesdb.columns.${type}.wait`); + } + + return { databaseId, tableId }; +} + export function curatedFlows(data) { const ctx = { ...data }; try { - group('account and mail flow', () => accountFlow(ctx)); + group('account flow', () => accountFlow(ctx)); group('tablesdb rows flow', () => tablesDbFlow(ctx)); group('storage files and tokens flow', () => storageFlow(ctx)); - group('functions and sites control-plane flow', () => computeFlow(ctx)); - group('health and queue probes', () => healthFlow(ctx)); + group('functions control-plane flow', () => computeFlow(ctx)); } catch (error) { flowFailures.add(1); throw error; @@ -230,16 +240,6 @@ export function teardown(data) { } } -function recordTablesWorkerDuration(duration, tags) { - tablesWorkerDuration.add(duration, tags); - tablesWorkerSamples.add(1, tags); -} - -function recordMailsWorkerDuration(duration, tags) { - mailsWorkerDuration.add(duration, tags); - mailsWorkerSamples.add(1, tags); -} - function accountFlow(ctx) { const userId = unique('user'); const email = `bench-user-${unique('mail')}@example.com`; @@ -271,109 +271,14 @@ function accountFlow(ctx) { api('PATCH', '/account/prefs', { prefs: { benchmark: true, runId: ctx.runId } }, sessionHeaders, [200], 'account.prefs.update'); api('PATCH', '/account/name', { name: 'Benchmark User Updated' }, sessionHeaders, [200], 'account.name.update'); api('PATCH', '/account/password', { password: `${PASSWORD}2`, oldPassword: PASSWORD }, sessionHeaders, [200], 'account.password.update'); - - const verificationStarted = Date.now(); - api('POST', '/account/verifications/email', { url: REDIRECT_URL }, sessionHeaders, [201], 'account.emailVerification.create'); - const verificationEmail = waitForEmail(email, (message) => { - return includes(message.subject, 'verify') - || includes(message.subject, 'verification') - || includes(message.html, 'verify') - || includes(message.html, 'verification') - || includes(message.text, 'verify') - || includes(message.text, 'verification'); - }, MAIL_TIMEOUT_MS); - recordMailsWorkerDuration(Date.now() - verificationStarted, { job: 'email_verification' }); - - const verification = extractQueryParams(verificationEmail); - if (verification.userId && verification.secret) { - api('PUT', '/account/verifications/email', { - userId: verification.userId, - secret: verification.secret, - }, sessionHeaders, [200], 'account.emailVerification.update'); - } - - const recoveryStarted = Date.now(); - api('POST', '/account/recovery', { email, url: REDIRECT_URL }, headers, [201], 'account.recovery.create'); - const recoveryEmail = waitForEmail(email, (message) => { - return includes(message.subject, 'recovery') - || includes(message.subject, 'recover') - || includes(message.subject, 'reset') - || includes(message.html, 'recovery') - || includes(message.html, 'recover') - || includes(message.html, 'reset') - || includes(message.text, 'recovery') - || includes(message.text, 'recover') - || includes(message.text, 'reset'); - }, MAIL_TIMEOUT_MS); - recordMailsWorkerDuration(Date.now() - recoveryStarted, { job: 'password_recovery' }); - - const recovery = extractQueryParams(recoveryEmail); - if (recovery.userId && recovery.secret) { - api('DELETE', '/account/sessions/current', null, sessionHeaders, [204], 'account.sessions.current.delete'); - - api('PUT', '/account/recovery', { - userId: recovery.userId, - secret: recovery.secret, - password: `${PASSWORD}3`, - }, headers, [200], 'account.recovery.update'); - - const recoveredSession = api('POST', '/account/sessions/email', { - email, - password: `${PASSWORD}3`, - }, headers, [201], 'account.sessions.email.recovered'); - - ctx.sessionHeaders = { - ...headers, - Cookie: cookieHeader(recoveredSession), - }; - - } } function tablesDbFlow(ctx) { requireSession(ctx, 'tablesDbFlow'); - const databaseId = unique('tdb'); - const tableId = unique('tbl'); + const databaseId = ctx.databaseId; + const tableId = ctx.tableId; const rowId = unique('row'); - const indexKey = unique('tidx'); - - api('POST', '/tablesdb', { databaseId, name: 'Benchmark TablesDB' }, ctx.apiHeaders, [201], 'tablesdb.create'); - api('POST', `/tablesdb/${databaseId}/tables`, { - tableId, - name: 'Benchmark Table', - permissions: BASE_PERMISSIONS, - rowSecurity: false, - }, ctx.apiHeaders, [201], 'tablesdb.tables.create'); - - const columns = [ - ['string', 'title', { size: 128 }], - ['integer', 'quantity', { min: 0, max: 100000 }], - ['email', 'email', {}], - ['boolean', 'active', {}], - ]; - - for (const [type, key, extra] of columns) { - const started = Date.now(); - api('POST', `/tablesdb/${databaseId}/tables/${tableId}/columns/${type}`, { - key, - required: false, - array: false, - ...extra, - }, ctx.apiHeaders, [202], `tablesdb.columns.${type}.create`); - waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/columns/${key}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS, `tablesdb.columns.${type}.wait`); - recordTablesWorkerDuration(Date.now() - started, { job: `column_${type}` }); - } - - const indexStarted = Date.now(); - api('POST', `/tablesdb/${databaseId}/tables/${tableId}/indexes`, { - key: indexKey, - type: 'key', - columns: ['title'], - orders: ['asc'], - }, ctx.apiHeaders, [202], 'tablesdb.indexes.create'); - waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/indexes/${indexKey}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS, 'tablesdb.indexes.wait'); - recordTablesWorkerDuration(Date.now() - indexStarted, { job: 'index' }); api('POST', `/tablesdb/${databaseId}/tables/${tableId}/rows`, { rowId, @@ -392,7 +297,6 @@ function tablesDbFlow(ctx) { value: 1, }, ctx.sessionHeaders, [200], 'tablesdb.rows.decrement'); api('DELETE', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, null, ctx.sessionHeaders, [204], 'tablesdb.rows.delete'); - api('DELETE', `/tablesdb/${databaseId}`, null, ctx.apiHeaders, [204], 'tablesdb.delete'); } function storageFlow(ctx) { @@ -426,9 +330,9 @@ function storageFlow(ctx) { tags: { name: 'storage.files.create' }, }); - httpDuration.add(upload.timings.duration, { name: 'storage.files.create' }); httpWaiting.add(upload.timings.waiting, { name: 'storage.files.create' }); apiDuration.add(upload.timings.duration, { name: 'storage.files.create' }); + apiWaiting.add(upload.timings.waiting, { name: 'storage.files.create' }); assertStatus(upload, [201], 'storage file created'); api('GET', `/storage/buckets/${bucketId}/files`, null, ctx.sessionHeaders, [200], 'storage.files.list'); @@ -456,8 +360,6 @@ function computeFlow(ctx) { const functionId = unique('fn'); let functionVariableId; - const siteId = unique('site'); - let siteVariableId; api('POST', '/functions', { functionId, @@ -490,66 +392,12 @@ function computeFlow(ctx) { api('GET', `/functions/${functionId}/variables/${functionVariableId}`, null, ctx.apiHeaders, [200], 'functions.variables.get'); api('DELETE', `/functions/${functionId}/variables/${functionVariableId}`, null, ctx.apiHeaders, [204], 'functions.variables.delete'); api('DELETE', `/functions/${functionId}`, null, ctx.apiHeaders, [204], 'functions.delete'); - - api('POST', '/sites', { - siteId, - name: 'Benchmark Site', - framework: 'other', - adapter: 'static', - buildRuntime: __ENV.APPWRITE_BENCHMARK_RUNTIME || 'node-22', - buildCommand: '', - outputDirectory: '.', - installCommand: '', - fallbackFile: 'index.html', - providerRootDirectory: '.', - specification: '', - }, ctx.apiHeaders, [201], 'sites.create'); - api('GET', '/sites/frameworks', null, ctx.sessionHeaders, [200], 'sites.frameworks.list'); - api('GET', '/sites/specifications', null, ctx.apiHeaders, [200], 'sites.specifications.list'); - const siteVariable = api('POST', `/sites/${siteId}/variables`, { - key: 'BENCHMARK', - value: 'true', - secret: false, - }, ctx.apiHeaders, [201], 'sites.variables.create'); - siteVariableId = siteVariable.json('$id'); - - api('PUT', `/sites/${siteId}/variables/${siteVariableId}`, { - key: 'BENCHMARK', - value: 'updated', - secret: false, - }, ctx.apiHeaders, [200], 'sites.variables.update'); - api('GET', `/sites/${siteId}/variables/${siteVariableId}`, null, ctx.apiHeaders, [200], 'sites.variables.get'); - api('DELETE', `/sites/${siteId}/variables/${siteVariableId}`, null, ctx.apiHeaders, [204], 'sites.variables.delete'); - api('DELETE', `/sites/${siteId}`, null, ctx.apiHeaders, [204], 'sites.delete'); -} - -function healthFlow(ctx) { - const probes = [ - '/health', - '/health/db', - '/health/cache', - '/health/pubsub', - '/health/storage', - '/health/storage/local', - '/health/time', - '/health/queue/mails', - '/health/queue/functions', - '/health/queue/builds', - '/health/queue/deletes', - '/health/queue/webhooks', - '/health/queue/stats-resources', - '/health/queue/stats-usage', - '/health/queue/failed/v1-mails', - ]; - - for (const path of probes) { - api('GET', path, null, ctx.apiHeaders, [200], `health${path.replace(/\//g, '.')}`); - } } function api(method, path, body, headers, expected, name) { const response = rawRequest(method, path, body, headers, name); apiDuration.add(response.timings.duration, { name }); + apiWaiting.add(response.timings.waiting, { name }); assertStatus(response, expected, name); return response; } @@ -567,7 +415,6 @@ function rawRequest(method, path, body, headers, name) { }; const payload = body === null || body === undefined ? null : JSON.stringify(body); const response = http.request(method, `${ENDPOINT}${path}`, payload, params); - httpDuration.add(response.timings.duration, { name }); httpWaiting.add(response.timings.waiting, { name }); return response; @@ -593,73 +440,6 @@ function waitForStatus(path, headers, wantedStatus, timeoutMs, name) { throw new Error(`Timed out waiting for ${path} to become ${wantedStatus}`); } -function waitForEmail(address, predicate, timeoutMs, allowMissingRecipient = false) { - const started = Date.now(); - - while (Date.now() - started < timeoutMs) { - const response = http.get(MAILDEV_ENDPOINT, { tags: { name: 'maildev.email.list' } }); - if (response.status === 200) { - const emails = response.json(); - for (let i = emails.length - 1; i >= 0; i--) { - const message = emails[i]; - if ((emailMatches(message, address) || (allowMissingRecipient && emailRecipientMissing(message))) && predicate(message)) { - return message; - } - } - } - sleep(0.5); - } - - throw new Error(`Timed out waiting for email to ${address}`); -} - -function emailMatches(message, address) { - const recipients = message.to || []; - return recipients.some((recipient) => recipient.address === address); -} - -function emailRecipientMissing(message) { - const recipients = message.to || []; - return recipients.length === 0 || recipients.every((recipient) => !recipient.address); -} - -function extractQueryParams(message) { - const content = `${message.html || ''}\n${message.text || ''}`; - const links = []; - const hrefPattern = /href="([^"]+)"/g; - let hrefMatch = hrefPattern.exec(content); - - while (hrefMatch !== null) { - links.push(hrefMatch[1]); - hrefMatch = hrefPattern.exec(content); - } - - if (links.length === 0) { - links.push(content); - } - - for (const link of links) { - const queryStart = link.indexOf('?'); - if (queryStart === -1) { - continue; - } - - const query = link.slice(queryStart + 1).split('#')[0].replace(/&/g, '&'); - const params = {}; - - for (const pair of query.split('&')) { - const [key, value] = pair.split('='); - params[decodeURIComponent(key)] = decodeURIComponent(value || ''); - } - - if (params.userId && params.secret) { - return params; - } - } - - return {}; -} - function assertStatus(response, expected, name) { const ok = check(response, { [`${name} status ${expected.join('|')}`]: (r) => expected.includes(r.status), @@ -719,10 +499,6 @@ function unique(prefix) { .slice(0, 36); } -function includes(value, needle) { - return String(value || '').toLowerCase().includes(String(needle).toLowerCase()); -} - function hostnameFromUrl(value) { return value.replace(/^https?:\/\//, '').split('/')[0].split(':')[0]; } @@ -731,13 +507,17 @@ export function handleSummary(data) { const lines = [ 'Appwrite curated benchmark review', '', - 'Before/after comparison', + 'Before', '', - comparisonTable(PREVIOUS_SUMMARY, data), + summaryTable(PREVIOUS_SUMMARY), '', - 'Current run details', + 'After', '', - detailsTable(data), + summaryTable(data), + '', + 'Delta', + '', + deltaTable(PREVIOUS_SUMMARY, data), '', ]; @@ -747,19 +527,16 @@ export function handleSummary(data) { }; } -function detailsTable(data) { +function summaryTable(data) { return [ '| Scenario | P50 (ms) | P95 (ms) | Iterations | RPS |', '| --- | ---: | ---: | ---: | ---: |', - detailRow(data, 'Load test', 'appwrite_http_duration', 'iterations', 'http_reqs'), - detailRow(data, 'API total', 'appwrite_api_duration'), - detailRow(data, 'TablesDB schema', 'appwrite_worker_tables_duration', 'appwrite_worker_tables_samples', 'appwrite_worker_tables_samples'), - detailRow(data, 'Mail delivery', 'appwrite_worker_mails_duration', 'appwrite_worker_mails_samples', 'appwrite_worker_mails_samples'), + summaryRow(data, 'API total', 'appwrite_api_duration'), ].join('\n'); } -function detailRow(data, label, metric, iterationsMetric = null, rpsMetric = null) { - const values = data.metrics[metric] && data.metrics[metric].values; +function summaryRow(data, label, metric, iterationsMetric = null, rpsMetric = null) { + const values = data && data.metrics[metric] && data.metrics[metric].values; if (!values || values.count === 0) { return `| ${label} | n/a | n/a | n/a | n/a |`; } @@ -798,23 +575,16 @@ function loadPreviousSummary() { return null; } -function comparisonTable(before, after) { - const rows = [ - ['Load test', 'appwrite_http_duration'], - ['API total', 'appwrite_api_duration'], - ['TablesDB schema', 'appwrite_worker_tables_duration'], - ['Mail delivery', 'appwrite_worker_mails_duration'], - ]; - +function deltaTable(before, after) { return [ - '| Scenario | Before P50 (ms) | Before P95 (ms) | After P50 (ms) | After P95 (ms) | Delta P95 (ms) |', - '| --- | ---: | ---: | ---: | ---: | ---: |', - ...rows.map(([label, metric]) => { - const beforeP50 = trendMetric(before, metric, 'med'); + '| Scenario | P95 delta (ms) |', + '| --- | ---: |', + ...[ + ['API total', 'appwrite_api_duration'], + ].map(([label, metric]) => { const beforeP95 = trendMetric(before, metric, 'p(95)'); - const afterP50 = trendMetric(after, metric, 'med'); const afterP95 = trendMetric(after, metric, 'p(95)'); - return `| ${label} | ${formatValue(beforeP50)} | ${formatValue(beforeP95)} | ${formatValue(afterP50)} | ${formatValue(afterP95)} | ${formatDelta(beforeP95, afterP95)} |`; + return `| ${label} | ${formatDelta(beforeP95, afterP95)} |`; }), ].join('\n'); } @@ -825,14 +595,6 @@ function trendMetric(data, metric, stat) { : null; } -function formatValue(value) { - if (value === null || value === undefined || Number.isNaN(value)) { - return 'n/a'; - } - - return `${round(value)}`; -} - function formatDetailValue(value) { if (value === null || value === undefined || Number.isNaN(value)) { return 'n/a';