From 774a0d7022d1e758de3537dd19530766ddeb4972 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 14:48:12 +0530 Subject: [PATCH 01/39] Improve HTTP benchmark coverage --- tests/benchmarks/http.js | 1028 +++++++++++++++++++++++++++++++++++++- 1 file changed, 1006 insertions(+), 22 deletions(-) diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 799c8fb23c..85f3daee95 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -1,34 +1,1018 @@ import http from 'k6/http'; -import { check } from 'k6'; -import { Counter } from 'k6/metrics'; +import { check, group, sleep } from 'k6'; +import { Counter, Trend } from 'k6/metrics'; -// A simple counter for http requests -export const requests = new Counter('http_reqs'); +const ENDPOINT = (__ENV.APPWRITE_ENDPOINT || 'http://localhost/v1').replace(/\/+$/, ''); +const MAILDEV_ENDPOINT = __ENV.APPWRITE_MAILDEV_ENDPOINT || 'http://localhost:9503/email'; +const CONSOLE_PROJECT = __ENV.APPWRITE_CONSOLE_PROJECT || 'console'; +const REGION = __ENV.APPWRITE_REGION || 'default'; +const REDIRECT_URL = __ENV.APPWRITE_BENCHMARK_REDIRECT_URL || 'http://localhost'; +const PASSWORD = __ENV.APPWRITE_BENCHMARK_PASSWORD || 'Password123!'; +const MAIL_TIMEOUT_MS = Number(__ENV.APPWRITE_MAIL_TIMEOUT_MS || 20000); +const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 60000); +const ITERATIONS = Number(__ENV.APPWRITE_BENCHMARK_ITERATIONS || 1); +const VUS = Number(__ENV.APPWRITE_BENCHMARK_VUS || 1); +const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || 'tests/benchmarks/http-summary.json'; +const PREVIOUS_SUMMARY = loadPreviousSummary(); -// you can specify stages of your test (ramp up/down patterns) through the options object -// target is the number of VUs you are aiming for +export const apiDuration = new Trend('appwrite_api_duration', true); +export const databaseWorkerDuration = new Trend('appwrite_worker_database_duration', true); +export const tablesWorkerDuration = new Trend('appwrite_worker_tables_duration', true); +export const mailsWorkerDuration = new Trend('appwrite_worker_mails_duration', true); +export const messagingWorkerDuration = new Trend('appwrite_worker_messaging_duration', true); +export const flowFailures = new Counter('appwrite_benchmark_flow_failures'); export const options = { - stages: [ - { target: 50, duration: '1m' }, - // { target: 15, duration: '1m' }, - // { target: 0, duration: '1m' }, - ], + scenarios: { + curated_flows: { + executor: 'shared-iterations', + exec: 'curatedFlows', + vus: VUS, + iterations: ITERATIONS, + maxDuration: __ENV.APPWRITE_BENCHMARK_MAX_DURATION || '30m', + }, + }, thresholds: { - requests: ['count < 100'], + http_req_failed: ['rate<0.05'], + appwrite_api_duration: ['p(95)<2000'], + appwrite_benchmark_flow_failures: ['count<1'], }, }; -export default function () { - const config = { - headers: { - 'X-Appwrite-Key': '24356eb021863f81eb7dd77c7750304d0464e141cad6e9a8befa1f7d2b066fde190df3dab1e8d2639dbb82ee848da30501424923f4cd80d887ee40ad77ded62763ee489448523f6e39667f290f9a54b2ab8fad131a0bc985e6c0f760015f7f3411e40626c75646bb19d2bb2f7bf2f63130918220a206758cbc48845fd725a695', - 'X-Appwrite-Project': '60479fe35d95d' - }} +const API_SCOPES = [ + 'sessions.write', + 'users.read', + 'users.write', + 'teams.read', + 'teams.write', + 'databases.read', + 'databases.write', + 'collections.read', + 'collections.write', + 'tables.read', + 'tables.write', + 'attributes.read', + 'attributes.write', + 'columns.read', + 'columns.write', + 'indexes.read', + 'indexes.write', + 'documents.read', + 'documents.write', + 'rows.read', + 'rows.write', + 'files.read', + 'files.write', + 'buckets.read', + 'buckets.write', + 'functions.read', + 'functions.write', + 'sites.read', + 'sites.write', + 'log.read', + 'log.write', + 'execution.read', + 'execution.write', + 'locale.read', + 'avatars.read', + 'health.read', + 'providers.read', + 'providers.write', + 'messages.read', + 'messages.write', + 'topics.read', + 'topics.write', + 'subscribers.read', + 'subscribers.write', + 'targets.read', + 'targets.write', + 'rules.read', + 'rules.write', + 'migrations.read', + 'migrations.write', + 'vcs.read', + 'vcs.write', + 'assistant.read', + 'tokens.read', + 'tokens.write', + 'platforms.read', + 'platforms.write', +]; - const resDb = http.get('http://localhost:9501/', config); +const BASE_PERMISSIONS = [ + 'read("any")', + 'create("any")', + 'update("any")', + 'delete("any")', +]; - check(resDb, { - 'status is 200': (r) => r.status === 200, +const ITEM_PERMISSIONS = [ + 'read("any")', + 'update("any")', + 'delete("any")', +]; + +export function setup() { + const runId = unique('run'); + const consoleEmail = __ENV.APPWRITE_ADMIN_EMAIL || `bench-admin-${runId}@example.com`; + const consolePassword = __ENV.APPWRITE_ADMIN_PASSWORD || PASSWORD; + + const consoleHeaders = { + 'Content-Type': 'application/json', + 'X-Appwrite-Project': CONSOLE_PROJECT, + }; + + const account = rawRequest('POST', '/account', { + userId: unique('admin'), + email: consoleEmail, + password: consolePassword, + name: 'Benchmark Admin', + }, consoleHeaders, 'setup.account.create'); + + if (![201, 409].includes(account.status)) { + failResponse(account, 'Unable to create or reuse the benchmark console account'); + } + + const session = rawRequest('POST', '/account/sessions/email', { + email: consoleEmail, + password: consolePassword, + }, consoleHeaders, 'setup.account.session'); + + assertStatus(session, [201], 'console session created'); + + const consoleSessionHeaders = { + ...consoleHeaders, + Cookie: cookieHeader(session), + }; + + const team = api('POST', '/teams', { + teamId: unique('team'), + name: `Benchmark Team ${runId}`, + }, consoleSessionHeaders, [201], 'setup.teams.create'); + + const teamId = team.json('$id'); + const project = api('POST', '/projects', { + projectId: unique('project'), + name: `Benchmark Project ${runId}`, + teamId, + region: REGION, + }, consoleSessionHeaders, [201], 'setup.projects.create'); + + const projectId = project.json('$id'); + const key = api('POST', `/projects/${projectId}/keys`, { + keyId: unique('key'), + name: 'Benchmark API key', + scopes: API_SCOPES, + }, consoleSessionHeaders, [201], 'setup.projects.keys.create'); + + const apiHeaders = { + 'Content-Type': 'application/json', + 'X-Appwrite-Project': projectId, + 'X-Appwrite-Key': key.json('secret'), + }; + + const platform = api('POST', '/project/platforms/web', { + platformId: unique('web'), + name: 'Benchmark web', + hostname: hostnameFromUrl(REDIRECT_URL), + }, apiHeaders, [201, 409], 'setup.project.platforms.web.create'); + + const smtp = rawRequest('PATCH', `/projects/${projectId}/smtp`, { + enabled: true, + senderName: 'Benchmark', + senderEmail: 'benchmark@appwrite.io', + replyTo: 'benchmark@appwrite.io', + host: __ENV.APPWRITE_SMTP_HOST || 'maildev', + port: Number(__ENV.APPWRITE_SMTP_PORT || 1025), + username: __ENV.APPWRITE_SMTP_USERNAME || 'user', + password: __ENV.APPWRITE_SMTP_PASSWORD || 'password', + ...(String(__ENV.APPWRITE_SMTP_SECURE || '') !== '' ? { secure: __ENV.APPWRITE_SMTP_SECURE } : {}), + }, consoleSessionHeaders, 'setup.projects.smtp.update'); + + if (smtp.status !== 200) { + console.warn(`Custom SMTP was not enabled (${smtp.status}). Mail worker timings may be unavailable.`); + } + + return { + runId, + teamId, + projectId, + consoleSessionHeaders, + apiHeaders, + platformStatus: platform.status, + }; +} + +export function curatedFlows(data) { + const ctx = { ...data }; + + try { + group('account and mail worker', () => accountFlow(ctx)); + group('databases documents flow', () => databasesFlow(ctx)); + group('tablesdb rows flow', () => tablesDbFlow(ctx)); + group('storage files and tokens flow', () => storageFlow(ctx)); + group('messaging worker flow', () => messagingFlow(ctx)); + group('functions and sites control-plane flow', () => computeFlow(ctx)); + group('health and queue probes', () => healthFlow(ctx)); + } catch (error) { + flowFailures.add(1); + throw error; + } +} + +export function teardown(data) { + if (data && data.teamId && data.consoleSessionHeaders) { + rawRequest('DELETE', `/teams/${data.teamId}`, null, data.consoleSessionHeaders, 'teardown.teams.delete'); + } +} + +function accountFlow(ctx) { + const userId = unique('user'); + const email = `bench-user-${unique('mail')}@example.com`; + const headers = projectHeaders(ctx.projectId); + + api('POST', '/account', { + userId, + email, + password: PASSWORD, + name: 'Benchmark User', + }, headers, [201], 'account.create'); + + const session = api('POST', '/account/sessions/email', { + email, + password: PASSWORD, + }, headers, [201], 'account.sessions.email.create'); + + const sessionHeaders = { + ...headers, + Cookie: cookieHeader(session), + }; + + ctx.userId = userId; + ctx.userEmail = email; + ctx.sessionHeaders = sessionHeaders; + + const jwt = api('POST', '/account/jwts', null, sessionHeaders, [201], 'account.jwts.create'); + ctx.jwtHeaders = { + ...headers, + 'X-Appwrite-JWT': jwt.json('jwt'), + }; + + api('GET', '/account', null, sessionHeaders, [200], 'account.get'); + api('GET', '/account/logs', null, sessionHeaders, [200], 'account.logs.list'); + api('PATCH', '/account/prefs', { prefs: { benchmark: true, runId: ctx.runId } }, sessionHeaders, [200], 'account.prefs.update'); + api('PATCH', '/account/name', { name: 'Benchmark User Updated' }, sessionHeaders, [200], 'account.name.update'); + api('PATCH', '/account/password', { password: `${PASSWORD}2`, oldPassword: PASSWORD }, sessionHeaders, [200], 'account.password.update'); + + const verificationStarted = Date.now(); + api('POST', '/account/verifications/email', { url: REDIRECT_URL }, sessionHeaders, [201], 'account.emailVerification.create'); + const verificationEmail = waitForEmail(email, (message) => { + return includes(message.subject, 'verify') + || includes(message.subject, 'verification') + || includes(message.html, 'verify') + || includes(message.html, 'verification') + || includes(message.text, 'verify') + || includes(message.text, 'verification'); + }, MAIL_TIMEOUT_MS); + mailsWorkerDuration.add(Date.now() - verificationStarted, { job: 'email_verification' }); + + const verification = extractQueryParams(verificationEmail); + if (verification.userId && verification.secret) { + api('PUT', '/account/verifications/email', { + userId: verification.userId, + secret: verification.secret, + }, sessionHeaders, [200], 'account.emailVerification.update'); + } + + const recoveryStarted = Date.now(); + api('POST', '/account/recovery', { email, url: REDIRECT_URL }, headers, [201], 'account.recovery.create'); + const recoveryEmail = waitForEmail(email, (message) => { + return includes(message.subject, 'recovery') + || includes(message.subject, 'recover') + || includes(message.subject, 'reset') + || includes(message.html, 'recovery') + || includes(message.html, 'recover') + || includes(message.html, 'reset') + || includes(message.text, 'recovery') + || includes(message.text, 'recover') + || includes(message.text, 'reset'); + }, MAIL_TIMEOUT_MS); + mailsWorkerDuration.add(Date.now() - recoveryStarted, { job: 'password_recovery' }); + + const recovery = extractQueryParams(recoveryEmail); + if (recovery.userId && recovery.secret) { + api('DELETE', '/account/sessions/current', null, sessionHeaders, [204], 'account.sessions.current.delete'); + + api('PUT', '/account/recovery', { + userId: recovery.userId, + secret: recovery.secret, + password: `${PASSWORD}3`, + }, headers, [200], 'account.recovery.update'); + + const recoveredSession = api('POST', '/account/sessions/email', { + email, + password: `${PASSWORD}3`, + }, headers, [201], 'account.sessions.email.recovered'); + + ctx.sessionHeaders = { + ...headers, + Cookie: cookieHeader(recoveredSession), + }; + + const recoveredJwt = api('POST', '/account/jwts', null, ctx.sessionHeaders, [201], 'account.jwts.recovered'); + ctx.jwtHeaders = { + ...headers, + 'X-Appwrite-JWT': recoveredJwt.json('jwt'), + }; + } +} + +function databasesFlow(ctx) { + const databaseId = unique('db'); + const collectionId = unique('col'); + const documentId = unique('doc'); + const indexKey = unique('idx'); + + api('POST', '/databases', { databaseId, name: 'Benchmark DB' }, ctx.apiHeaders, [201], 'databases.create'); + api('POST', `/databases/${databaseId}/collections`, { + collectionId, + name: 'Benchmark Collection', + permissions: BASE_PERMISSIONS, + documentSecurity: false, + }, ctx.apiHeaders, [201], 'databases.collections.create'); + + const attributes = [ + ['string', 'title', { size: 128 }], + ['integer', 'count', { min: 0, max: 100000 }], + ['email', 'email', {}], + ['boolean', 'active', {}], + ['datetime', 'publishedAt', {}], + ['float', 'score', { min: 0, max: 1000 }], + ['url', 'url', {}], + ['ip', 'ip', {}], + ]; + + for (const [type, key, extra] of attributes) { + const started = Date.now(); + api('POST', `/databases/${databaseId}/collections/${collectionId}/attributes/${type}`, { + key, + required: false, + array: false, + ...extra, + }, ctx.apiHeaders, [202], `databases.attributes.${type}.create`); + waitForStatus(`/databases/${databaseId}/collections/${collectionId}/attributes/${key}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); + databaseWorkerDuration.add(Date.now() - started, { job: `attribute_${type}` }); + } + + const indexStarted = Date.now(); + api('POST', `/databases/${databaseId}/collections/${collectionId}/indexes`, { + key: indexKey, + type: 'key', + attributes: ['title'], + orders: ['asc'], + }, ctx.apiHeaders, [202], 'databases.indexes.create'); + waitForStatus(`/databases/${databaseId}/collections/${collectionId}/indexes/${indexKey}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); + databaseWorkerDuration.add(Date.now() - indexStarted, { job: 'index' }); + + api('POST', `/databases/${databaseId}/collections/${collectionId}/documents`, { + documentId, + data: documentPayload(), + permissions: ITEM_PERMISSIONS, + }, ctx.apiHeaders, [201], 'databases.documents.create'); + api('GET', `/databases/${databaseId}/collections/${collectionId}/documents`, null, ctx.apiHeaders, [200], 'databases.documents.list'); + api('GET', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}`, null, ctx.apiHeaders, [200], 'databases.documents.get'); + api('PATCH', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}`, { + data: { title: 'Benchmark Document Updated' }, + }, ctx.apiHeaders, [200], 'databases.documents.update'); + api('PATCH', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}/count/increment`, { + value: 1, + }, ctx.apiHeaders, [200], 'databases.documents.increment'); + api('PATCH', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}/count/decrement`, { + value: 1, + }, ctx.apiHeaders, [200], 'databases.documents.decrement'); + api('DELETE', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}`, null, ctx.apiHeaders, [204], 'databases.documents.delete'); + api('DELETE', `/databases/${databaseId}`, null, ctx.apiHeaders, [204], 'databases.delete'); +} + +function tablesDbFlow(ctx) { + const databaseId = unique('tdb'); + const tableId = unique('tbl'); + const rowId = unique('row'); + const indexKey = unique('tidx'); + + api('POST', '/tablesdb', { databaseId, name: 'Benchmark TablesDB' }, ctx.apiHeaders, [201], 'tablesdb.create'); + api('POST', `/tablesdb/${databaseId}/tables`, { + tableId, + name: 'Benchmark Table', + permissions: BASE_PERMISSIONS, + rowSecurity: false, + }, ctx.apiHeaders, [201], 'tablesdb.tables.create'); + + const columns = [ + ['string', 'title', { size: 128 }], + ['integer', 'count', { min: 0, max: 100000 }], + ['email', 'email', {}], + ['boolean', 'active', {}], + ]; + + for (const [type, key, extra] of columns) { + const started = Date.now(); + api('POST', `/tablesdb/${databaseId}/tables/${tableId}/columns/${type}`, { + key, + required: false, + array: false, + ...extra, + }, ctx.apiHeaders, [202], `tablesdb.columns.${type}.create`); + waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/columns/${key}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); + tablesWorkerDuration.add(Date.now() - started, { job: `column_${type}` }); + } + + const indexStarted = Date.now(); + api('POST', `/tablesdb/${databaseId}/tables/${tableId}/indexes`, { + key: indexKey, + type: 'key', + columns: ['title'], + orders: ['asc'], + }, ctx.apiHeaders, [202], 'tablesdb.indexes.create'); + waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/indexes/${indexKey}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); + tablesWorkerDuration.add(Date.now() - indexStarted, { job: 'index' }); + + api('POST', `/tablesdb/${databaseId}/tables/${tableId}/rows`, { + rowId, + data: tablePayload(), + permissions: ITEM_PERMISSIONS, + }, ctx.sessionHeaders, [201], 'tablesdb.rows.create'); + api('GET', `/tablesdb/${databaseId}/tables/${tableId}/rows`, null, ctx.sessionHeaders, [200], 'tablesdb.rows.list'); + api('GET', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, null, ctx.sessionHeaders, [200], 'tablesdb.rows.get'); + api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, { + data: { title: 'Benchmark Row Updated' }, + }, ctx.sessionHeaders, [200], 'tablesdb.rows.update'); + api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}/count/increment`, { + value: 1, + }, ctx.sessionHeaders, [200], 'tablesdb.rows.increment'); + api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}/count/decrement`, { + value: 1, + }, ctx.sessionHeaders, [200], 'tablesdb.rows.decrement'); + api('DELETE', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, null, ctx.sessionHeaders, [204], 'tablesdb.rows.delete'); + api('DELETE', `/tablesdb/${databaseId}`, null, ctx.apiHeaders, [204], 'tablesdb.delete'); +} + +function storageFlow(ctx) { + const bucketId = unique('bucket'); + const fileId = unique('file'); + + api('POST', '/storage/buckets', { + bucketId, + name: 'Benchmark Bucket', + permissions: BASE_PERMISSIONS, + fileSecurity: false, + enabled: true, + maximumFileSize: 30000000, + allowedFileExtensions: [], + compression: 'none', + encryption: false, + antivirus: false, + }, ctx.apiHeaders, [201], 'storage.buckets.create'); + + const multipartHeaders = { ...ctx.sessionHeaders }; + delete multipartHeaders['Content-Type']; + + const upload = http.post(`${ENDPOINT}/storage/buckets/${bucketId}/files`, { + fileId, + file: http.file(onePixelPng(), 'benchmark.png', 'image/png'), + ...flattenMultipartArray('permissions', ITEM_PERMISSIONS), + }, { + headers: multipartHeaders, + tags: { name: 'storage.files.create' }, }); -} \ No newline at end of file + + apiDuration.add(upload.timings.duration, { name: 'storage.files.create' }); + assertStatus(upload, [201], 'storage file created'); + + api('GET', `/storage/buckets/${bucketId}/files`, null, ctx.sessionHeaders, [200], 'storage.files.list'); + api('GET', `/storage/buckets/${bucketId}/files/${fileId}`, null, ctx.sessionHeaders, [200], 'storage.files.get'); + api('GET', `/storage/buckets/${bucketId}/files/${fileId}/view`, null, ctx.sessionHeaders, [200], 'storage.files.view'); + api('GET', `/storage/buckets/${bucketId}/files/${fileId}/download`, null, ctx.sessionHeaders, [200], 'storage.files.download'); + api('GET', `/storage/buckets/${bucketId}/files/${fileId}/preview`, null, ctx.sessionHeaders, [200], 'storage.files.preview'); + api('PUT', `/storage/buckets/${bucketId}/files/${fileId}`, { + name: 'benchmark-renamed.png', + permissions: ITEM_PERMISSIONS, + }, ctx.sessionHeaders, [200], 'storage.files.update'); + + const token = api('POST', `/tokens/buckets/${bucketId}/files/${fileId}`, {}, ctx.apiHeaders, [201], 'tokens.files.create'); + api('GET', `/tokens/buckets/${bucketId}/files/${fileId}`, null, ctx.apiHeaders, [200], 'tokens.files.list'); + api('GET', `/tokens/${token.json('$id')}`, null, ctx.apiHeaders, [200], 'tokens.get'); + api('PATCH', `/tokens/${token.json('$id')}`, { expire: null }, ctx.apiHeaders, [200], 'tokens.update'); + api('DELETE', `/tokens/${token.json('$id')}`, null, ctx.apiHeaders, [204], 'tokens.delete'); + + api('DELETE', `/storage/buckets/${bucketId}/files/${fileId}`, null, ctx.sessionHeaders, [204], 'storage.files.delete'); + api('DELETE', `/storage/buckets/${bucketId}`, null, ctx.apiHeaders, [204], 'storage.buckets.delete'); +} + +function messagingFlow(ctx) { + const providerId = unique('smtp'); + let targetId = unique('target'); + const topicId = unique('topic'); + const subscriberId = unique('sub'); + const messageId = unique('msg'); + + api('POST', '/messaging/providers/smtp', { + providerId, + name: 'Benchmark SMTP', + host: __ENV.APPWRITE_SMTP_HOST || 'maildev', + port: Number(__ENV.APPWRITE_SMTP_PORT || 1025), + username: __ENV.APPWRITE_SMTP_USERNAME || 'user', + password: __ENV.APPWRITE_SMTP_PASSWORD || 'password', + encryption: __ENV.APPWRITE_SMTP_ENCRYPTION || 'none', + autoTLS: false, + fromName: 'Benchmark', + fromEmail: 'benchmark@appwrite.io', + replyToName: 'Benchmark', + replyToEmail: 'benchmark@appwrite.io', + enabled: true, + }, ctx.apiHeaders, [201], 'messaging.providers.smtp.create'); + + const targets = api('GET', `/users/${ctx.userId}/targets`, null, ctx.apiHeaders, [200], 'users.targets.list'); + const existingTarget = (targets.json('targets') || []).find((target) => { + return target.providerType === 'email' && target.identifier === ctx.userEmail; + }); + + if (existingTarget) { + targetId = existingTarget.$id; + api('PATCH', `/users/${ctx.userId}/targets/${targetId}`, { + providerId, + name: 'Benchmark email target', + }, ctx.apiHeaders, [200], 'users.targets.update'); + } else { + api('POST', `/users/${ctx.userId}/targets`, { + targetId, + providerType: 'email', + identifier: ctx.userEmail, + providerId, + name: 'Benchmark email target', + }, ctx.apiHeaders, [201], 'users.targets.create'); + } + + api('POST', '/messaging/topics', { + topicId, + name: 'Benchmark Topic', + subscribe: ['users'], + }, ctx.apiHeaders, [201], 'messaging.topics.create'); + + api('POST', `/messaging/topics/${topicId}/subscribers`, { + subscriberId, + targetId, + }, ctx.sessionHeaders, [201], 'messaging.subscribers.create'); + + const started = Date.now(); + api('POST', '/messaging/messages/email', { + messageId, + subject: `Benchmark message ${ctx.runId}`, + content: `Benchmark messaging worker probe ${ctx.runId}`, + targets: [targetId], + draft: false, + html: false, + }, ctx.apiHeaders, [201], 'messaging.messages.email.create'); + + waitForMessage(messageId, ctx.apiHeaders, WORKER_TIMEOUT_MS); + waitForEmail(ctx.userEmail, (message) => includes(message.subject, `Benchmark message ${ctx.runId}`), MAIL_TIMEOUT_MS, true); + messagingWorkerDuration.add(Date.now() - started, { job: 'email_message' }); + + api('GET', '/messaging/messages', null, ctx.apiHeaders, [200], 'messaging.messages.list'); + api('GET', `/messaging/messages/${messageId}/logs`, null, ctx.apiHeaders, [200], 'messaging.messages.logs.list'); + api('GET', `/messaging/messages/${messageId}/targets`, null, ctx.apiHeaders, [200], 'messaging.messages.targets.list'); + api('GET', `/messaging/providers/${providerId}/logs`, null, ctx.apiHeaders, [200], 'messaging.providers.logs.list'); + api('GET', `/messaging/topics/${topicId}/logs`, null, ctx.apiHeaders, [200], 'messaging.topics.logs.list'); + api('GET', `/messaging/subscribers/${subscriberId}/logs`, null, ctx.apiHeaders, [200], 'messaging.subscribers.logs.list'); + api('DELETE', `/messaging/topics/${topicId}/subscribers/${subscriberId}`, null, ctx.sessionHeaders, [204], 'messaging.subscribers.delete'); + api('DELETE', `/messaging/topics/${topicId}`, null, ctx.apiHeaders, [204], 'messaging.topics.delete'); + api('DELETE', `/messaging/messages/${messageId}`, null, ctx.apiHeaders, [204], 'messaging.messages.delete'); + api('DELETE', `/messaging/providers/${providerId}`, null, ctx.apiHeaders, [204], 'messaging.providers.delete'); +} + +function computeFlow(ctx) { + const functionId = unique('fn'); + let functionVariableId; + const siteId = unique('site'); + let siteVariableId; + + api('POST', '/functions', { + functionId, + name: 'Benchmark Function', + runtime: __ENV.APPWRITE_BENCHMARK_RUNTIME || 'node-22', + execute: ['any'], + events: [], + schedule: '', + timeout: 15, + enabled: true, + logging: true, + entrypoint: 'index.js', + commands: 'npm install', + scopes: ['users.read'], + }, ctx.apiHeaders, [201], 'functions.create'); + api('GET', '/functions/runtimes', null, ctx.sessionHeaders, [200], 'functions.runtimes.list'); + api('GET', '/functions/specifications', null, ctx.apiHeaders, [200], 'functions.specifications.list'); + const functionVariable = api('POST', `/functions/${functionId}/variables`, { + key: 'BENCHMARK', + value: 'true', + secret: false, + }, ctx.apiHeaders, [201], 'functions.variables.create'); + functionVariableId = functionVariable.json('$id'); + + api('PUT', `/functions/${functionId}/variables/${functionVariableId}`, { + key: 'BENCHMARK', + value: 'updated', + secret: false, + }, ctx.apiHeaders, [200], 'functions.variables.update'); + api('GET', `/functions/${functionId}/variables/${functionVariableId}`, null, ctx.apiHeaders, [200], 'functions.variables.get'); + api('DELETE', `/functions/${functionId}/variables/${functionVariableId}`, null, ctx.apiHeaders, [204], 'functions.variables.delete'); + api('DELETE', `/functions/${functionId}`, null, ctx.apiHeaders, [204], 'functions.delete'); + + api('POST', '/sites', { + siteId, + name: 'Benchmark Site', + framework: 'other', + adapter: 'static', + buildRuntime: __ENV.APPWRITE_BENCHMARK_RUNTIME || 'node-22', + buildCommand: '', + outputDirectory: '.', + installCommand: '', + fallbackFile: 'index.html', + providerRootDirectory: '.', + specification: '', + }, ctx.apiHeaders, [201], 'sites.create'); + api('GET', '/sites/frameworks', null, ctx.sessionHeaders, [200], 'sites.frameworks.list'); + api('GET', '/sites/specifications', null, ctx.apiHeaders, [200], 'sites.specifications.list'); + const siteVariable = api('POST', `/sites/${siteId}/variables`, { + key: 'BENCHMARK', + value: 'true', + secret: false, + }, ctx.apiHeaders, [201], 'sites.variables.create'); + siteVariableId = siteVariable.json('$id'); + + api('PUT', `/sites/${siteId}/variables/${siteVariableId}`, { + key: 'BENCHMARK', + value: 'updated', + secret: false, + }, ctx.apiHeaders, [200], 'sites.variables.update'); + api('GET', `/sites/${siteId}/variables/${siteVariableId}`, null, ctx.apiHeaders, [200], 'sites.variables.get'); + api('DELETE', `/sites/${siteId}/variables/${siteVariableId}`, null, ctx.apiHeaders, [204], 'sites.variables.delete'); + api('DELETE', `/sites/${siteId}`, null, ctx.apiHeaders, [204], 'sites.delete'); +} + +function healthFlow(ctx) { + const probes = [ + '/health', + '/health/db', + '/health/cache', + '/health/pubsub', + '/health/storage', + '/health/storage/local', + '/health/time', + '/health/queue/databases', + '/health/queue/mails', + '/health/queue/messaging', + '/health/queue/functions', + '/health/queue/builds', + '/health/queue/deletes', + '/health/queue/webhooks', + '/health/queue/stats-resources', + '/health/queue/stats-usage', + '/health/queue/failed/v1-mails', + ]; + + for (const path of probes) { + api('GET', path, null, ctx.apiHeaders, [200], `health${path.replace(/\//g, '.')}`); + } +} + +function api(method, path, body, headers, expected, name) { + const response = rawRequest(method, path, body, headers, name); + apiDuration.add(response.timings.duration, { name }); + assertStatus(response, expected, name); + return response; +} + +function rawRequest(method, path, body, headers, name) { + const params = { + headers, + tags: { name }, + }; + const payload = body === null || body === undefined ? null : JSON.stringify(body); + return http.request(method, `${ENDPOINT}${path}`, payload, params); +} + +function waitForStatus(path, headers, wantedStatus, timeoutMs) { + const started = Date.now(); + + while (Date.now() - started < timeoutMs) { + const response = api('GET', path, null, headers, [200], `wait${path}`); + if (response.json('status') === wantedStatus) { + return response; + } + sleep(0.5); + } + + throw new Error(`Timed out waiting for ${path} to become ${wantedStatus}`); +} + +function waitForMessage(messageId, headers, timeoutMs) { + const started = Date.now(); + + while (Date.now() - started < timeoutMs) { + const response = api('GET', `/messaging/messages/${messageId}`, null, headers, [200], 'messaging.messages.poll'); + const status = response.json('status'); + + if (['sent', 'failed'].includes(status)) { + if (status === 'failed') { + throw new Error(`Messaging worker marked message ${messageId} as failed`); + } + return response; + } + + sleep(0.5); + } + + throw new Error(`Timed out waiting for messaging worker to send message ${messageId}`); +} + +function waitForEmail(address, predicate, timeoutMs, allowMissingRecipient = false) { + const started = Date.now(); + + while (Date.now() - started < timeoutMs) { + const response = http.get(MAILDEV_ENDPOINT, { tags: { name: 'maildev.email.list' } }); + if (response.status === 200) { + const emails = response.json(); + for (let i = emails.length - 1; i >= 0; i--) { + const message = emails[i]; + if ((emailMatches(message, address) || (allowMissingRecipient && emailRecipientMissing(message))) && predicate(message)) { + return message; + } + } + } + sleep(0.5); + } + + throw new Error(`Timed out waiting for email to ${address}`); +} + +function emailMatches(message, address) { + const recipients = message.to || []; + return recipients.some((recipient) => recipient.address === address); +} + +function emailRecipientMissing(message) { + const recipients = message.to || []; + return recipients.length === 0 || recipients.every((recipient) => !recipient.address); +} + +function extractQueryParams(message) { + const content = `${message.html || ''}\n${message.text || ''}`; + const links = []; + const hrefPattern = /href="([^"]+)"/g; + let hrefMatch = hrefPattern.exec(content); + + while (hrefMatch !== null) { + links.push(hrefMatch[1]); + hrefMatch = hrefPattern.exec(content); + } + + if (links.length === 0) { + links.push(content); + } + + for (const link of links) { + const queryStart = link.indexOf('?'); + if (queryStart === -1) { + continue; + } + + const query = link.slice(queryStart + 1).split('#')[0].replace(/&/g, '&'); + const params = {}; + + for (const pair of query.split('&')) { + const [key, value] = pair.split('='); + params[decodeURIComponent(key)] = decodeURIComponent(value || ''); + } + + if (params.userId && params.secret) { + return params; + } + } + + return {}; +} + +function assertStatus(response, expected, name) { + const ok = check(response, { + [`${name} status ${expected.join('|')}`]: (r) => expected.includes(r.status), + }); + + if (!ok) { + failResponse(response, `${name} returned an unexpected status`); + } +} + +function failResponse(response, message) { + throw new Error(`${message}. Status: ${response.status}. Body: ${response.body}`); +} + +function cookieHeader(response) { + return response.headers['Set-Cookie'] || response.headers['set-cookie'] || ''; +} + +function projectHeaders(projectId) { + return { + 'Content-Type': 'application/json', + 'X-Appwrite-Project': projectId, + }; +} + +function documentPayload() { + return { + title: 'Benchmark Document', + count: 1, + email: 'document@example.com', + active: true, + publishedAt: new Date().toISOString(), + score: 10.5, + url: 'https://appwrite.io', + ip: '127.0.0.1', + }; +} + +function tablePayload() { + return { + title: 'Benchmark Row', + count: 1, + email: 'row@example.com', + active: true, + }; +} + +function onePixelPng() { + return base64ToBinary('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII='); +} + +function base64ToBinary(input) { + const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='; + let output = ''; + let buffer = 0; + let bits = 0; + + for (let i = 0; i < input.length; i++) { + const value = chars.indexOf(input.charAt(i)); + if (value < 0 || value === 64) { + continue; + } + + buffer = (buffer << 6) | value; + bits += 6; + + if (bits >= 8) { + bits -= 8; + output += String.fromCharCode((buffer >> bits) & 0xff); + } + } + + return output; +} + +function flattenMultipartArray(key, values) { + const output = {}; + values.forEach((value, index) => { + output[`${key}[${index}]`] = value; + }); + return output; +} + +function unique(prefix) { + return `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}` + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .slice(0, 36); +} + +function includes(value, needle) { + return String(value || '').toLowerCase().includes(String(needle).toLowerCase()); +} + +function hostnameFromUrl(value) { + return value.replace(/^https?:\/\//, '').split('/')[0].split(':')[0]; +} + +export function handleSummary(data) { + const lines = [ + 'Appwrite curated benchmark review', + '', + 'Before/after comparison', + '', + comparisonTable(PREVIOUS_SUMMARY, data), + '', + 'Current run details', + '', + metricLine(data, 'http_req_duration', 'HTTP total'), + metricLine(data, 'appwrite_api_duration', 'API endpoints'), + metricLine(data, 'appwrite_worker_database_duration', 'Database worker schema jobs'), + metricLine(data, 'appwrite_worker_tables_duration', 'TablesDB worker schema jobs'), + metricLine(data, 'appwrite_worker_mails_duration', 'Mail worker delivery'), + metricLine(data, 'appwrite_worker_messaging_duration', 'Messaging worker delivery'), + counterLine(data, 'appwrite_benchmark_flow_failures', 'Flow failures'), + '', + `Endpoint: ${ENDPOINT}`, + `Maildev API: ${MAILDEV_ENDPOINT}`, + '', + ]; + + return { + stdout: `${lines.filter(Boolean).join('\n')}\n`, + [SUMMARY_PATH]: JSON.stringify(data, null, 2), + }; +} + +function loadPreviousSummary() { + try { + if (SUMMARY_PATH === 'tests/benchmarks/http-summary.json') { + return JSON.parse(open('http-summary.json')); + } + + return JSON.parse(open(SUMMARY_PATH)); + } catch (error) { + return null; + } +} + +function comparisonTable(before, after) { + const rows = [ + ['HTTP total p95', trendMetric(before, 'http_req_duration', 'p(95)'), trendMetric(after, 'http_req_duration', 'p(95)'), 'ms'], + ['API endpoints p95', trendMetric(before, 'appwrite_api_duration', 'p(95)'), trendMetric(after, 'appwrite_api_duration', 'p(95)'), 'ms'], + ['Database worker p95', trendMetric(before, 'appwrite_worker_database_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_database_duration', 'p(95)'), 'ms'], + ['TablesDB worker p95', trendMetric(before, 'appwrite_worker_tables_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_tables_duration', 'p(95)'), 'ms'], + ['Mail worker p95', trendMetric(before, 'appwrite_worker_mails_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_mails_duration', 'p(95)'), 'ms'], + ['Messaging worker p95', trendMetric(before, 'appwrite_worker_messaging_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_messaging_duration', 'p(95)'), 'ms'], + ['Flow failures', counterMetric(before, 'appwrite_benchmark_flow_failures'), counterMetric(after, 'appwrite_benchmark_flow_failures'), ''], + ['Check failures', checkFailures(before), checkFailures(after), ''], + ]; + + return [ + '| Metric | Before | After | Delta |', + '| --- | ---: | ---: | ---: |', + ...rows.map(([label, beforeValue, afterValue, unit]) => { + return `| ${label} | ${formatValue(beforeValue, unit)} | ${formatValue(afterValue, unit)} | ${formatDelta(beforeValue, afterValue, unit)} |`; + }), + ].join('\n'); +} + +function trendMetric(data, metric, stat) { + return data && data.metrics[metric] && data.metrics[metric].values + ? data.metrics[metric].values[stat] + : null; +} + +function counterMetric(data, metric) { + return data && data.metrics[metric] && data.metrics[metric].values + ? data.metrics[metric].values.count + : null; +} + +function checkFailures(data) { + return data && data.metrics.checks && data.metrics.checks.values + ? data.metrics.checks.values.fails + : null; +} + +function formatValue(value, unit) { + if (value === null || value === undefined || Number.isNaN(value)) { + return 'n/a'; + } + + return `${round(value)}${unit}`; +} + +function formatDelta(before, after, unit) { + if (before === null || before === undefined || after === null || after === undefined || Number.isNaN(before) || Number.isNaN(after)) { + return 'n/a'; + } + + const delta = round(after - before); + const sign = delta > 0 ? '+' : ''; + return `${sign}${delta}${unit}`; +} + +function metricLine(data, metric, label) { + const values = data.metrics[metric] && data.metrics[metric].values; + if (!values || values.count === 0) { + return `${label}: no samples`; + } + + return `${label}: avg=${round(values.avg)}ms p90=${round(values['p(90)'])}ms p95=${round(values['p(95)'])}ms max=${round(values.max)}ms`; +} + +function counterLine(data, metric, label) { + const values = data.metrics[metric] && data.metrics[metric].values; + return `${label}: ${values ? values.count : 0}`; +} + +function round(value) { + return Math.round((value || 0) * 100) / 100; +} From e4f74a3fb140f2f0d40296dbb4404a9b1717212a Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 14:56:33 +0530 Subject: [PATCH 02/39] Run curated HTTP benchmark in CI --- .github/workflows/ci.yml | 66 ++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b02d021f1a..3f5aa9034f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -676,56 +676,42 @@ jobs: docker compose up -d sleep 10 - - name: Install Oha + - name: Benchmark baseline run: | - echo "deb [signed-by=/usr/share/keyrings/azlux-archive-keyring.gpg] http://packages.azlux.fr/debian/ stable main" | sudo tee /etc/apt/sources.list.d/azlux.list - sudo wget -O /usr/share/keyrings/azlux-archive-keyring.gpg https://azlux.fr/repo.gpg - sudo apt update - sudo apt install oha - oha --version + rm -f tests/benchmarks/http-summary.json benchmark-before.txt benchmark.txt + docker run --rm -i --network host -v "$PWD:/scripts" -w /scripts grafana/k6 run --quiet \ + -e APPWRITE_ENDPOINT=http://localhost/v1 \ + -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ + -e APPWRITE_BENCHMARK_ITERATIONS=1 \ + -e APPWRITE_BENCHMARK_VUS=1 \ + tests/benchmarks/http.js | tee benchmark-before.txt - - name: Benchmark PR - run: 'oha -z 180s http://localhost/v1/health/version --output-format json > benchmark.json' - - - name: Cleaning - run: docker compose down -v - - - name: Installing latest version + - name: Benchmark after run: | - rm docker-compose.yml - rm .env - curl https://appwrite.io/install/compose -o docker-compose.yml - curl https://appwrite.io/install/env -o .env - sed -i 's/_APP_OPTIONS_ABUSE=enabled/_APP_OPTIONS_ABUSE=disabled/g' .env - docker compose up -d - sleep 10 - - - name: Benchmark Latest - run: oha -z 180s http://localhost/v1/health/version --output-format json > benchmark-latest.json + docker run --rm -i --network host -v "$PWD:/scripts" -w /scripts grafana/k6 run --quiet \ + -e APPWRITE_ENDPOINT=http://localhost/v1 \ + -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ + -e APPWRITE_BENCHMARK_ITERATIONS=1 \ + -e APPWRITE_BENCHMARK_VUS=1 \ + tests/benchmarks/http.js | tee benchmark.txt - name: Prepare comment run: | - echo '## :sparkles: Benchmark results' > benchmark.txt - echo ' ' >> benchmark.txt - echo "- Requests per second: $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt - echo "- Requests with 200 status code: $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt - echo "- P99 latency: $(jq -r '.latencyPercentiles.p99' benchmark.json )" >> benchmark.txt - echo " " >> benchmark.txt - echo " " >> benchmark.txt - echo "## :zap: Benchmark Comparison" >> benchmark.txt - echo " " >> benchmark.txt - echo "| Metric | This PR | Latest version | " >> benchmark.txt - echo "| --- | --- | --- | " >> benchmark.txt - echo "| RPS | $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.summary.requestsPerSec|tonumber|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt - echo "| 200 | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt - echo "| P99 | $(jq -r '.latencyPercentiles.p99' benchmark.json ) | $(jq -r '.latencyPercentiles.p99' benchmark-latest.json ) | " >> benchmark.txt + { + echo '## :sparkles: Benchmark results' + echo + cat benchmark.txt + } > benchmark-comment.txt - name: Save results uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: - name: benchmark.json - path: benchmark.json + name: benchmark-results + path: | + benchmark-before.txt + benchmark.txt + tests/benchmarks/http-summary.json retention-days: 7 - name: Find Comment @@ -743,5 +729,5 @@ jobs: with: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} - body-path: benchmark.txt + body-path: benchmark-comment.txt edit-mode: replace From 15e45df81e900f6306cbd7d0f47a07bdb28b1fed Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 15:07:20 +0530 Subject: [PATCH 03/39] Address HTTP benchmark review feedback --- .github/workflows/ci.yml | 71 ++++++++++++++++++++++++++++++++++++---- tests/benchmarks/http.js | 41 ++++++++--------------- 2 files changed, 78 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f5aa9034f..6f2f0c3bc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -678,13 +678,14 @@ jobs: - name: Benchmark baseline run: | - rm -f tests/benchmarks/http-summary.json benchmark-before.txt benchmark.txt + rm -f tests/benchmarks/http-summary.json benchmark-before-summary.json benchmark-after-summary.json benchmark-before.txt benchmark.txt docker run --rm -i --network host -v "$PWD:/scripts" -w /scripts grafana/k6 run --quiet \ -e APPWRITE_ENDPOINT=http://localhost/v1 \ -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ -e APPWRITE_BENCHMARK_ITERATIONS=1 \ -e APPWRITE_BENCHMARK_VUS=1 \ tests/benchmarks/http.js | tee benchmark-before.txt + cp tests/benchmarks/http-summary.json benchmark-before-summary.json - name: Benchmark after run: | @@ -694,14 +695,69 @@ jobs: -e APPWRITE_BENCHMARK_ITERATIONS=1 \ -e APPWRITE_BENCHMARK_VUS=1 \ tests/benchmarks/http.js | tee benchmark.txt + cp tests/benchmarks/http-summary.json benchmark-after-summary.json - name: Prepare comment run: | - { - echo '## :sparkles: Benchmark results' - echo - cat benchmark.txt - } > benchmark-comment.txt + node <<'NODE' > benchmark-comment.txt + const fs = require('fs'); + + const before = JSON.parse(fs.readFileSync('benchmark-before-summary.json', 'utf8')); + const after = JSON.parse(fs.readFileSync('benchmark-after-summary.json', 'utf8')); + + const trend = (data, metric, stat) => data.metrics?.[metric]?.values?.[stat]; + const value = (data, metric, stat) => data.metrics?.[metric]?.values?.[stat]; + const counter = (data, metric) => value(data, metric, 'count'); + const delta = (beforeValue, afterValue, suffix = '') => { + if (beforeValue === undefined || afterValue === undefined) { + return 'n/a'; + } + + const difference = afterValue - beforeValue; + return `${difference > 0 ? '+' : ''}${formatNumber(difference)}${suffix}`; + }; + const format = (value, suffix = '') => value === undefined ? 'n/a' : `${formatNumber(value)}${suffix}`; + const formatNumber = (value) => Number.isInteger(value) ? String(value) : value.toFixed(2).replace(/\.?0+$/, ''); + const row = (label, beforeValue, afterValue, suffix = '') => `| ${label} | ${format(beforeValue, suffix)} | ${format(afterValue, suffix)} | ${delta(beforeValue, afterValue, suffix)} |`; + + const rows = [ + row('HTTP total p95', trend(before, 'http_req_duration', 'p(95)'), trend(after, 'http_req_duration', 'p(95)'), 'ms'), + row('API endpoints p95', trend(before, 'appwrite_api_duration', 'p(95)'), trend(after, 'appwrite_api_duration', 'p(95)'), 'ms'), + row('Database worker p95', trend(before, 'appwrite_worker_database_duration', 'p(95)'), trend(after, 'appwrite_worker_database_duration', 'p(95)'), 'ms'), + row('TablesDB worker p95', trend(before, 'appwrite_worker_tables_duration', 'p(95)'), trend(after, 'appwrite_worker_tables_duration', 'p(95)'), 'ms'), + row('Mail worker p95', trend(before, 'appwrite_worker_mails_duration', 'p(95)'), trend(after, 'appwrite_worker_mails_duration', 'p(95)'), 'ms'), + row('Messaging worker p95', trend(before, 'appwrite_worker_messaging_duration', 'p(95)'), trend(after, 'appwrite_worker_messaging_duration', 'p(95)'), 'ms'), + row('Flow failures', counter(before, 'appwrite_benchmark_flow_failures'), counter(after, 'appwrite_benchmark_flow_failures')), + row('Check failures', value(before, 'checks', 'fails'), value(after, 'checks', 'fails')), + ]; + + const detail = (label, metric, suffix = 'ms') => { + const values = after.metrics?.[metric]?.values; + if (!values) { + return `${label}: no samples`; + } + + return `${label}: avg=${format(values.avg, suffix)} p90=${format(values['p(90)'], suffix)} p95=${format(values['p(95)'], suffix)} max=${format(values.max, suffix)}`; + }; + + console.log('## :sparkles: Benchmark results'); + console.log(); + console.log('Appwrite curated benchmark review'); + console.log('Before/after comparison'); + console.log('| Metric | Before | After | Delta |'); + console.log('| --- | ---: | ---: | ---: |'); + console.log(rows.join('\n')); + console.log('Current run details'); + console.log(detail('HTTP total', 'http_req_duration')); + console.log(detail('API endpoints', 'appwrite_api_duration')); + console.log(detail('Database worker schema jobs', 'appwrite_worker_database_duration')); + console.log(detail('TablesDB worker schema jobs', 'appwrite_worker_tables_duration')); + console.log(detail('Mail worker delivery', 'appwrite_worker_mails_duration')); + console.log(detail('Messaging worker delivery', 'appwrite_worker_messaging_duration')); + console.log(`Flow failures: ${format(counter(after, 'appwrite_benchmark_flow_failures'))}`); + console.log('Endpoint: http://localhost/v1'); + console.log('Maildev API: http://localhost:9503/email'); + NODE - name: Save results uses: actions/upload-artifact@v7 @@ -711,7 +767,8 @@ jobs: path: | benchmark-before.txt benchmark.txt - tests/benchmarks/http-summary.json + benchmark-before-summary.json + benchmark-after-summary.json retention-days: 7 - name: Find Comment diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 85f3daee95..7e04b40cc2 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -1,5 +1,6 @@ import http from 'k6/http'; import { check, group, sleep } from 'k6'; +import encoding from 'k6/encoding'; import { Counter, Trend } from 'k6/metrics'; const ENDPOINT = (__ENV.APPWRITE_ENDPOINT || 'http://localhost/v1').replace(/\/+$/, ''); @@ -221,6 +222,10 @@ export function curatedFlows(data) { } export function teardown(data) { + if (data && data.projectId && data.consoleSessionHeaders) { + rawRequest('DELETE', `/projects/${data.projectId}`, null, data.consoleSessionHeaders, 'teardown.projects.delete'); + } + if (data && data.teamId && data.consoleSessionHeaders) { rawRequest('DELETE', `/teams/${data.teamId}`, null, data.consoleSessionHeaders, 'teardown.teams.delete'); } @@ -706,7 +711,8 @@ function waitForStatus(path, headers, wantedStatus, timeoutMs) { const started = Date.now(); while (Date.now() - started < timeoutMs) { - const response = api('GET', path, null, headers, [200], `wait${path}`); + const response = rawRequest('GET', path, null, headers, `wait${path}`); + assertStatus(response, [200], `wait${path}`); if (response.json('status') === wantedStatus) { return response; } @@ -720,7 +726,8 @@ function waitForMessage(messageId, headers, timeoutMs) { const started = Date.now(); while (Date.now() - started < timeoutMs) { - const response = api('GET', `/messaging/messages/${messageId}`, null, headers, [200], 'messaging.messages.poll'); + const response = rawRequest('GET', `/messaging/messages/${messageId}`, null, headers, 'messaging.messages.poll'); + assertStatus(response, [200], 'messaging.messages.poll'); const status = response.json('status'); if (['sent', 'failed'].includes(status)) { @@ -851,28 +858,12 @@ function tablePayload() { } function onePixelPng() { - return base64ToBinary('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII='); -} - -function base64ToBinary(input) { - const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='; + const bytes = new Uint8Array(encoding.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=')); let output = ''; - let buffer = 0; - let bits = 0; - for (let i = 0; i < input.length; i++) { - const value = chars.indexOf(input.charAt(i)); - if (value < 0 || value === 64) { - continue; - } - - buffer = (buffer << 6) | value; - bits += 6; - - if (bits >= 8) { - bits -= 8; - output += String.fromCharCode((buffer >> bits) & 0xff); - } + // Appwrite's multipart upload path accepts this k6 fixture as a binary string. + for (let i = 0; i < bytes.length; i++) { + output += String.fromCharCode(bytes[i]); } return output; @@ -932,11 +923,7 @@ export function handleSummary(data) { function loadPreviousSummary() { try { - if (SUMMARY_PATH === 'tests/benchmarks/http-summary.json') { - return JSON.parse(open('http-summary.json')); - } - - return JSON.parse(open(SUMMARY_PATH)); + return JSON.parse(open('http-summary.json')); } catch (error) { return null; } From 2cfe40e98e5abd7efd927ae9b1bc84f5bae7d075 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 15:11:24 +0530 Subject: [PATCH 04/39] Compare benchmark against base branch --- .github/workflows/ci.yml | 48 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f2f0c3bc7..812b00fab2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -536,6 +536,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v6 + with: + fetch-depth: 1 - name: Download Docker Image uses: actions/download-artifact@v7 @@ -656,6 +658,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v6 + with: + fetch-depth: 1 - name: Download Docker Image uses: actions/download-artifact@v7 @@ -669,14 +673,50 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Load and Start Appwrite + - name: Load Appwrite image + run: | + docker load --input /tmp/${{ env.IMAGE }}.tar + + - name: Prepare benchmark baseline + if: github.event_name == 'pull_request' + run: | + git fetch --depth=1 origin ${{ github.event.pull_request.base.ref }} + git worktree add --detach /tmp/appwrite-benchmark-baseline FETCH_HEAD + + - name: Start baseline Appwrite + if: github.event_name == 'pull_request' + working-directory: /tmp/appwrite-benchmark-baseline run: | sed -i 's/traefik/localhost/g' .env - docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose up -d - sleep 10 + docker compose up -d --wait - name: Benchmark baseline + if: github.event_name == 'pull_request' + run: | + rm -f tests/benchmarks/http-summary.json benchmark-before-summary.json benchmark-after-summary.json benchmark-before.txt benchmark.txt + docker run --rm -i --network host -v "$PWD:/scripts" -w /scripts grafana/k6 run --quiet \ + -e APPWRITE_ENDPOINT=http://localhost/v1 \ + -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ + -e APPWRITE_BENCHMARK_ITERATIONS=1 \ + -e APPWRITE_BENCHMARK_VUS=1 \ + tests/benchmarks/http.js | tee benchmark-before.txt + cp tests/benchmarks/http-summary.json benchmark-before-summary.json + + - name: Stop baseline Appwrite + if: always() && github.event_name == 'pull_request' + run: | + if [ -d /tmp/appwrite-benchmark-baseline ]; then + cd /tmp/appwrite-benchmark-baseline + docker compose down -v + fi + + - name: Start PR Appwrite + run: | + sed -i 's/traefik/localhost/g' .env + docker compose up -d --wait + + - name: Seed workflow benchmark baseline + if: github.event_name != 'pull_request' run: | rm -f tests/benchmarks/http-summary.json benchmark-before-summary.json benchmark-after-summary.json benchmark-before.txt benchmark.txt docker run --rm -i --network host -v "$PWD:/scripts" -w /scripts grafana/k6 run --quiet \ From 6aeb2d2be08bb974f12bc7dcbe48182ee0da61d1 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 15:51:30 +0530 Subject: [PATCH 05/39] Fix benchmark before branch comparison --- .github/workflows/ci.yml | 64 +++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 812b00fab2..3af7581b66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -651,9 +651,12 @@ jobs: benchmark: name: Benchmark + if: github.event_name == 'pull_request' runs-on: ubuntu-latest needs: build permissions: + actions: read + contents: read pull-requests: write steps: - name: Checkout repository @@ -676,66 +679,61 @@ jobs: - name: Load Appwrite image run: | docker load --input /tmp/${{ env.IMAGE }}.tar + docker tag ${{ env.IMAGE }} ${{ env.IMAGE }}:after - - name: Prepare benchmark baseline - if: github.event_name == 'pull_request' + - name: Prepare benchmark before run: | - git fetch --depth=1 origin ${{ github.event.pull_request.base.ref }} - git worktree add --detach /tmp/appwrite-benchmark-baseline FETCH_HEAD + git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} + git worktree add --detach /tmp/appwrite-benchmark-before ${{ github.event.pull_request.base.sha }} + docker build \ + --target development \ + --build-arg DEBUG=false \ + --build-arg TESTING=true \ + --build-arg VERSION=dev \ + --tag ${{ env.IMAGE }}:before \ + /tmp/appwrite-benchmark-before - - name: Start baseline Appwrite - if: github.event_name == 'pull_request' - working-directory: /tmp/appwrite-benchmark-baseline + - name: Start before Appwrite + working-directory: /tmp/appwrite-benchmark-before run: | + docker tag ${{ env.IMAGE }}:before ${{ env.IMAGE }} sed -i 's/traefik/localhost/g' .env - docker compose up -d --wait + docker compose up -d --wait --no-build - - name: Benchmark baseline - if: github.event_name == 'pull_request' + - name: Benchmark before run: | - rm -f tests/benchmarks/http-summary.json benchmark-before-summary.json benchmark-after-summary.json benchmark-before.txt benchmark.txt - docker run --rm -i --network host -v "$PWD:/scripts" -w /scripts grafana/k6 run --quiet \ + rm -f benchmark-before-summary.json benchmark-after-summary.json benchmark-before.txt benchmark.txt + docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts grafana/k6 run --quiet \ -e APPWRITE_ENDPOINT=http://localhost/v1 \ -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ -e APPWRITE_BENCHMARK_ITERATIONS=1 \ -e APPWRITE_BENCHMARK_VUS=1 \ + -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-before-summary.json \ tests/benchmarks/http.js | tee benchmark-before.txt - cp tests/benchmarks/http-summary.json benchmark-before-summary.json - - name: Stop baseline Appwrite - if: always() && github.event_name == 'pull_request' + - name: Stop before Appwrite + if: always() run: | - if [ -d /tmp/appwrite-benchmark-baseline ]; then - cd /tmp/appwrite-benchmark-baseline + if [ -d /tmp/appwrite-benchmark-before ]; then + cd /tmp/appwrite-benchmark-before docker compose down -v fi - - name: Start PR Appwrite + - name: Start after Appwrite run: | + docker tag ${{ env.IMAGE }}:after ${{ env.IMAGE }} sed -i 's/traefik/localhost/g' .env - docker compose up -d --wait - - - name: Seed workflow benchmark baseline - if: github.event_name != 'pull_request' - run: | - rm -f tests/benchmarks/http-summary.json benchmark-before-summary.json benchmark-after-summary.json benchmark-before.txt benchmark.txt - docker run --rm -i --network host -v "$PWD:/scripts" -w /scripts grafana/k6 run --quiet \ - -e APPWRITE_ENDPOINT=http://localhost/v1 \ - -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ - -e APPWRITE_BENCHMARK_ITERATIONS=1 \ - -e APPWRITE_BENCHMARK_VUS=1 \ - tests/benchmarks/http.js | tee benchmark-before.txt - cp tests/benchmarks/http-summary.json benchmark-before-summary.json + docker compose up -d --wait --no-build - name: Benchmark after run: | - docker run --rm -i --network host -v "$PWD:/scripts" -w /scripts grafana/k6 run --quiet \ + docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts grafana/k6 run --quiet \ -e APPWRITE_ENDPOINT=http://localhost/v1 \ -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ -e APPWRITE_BENCHMARK_ITERATIONS=1 \ -e APPWRITE_BENCHMARK_VUS=1 \ + -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-after-summary.json \ tests/benchmarks/http.js | tee benchmark.txt - cp tests/benchmarks/http-summary.json benchmark-after-summary.json - name: Prepare comment run: | From dcd01a8fb04fbd1d6d364ce237e3fac3862dacfb Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 16:06:30 +0530 Subject: [PATCH 06/39] Tidy benchmark PR comment --- .github/workflows/ci.yml | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3af7581b66..d0d1330700 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -657,6 +657,7 @@ jobs: permissions: actions: read contents: read + issues: write pull-requests: write steps: - name: Checkout repository @@ -772,29 +773,32 @@ jobs: const detail = (label, metric, suffix = 'ms') => { const values = after.metrics?.[metric]?.values; if (!values) { - return `${label}: no samples`; + return `- **${label}:** no samples`; } - return `${label}: avg=${format(values.avg, suffix)} p90=${format(values['p(90)'], suffix)} p95=${format(values['p(95)'], suffix)} max=${format(values.max, suffix)}`; + return `- **${label}:** avg=${format(values.avg, suffix)} p90=${format(values['p(90)'], suffix)} p95=${format(values['p(95)'], suffix)} max=${format(values.max, suffix)}`; }; + console.log(''); console.log('## :sparkles: Benchmark results'); console.log(); - console.log('Appwrite curated benchmark review'); - console.log('Before/after comparison'); + console.log(`Comparing \`${{ github.event.pull_request.base.ref }}\` (before) to \`${{ github.event.pull_request.head.ref }}\` (after).`); + console.log(); console.log('| Metric | Before | After | Delta |'); console.log('| --- | ---: | ---: | ---: |'); console.log(rows.join('\n')); - console.log('Current run details'); + console.log(); + console.log('
'); + console.log('Current run details'); + console.log(); console.log(detail('HTTP total', 'http_req_duration')); console.log(detail('API endpoints', 'appwrite_api_duration')); console.log(detail('Database worker schema jobs', 'appwrite_worker_database_duration')); console.log(detail('TablesDB worker schema jobs', 'appwrite_worker_tables_duration')); console.log(detail('Mail worker delivery', 'appwrite_worker_mails_duration')); console.log(detail('Messaging worker delivery', 'appwrite_worker_messaging_duration')); - console.log(`Flow failures: ${format(counter(after, 'appwrite_benchmark_flow_failures'))}`); - console.log('Endpoint: http://localhost/v1'); - console.log('Maildev API: http://localhost:9503/email'); + console.log(); + console.log('
'); NODE - name: Save results @@ -813,6 +817,15 @@ jobs: if: github.event.pull_request.head.repo.full_name == github.repository uses: peter-evans/find-comment@v3 id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: appwrite-benchmark-results + + - name: Find Legacy Comment + if: github.event.pull_request.head.repo.full_name == github.repository && steps.fc.outputs.comment-id == '' + uses: peter-evans/find-comment@v3 + id: legacy_fc with: issue-number: ${{ github.event.pull_request.number }} comment-author: 'github-actions[bot]' @@ -822,7 +835,7 @@ jobs: if: github.event.pull_request.head.repo.full_name == github.repository uses: peter-evans/create-or-update-comment@v4 with: - comment-id: ${{ steps.fc.outputs.comment-id }} + comment-id: ${{ steps.fc.outputs.comment-id || steps.legacy_fc.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} body-path: benchmark-comment.txt edit-mode: replace From 83f182b444228b64839dead5af848d5dfaa6d951 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 16:10:08 +0530 Subject: [PATCH 07/39] Address benchmark review feedback --- .github/workflows/ci.yml | 9 ++++++++- tests/benchmarks/http.js | 19 +++++-------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0d1330700..bf9831582f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -704,13 +704,15 @@ jobs: - name: Benchmark before run: | rm -f benchmark-before-summary.json benchmark-after-summary.json benchmark-before.txt benchmark.txt - docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts grafana/k6 run --quiet \ + docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "/tmp/appwrite-benchmark-before:/scripts" -w /scripts grafana/k6 run --quiet \ + --summary-export benchmark-before-summary.json \ -e APPWRITE_ENDPOINT=http://localhost/v1 \ -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ -e APPWRITE_BENCHMARK_ITERATIONS=1 \ -e APPWRITE_BENCHMARK_VUS=1 \ -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-before-summary.json \ tests/benchmarks/http.js | tee benchmark-before.txt + cp /tmp/appwrite-benchmark-before/benchmark-before-summary.json benchmark-before-summary.json - name: Stop before Appwrite if: always() @@ -733,9 +735,14 @@ jobs: -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ -e APPWRITE_BENCHMARK_ITERATIONS=1 \ -e APPWRITE_BENCHMARK_VUS=1 \ + -e APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH=/scripts/benchmark-before-summary.json \ -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-after-summary.json \ tests/benchmarks/http.js | tee benchmark.txt + - name: Stop after Appwrite + if: always() + run: docker compose down -v + - name: Prepare comment run: | node <<'NODE' > benchmark-comment.txt diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 7e04b40cc2..6ce5afd661 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -14,6 +14,7 @@ const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 60000); const ITERATIONS = Number(__ENV.APPWRITE_BENCHMARK_ITERATIONS || 1); const VUS = Number(__ENV.APPWRITE_BENCHMARK_VUS || 1); const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || 'tests/benchmarks/http-summary.json'; +const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || SUMMARY_PATH; const PREVIOUS_SUMMARY = loadPreviousSummary(); export const apiDuration = new Trend('appwrite_api_duration', true); @@ -712,8 +713,7 @@ function waitForStatus(path, headers, wantedStatus, timeoutMs) { while (Date.now() - started < timeoutMs) { const response = rawRequest('GET', path, null, headers, `wait${path}`); - assertStatus(response, [200], `wait${path}`); - if (response.json('status') === wantedStatus) { + if (response.status === 200 && response.json('status') === wantedStatus) { return response; } sleep(0.5); @@ -727,8 +727,7 @@ function waitForMessage(messageId, headers, timeoutMs) { while (Date.now() - started < timeoutMs) { const response = rawRequest('GET', `/messaging/messages/${messageId}`, null, headers, 'messaging.messages.poll'); - assertStatus(response, [200], 'messaging.messages.poll'); - const status = response.json('status'); + const status = response.status === 200 ? response.json('status') : null; if (['sent', 'failed'].includes(status)) { if (status === 'failed') { @@ -858,15 +857,7 @@ function tablePayload() { } function onePixelPng() { - const bytes = new Uint8Array(encoding.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=')); - let output = ''; - - // Appwrite's multipart upload path accepts this k6 fixture as a binary string. - for (let i = 0; i < bytes.length; i++) { - output += String.fromCharCode(bytes[i]); - } - - return output; + return encoding.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=', 'std', 'b'); } function flattenMultipartArray(key, values) { @@ -923,7 +914,7 @@ export function handleSummary(data) { function loadPreviousSummary() { try { - return JSON.parse(open('http-summary.json')); + return JSON.parse(open(PREVIOUS_SUMMARY_PATH)); } catch (error) { return null; } From 51bc3dc1d55321df3f4da4db8f306c2176182e92 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 16:22:04 +0530 Subject: [PATCH 08/39] Migrate HTTP benchmark to PHP --- .github/workflows/ci.yml | 139 ++-- tests/benchmarks/http.js | 996 ---------------------------- tests/benchmarks/http.php | 1299 +++++++++++++++++++++++++++++++++++++ 3 files changed, 1375 insertions(+), 1059 deletions(-) delete mode 100644 tests/benchmarks/http.js create mode 100644 tests/benchmarks/http.php diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf9831582f..78d959779c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -704,15 +704,13 @@ jobs: - name: Benchmark before run: | rm -f benchmark-before-summary.json benchmark-after-summary.json benchmark-before.txt benchmark.txt - docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "/tmp/appwrite-benchmark-before:/scripts" -w /scripts grafana/k6 run --quiet \ - --summary-export benchmark-before-summary.json \ + docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts \ -e APPWRITE_ENDPOINT=http://localhost/v1 \ -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ -e APPWRITE_BENCHMARK_ITERATIONS=1 \ -e APPWRITE_BENCHMARK_VUS=1 \ -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-before-summary.json \ - tests/benchmarks/http.js | tee benchmark-before.txt - cp /tmp/appwrite-benchmark-before/benchmark-before-summary.json benchmark-before-summary.json + ${{ env.IMAGE }}:after php tests/benchmarks/http.php | tee benchmark-before.txt - name: Stop before Appwrite if: always() @@ -730,14 +728,14 @@ jobs: - name: Benchmark after run: | - docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts grafana/k6 run --quiet \ + docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts \ -e APPWRITE_ENDPOINT=http://localhost/v1 \ -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ -e APPWRITE_BENCHMARK_ITERATIONS=1 \ -e APPWRITE_BENCHMARK_VUS=1 \ - -e APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH=/scripts/benchmark-before-summary.json \ + -e APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH=benchmark-before-summary.json \ -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-after-summary.json \ - tests/benchmarks/http.js | tee benchmark.txt + ${{ env.IMAGE }}:after php tests/benchmarks/http.php | tee benchmark.txt - name: Stop after Appwrite if: always() @@ -745,68 +743,83 @@ jobs: - name: Prepare comment run: | - node <<'NODE' > benchmark-comment.txt - const fs = require('fs'); + docker run --rm -i -v "$PWD:/scripts" -w /scripts ${{ env.IMAGE }}:after php <<'PHP' > benchmark-comment.txt + data.metrics?.[metric]?.values?.[stat]; - const value = (data, metric, stat) => data.metrics?.[metric]?.values?.[stat]; - const counter = (data, metric) => value(data, metric, 'count'); - const delta = (beforeValue, afterValue, suffix = '') => { - if (beforeValue === undefined || afterValue === undefined) { - return 'n/a'; - } + function metric_value(?array $data, string $metric, string $stat): mixed + { + return $data['metrics'][$metric]['values'][$stat] ?? null; + } - const difference = afterValue - beforeValue; - return `${difference > 0 ? '+' : ''}${formatNumber(difference)}${suffix}`; - }; - const format = (value, suffix = '') => value === undefined ? 'n/a' : `${formatNumber(value)}${suffix}`; - const formatNumber = (value) => Number.isInteger(value) ? String(value) : value.toFixed(2).replace(/\.?0+$/, ''); - const row = (label, beforeValue, afterValue, suffix = '') => `| ${label} | ${format(beforeValue, suffix)} | ${format(afterValue, suffix)} | ${delta(beforeValue, afterValue, suffix)} |`; + function format_number(mixed $value): string + { + $value = round((float) $value, 2); + return rtrim(rtrim(number_format($value, 2, '.', ''), '0'), '.'); + } - const rows = [ - row('HTTP total p95', trend(before, 'http_req_duration', 'p(95)'), trend(after, 'http_req_duration', 'p(95)'), 'ms'), - row('API endpoints p95', trend(before, 'appwrite_api_duration', 'p(95)'), trend(after, 'appwrite_api_duration', 'p(95)'), 'ms'), - row('Database worker p95', trend(before, 'appwrite_worker_database_duration', 'p(95)'), trend(after, 'appwrite_worker_database_duration', 'p(95)'), 'ms'), - row('TablesDB worker p95', trend(before, 'appwrite_worker_tables_duration', 'p(95)'), trend(after, 'appwrite_worker_tables_duration', 'p(95)'), 'ms'), - row('Mail worker p95', trend(before, 'appwrite_worker_mails_duration', 'p(95)'), trend(after, 'appwrite_worker_mails_duration', 'p(95)'), 'ms'), - row('Messaging worker p95', trend(before, 'appwrite_worker_messaging_duration', 'p(95)'), trend(after, 'appwrite_worker_messaging_duration', 'p(95)'), 'ms'), - row('Flow failures', counter(before, 'appwrite_benchmark_flow_failures'), counter(after, 'appwrite_benchmark_flow_failures')), - row('Check failures', value(before, 'checks', 'fails'), value(after, 'checks', 'fails')), + function format_value(mixed $value, string $suffix = ''): string + { + return $value === null ? 'n/a' : format_number($value) . $suffix; + } + + function delta(mixed $beforeValue, mixed $afterValue, string $suffix = ''): string + { + if ($beforeValue === null || $afterValue === null) { + return 'n/a'; + } + + $difference = round((float) $afterValue - (float) $beforeValue, 2); + return ($difference > 0 ? '+' : '') . format_number($difference) . $suffix; + } + + function row(string $label, mixed $beforeValue, mixed $afterValue, string $suffix = ''): string + { + return '| ' . $label . ' | ' . format_value($beforeValue, $suffix) . ' | ' . format_value($afterValue, $suffix) . ' | ' . delta($beforeValue, $afterValue, $suffix) . ' |'; + } + + function detail(array $after, string $label, string $metric, string $suffix = 'ms'): string + { + $values = $after['metrics'][$metric]['values'] ?? null; + if (!is_array($values)) { + return '- **' . $label . ':** no samples'; + } + + return '- **' . $label . ':** avg=' . format_value($values['avg'] ?? null, $suffix) + . ' p90=' . format_value($values['p(90)'] ?? null, $suffix) + . ' p95=' . format_value($values['p(95)'] ?? null, $suffix) + . ' max=' . format_value($values['max'] ?? null, $suffix); + } + + $rows = [ + row('HTTP total p95', metric_value($before, 'http_req_duration', 'p(95)'), metric_value($after, 'http_req_duration', 'p(95)'), 'ms'), + row('API endpoints p95', metric_value($before, 'appwrite_api_duration', 'p(95)'), metric_value($after, 'appwrite_api_duration', 'p(95)'), 'ms'), + row('Database worker p95', metric_value($before, 'appwrite_worker_database_duration', 'p(95)'), metric_value($after, 'appwrite_worker_database_duration', 'p(95)'), 'ms'), + row('TablesDB worker p95', metric_value($before, 'appwrite_worker_tables_duration', 'p(95)'), metric_value($after, 'appwrite_worker_tables_duration', 'p(95)'), 'ms'), + row('Mail worker p95', metric_value($before, 'appwrite_worker_mails_duration', 'p(95)'), metric_value($after, 'appwrite_worker_mails_duration', 'p(95)'), 'ms'), + row('Messaging worker p95', metric_value($before, 'appwrite_worker_messaging_duration', 'p(95)'), metric_value($after, 'appwrite_worker_messaging_duration', 'p(95)'), 'ms'), + row('Flow failures', metric_value($before, 'appwrite_benchmark_flow_failures', 'count'), metric_value($after, 'appwrite_benchmark_flow_failures', 'count')), + row('Check failures', metric_value($before, 'checks', 'fails'), metric_value($after, 'checks', 'fails')), ]; - const detail = (label, metric, suffix = 'ms') => { - const values = after.metrics?.[metric]?.values; - if (!values) { - return `- **${label}:** no samples`; - } - - return `- **${label}:** avg=${format(values.avg, suffix)} p90=${format(values['p(90)'], suffix)} p95=${format(values['p(95)'], suffix)} max=${format(values.max, suffix)}`; - }; - - console.log(''); - console.log('## :sparkles: Benchmark results'); - console.log(); - console.log(`Comparing \`${{ github.event.pull_request.base.ref }}\` (before) to \`${{ github.event.pull_request.head.ref }}\` (after).`); - console.log(); - console.log('| Metric | Before | After | Delta |'); - console.log('| --- | ---: | ---: | ---: |'); - console.log(rows.join('\n')); - console.log(); - console.log('
'); - console.log('Current run details'); - console.log(); - console.log(detail('HTTP total', 'http_req_duration')); - console.log(detail('API endpoints', 'appwrite_api_duration')); - console.log(detail('Database worker schema jobs', 'appwrite_worker_database_duration')); - console.log(detail('TablesDB worker schema jobs', 'appwrite_worker_tables_duration')); - console.log(detail('Mail worker delivery', 'appwrite_worker_mails_duration')); - console.log(detail('Messaging worker delivery', 'appwrite_worker_messaging_duration')); - console.log(); - console.log('
'); - NODE + echo "\n"; + echo "## :sparkles: Benchmark results\n\n"; + echo 'Comparing `${{ github.event.pull_request.base.ref }}` (before) to `${{ github.event.pull_request.head.ref }}` (after).' . "\n\n"; + echo "| Metric | Before | After | Delta |\n"; + echo "| --- | ---: | ---: | ---: |\n"; + echo implode("\n", $rows) . "\n\n"; + echo "
\n"; + echo "Current run details\n\n"; + echo detail($after, 'HTTP total', 'http_req_duration') . "\n"; + echo detail($after, 'API endpoints', 'appwrite_api_duration') . "\n"; + echo detail($after, 'Database worker schema jobs', 'appwrite_worker_database_duration') . "\n"; + echo detail($after, 'TablesDB worker schema jobs', 'appwrite_worker_tables_duration') . "\n"; + echo detail($after, 'Mail worker delivery', 'appwrite_worker_mails_duration') . "\n"; + echo detail($after, 'Messaging worker delivery', 'appwrite_worker_messaging_duration') . "\n\n"; + echo "
\n"; + PHP - name: Save results uses: actions/upload-artifact@v7 diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js deleted file mode 100644 index 6ce5afd661..0000000000 --- a/tests/benchmarks/http.js +++ /dev/null @@ -1,996 +0,0 @@ -import http from 'k6/http'; -import { check, group, sleep } from 'k6'; -import encoding from 'k6/encoding'; -import { Counter, Trend } from 'k6/metrics'; - -const ENDPOINT = (__ENV.APPWRITE_ENDPOINT || 'http://localhost/v1').replace(/\/+$/, ''); -const MAILDEV_ENDPOINT = __ENV.APPWRITE_MAILDEV_ENDPOINT || 'http://localhost:9503/email'; -const CONSOLE_PROJECT = __ENV.APPWRITE_CONSOLE_PROJECT || 'console'; -const REGION = __ENV.APPWRITE_REGION || 'default'; -const REDIRECT_URL = __ENV.APPWRITE_BENCHMARK_REDIRECT_URL || 'http://localhost'; -const PASSWORD = __ENV.APPWRITE_BENCHMARK_PASSWORD || 'Password123!'; -const MAIL_TIMEOUT_MS = Number(__ENV.APPWRITE_MAIL_TIMEOUT_MS || 20000); -const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 60000); -const ITERATIONS = Number(__ENV.APPWRITE_BENCHMARK_ITERATIONS || 1); -const VUS = Number(__ENV.APPWRITE_BENCHMARK_VUS || 1); -const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || 'tests/benchmarks/http-summary.json'; -const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || SUMMARY_PATH; -const PREVIOUS_SUMMARY = loadPreviousSummary(); - -export const apiDuration = new Trend('appwrite_api_duration', true); -export const databaseWorkerDuration = new Trend('appwrite_worker_database_duration', true); -export const tablesWorkerDuration = new Trend('appwrite_worker_tables_duration', true); -export const mailsWorkerDuration = new Trend('appwrite_worker_mails_duration', true); -export const messagingWorkerDuration = new Trend('appwrite_worker_messaging_duration', true); -export const flowFailures = new Counter('appwrite_benchmark_flow_failures'); - -export const options = { - scenarios: { - curated_flows: { - executor: 'shared-iterations', - exec: 'curatedFlows', - vus: VUS, - iterations: ITERATIONS, - maxDuration: __ENV.APPWRITE_BENCHMARK_MAX_DURATION || '30m', - }, - }, - thresholds: { - http_req_failed: ['rate<0.05'], - appwrite_api_duration: ['p(95)<2000'], - appwrite_benchmark_flow_failures: ['count<1'], - }, -}; - -const API_SCOPES = [ - 'sessions.write', - 'users.read', - 'users.write', - 'teams.read', - 'teams.write', - 'databases.read', - 'databases.write', - 'collections.read', - 'collections.write', - 'tables.read', - 'tables.write', - 'attributes.read', - 'attributes.write', - 'columns.read', - 'columns.write', - 'indexes.read', - 'indexes.write', - 'documents.read', - 'documents.write', - 'rows.read', - 'rows.write', - 'files.read', - 'files.write', - 'buckets.read', - 'buckets.write', - 'functions.read', - 'functions.write', - 'sites.read', - 'sites.write', - 'log.read', - 'log.write', - 'execution.read', - 'execution.write', - 'locale.read', - 'avatars.read', - 'health.read', - 'providers.read', - 'providers.write', - 'messages.read', - 'messages.write', - 'topics.read', - 'topics.write', - 'subscribers.read', - 'subscribers.write', - 'targets.read', - 'targets.write', - 'rules.read', - 'rules.write', - 'migrations.read', - 'migrations.write', - 'vcs.read', - 'vcs.write', - 'assistant.read', - 'tokens.read', - 'tokens.write', - 'platforms.read', - 'platforms.write', -]; - -const BASE_PERMISSIONS = [ - 'read("any")', - 'create("any")', - 'update("any")', - 'delete("any")', -]; - -const ITEM_PERMISSIONS = [ - 'read("any")', - 'update("any")', - 'delete("any")', -]; - -export function setup() { - const runId = unique('run'); - const consoleEmail = __ENV.APPWRITE_ADMIN_EMAIL || `bench-admin-${runId}@example.com`; - const consolePassword = __ENV.APPWRITE_ADMIN_PASSWORD || PASSWORD; - - const consoleHeaders = { - 'Content-Type': 'application/json', - 'X-Appwrite-Project': CONSOLE_PROJECT, - }; - - const account = rawRequest('POST', '/account', { - userId: unique('admin'), - email: consoleEmail, - password: consolePassword, - name: 'Benchmark Admin', - }, consoleHeaders, 'setup.account.create'); - - if (![201, 409].includes(account.status)) { - failResponse(account, 'Unable to create or reuse the benchmark console account'); - } - - const session = rawRequest('POST', '/account/sessions/email', { - email: consoleEmail, - password: consolePassword, - }, consoleHeaders, 'setup.account.session'); - - assertStatus(session, [201], 'console session created'); - - const consoleSessionHeaders = { - ...consoleHeaders, - Cookie: cookieHeader(session), - }; - - const team = api('POST', '/teams', { - teamId: unique('team'), - name: `Benchmark Team ${runId}`, - }, consoleSessionHeaders, [201], 'setup.teams.create'); - - const teamId = team.json('$id'); - const project = api('POST', '/projects', { - projectId: unique('project'), - name: `Benchmark Project ${runId}`, - teamId, - region: REGION, - }, consoleSessionHeaders, [201], 'setup.projects.create'); - - const projectId = project.json('$id'); - const key = api('POST', `/projects/${projectId}/keys`, { - keyId: unique('key'), - name: 'Benchmark API key', - scopes: API_SCOPES, - }, consoleSessionHeaders, [201], 'setup.projects.keys.create'); - - const apiHeaders = { - 'Content-Type': 'application/json', - 'X-Appwrite-Project': projectId, - 'X-Appwrite-Key': key.json('secret'), - }; - - const platform = api('POST', '/project/platforms/web', { - platformId: unique('web'), - name: 'Benchmark web', - hostname: hostnameFromUrl(REDIRECT_URL), - }, apiHeaders, [201, 409], 'setup.project.platforms.web.create'); - - const smtp = rawRequest('PATCH', `/projects/${projectId}/smtp`, { - enabled: true, - senderName: 'Benchmark', - senderEmail: 'benchmark@appwrite.io', - replyTo: 'benchmark@appwrite.io', - host: __ENV.APPWRITE_SMTP_HOST || 'maildev', - port: Number(__ENV.APPWRITE_SMTP_PORT || 1025), - username: __ENV.APPWRITE_SMTP_USERNAME || 'user', - password: __ENV.APPWRITE_SMTP_PASSWORD || 'password', - ...(String(__ENV.APPWRITE_SMTP_SECURE || '') !== '' ? { secure: __ENV.APPWRITE_SMTP_SECURE } : {}), - }, consoleSessionHeaders, 'setup.projects.smtp.update'); - - if (smtp.status !== 200) { - console.warn(`Custom SMTP was not enabled (${smtp.status}). Mail worker timings may be unavailable.`); - } - - return { - runId, - teamId, - projectId, - consoleSessionHeaders, - apiHeaders, - platformStatus: platform.status, - }; -} - -export function curatedFlows(data) { - const ctx = { ...data }; - - try { - group('account and mail worker', () => accountFlow(ctx)); - group('databases documents flow', () => databasesFlow(ctx)); - group('tablesdb rows flow', () => tablesDbFlow(ctx)); - group('storage files and tokens flow', () => storageFlow(ctx)); - group('messaging worker flow', () => messagingFlow(ctx)); - group('functions and sites control-plane flow', () => computeFlow(ctx)); - group('health and queue probes', () => healthFlow(ctx)); - } catch (error) { - flowFailures.add(1); - throw error; - } -} - -export function teardown(data) { - if (data && data.projectId && data.consoleSessionHeaders) { - rawRequest('DELETE', `/projects/${data.projectId}`, null, data.consoleSessionHeaders, 'teardown.projects.delete'); - } - - if (data && data.teamId && data.consoleSessionHeaders) { - rawRequest('DELETE', `/teams/${data.teamId}`, null, data.consoleSessionHeaders, 'teardown.teams.delete'); - } -} - -function accountFlow(ctx) { - const userId = unique('user'); - const email = `bench-user-${unique('mail')}@example.com`; - const headers = projectHeaders(ctx.projectId); - - api('POST', '/account', { - userId, - email, - password: PASSWORD, - name: 'Benchmark User', - }, headers, [201], 'account.create'); - - const session = api('POST', '/account/sessions/email', { - email, - password: PASSWORD, - }, headers, [201], 'account.sessions.email.create'); - - const sessionHeaders = { - ...headers, - Cookie: cookieHeader(session), - }; - - ctx.userId = userId; - ctx.userEmail = email; - ctx.sessionHeaders = sessionHeaders; - - const jwt = api('POST', '/account/jwts', null, sessionHeaders, [201], 'account.jwts.create'); - ctx.jwtHeaders = { - ...headers, - 'X-Appwrite-JWT': jwt.json('jwt'), - }; - - api('GET', '/account', null, sessionHeaders, [200], 'account.get'); - api('GET', '/account/logs', null, sessionHeaders, [200], 'account.logs.list'); - api('PATCH', '/account/prefs', { prefs: { benchmark: true, runId: ctx.runId } }, sessionHeaders, [200], 'account.prefs.update'); - api('PATCH', '/account/name', { name: 'Benchmark User Updated' }, sessionHeaders, [200], 'account.name.update'); - api('PATCH', '/account/password', { password: `${PASSWORD}2`, oldPassword: PASSWORD }, sessionHeaders, [200], 'account.password.update'); - - const verificationStarted = Date.now(); - api('POST', '/account/verifications/email', { url: REDIRECT_URL }, sessionHeaders, [201], 'account.emailVerification.create'); - const verificationEmail = waitForEmail(email, (message) => { - return includes(message.subject, 'verify') - || includes(message.subject, 'verification') - || includes(message.html, 'verify') - || includes(message.html, 'verification') - || includes(message.text, 'verify') - || includes(message.text, 'verification'); - }, MAIL_TIMEOUT_MS); - mailsWorkerDuration.add(Date.now() - verificationStarted, { job: 'email_verification' }); - - const verification = extractQueryParams(verificationEmail); - if (verification.userId && verification.secret) { - api('PUT', '/account/verifications/email', { - userId: verification.userId, - secret: verification.secret, - }, sessionHeaders, [200], 'account.emailVerification.update'); - } - - const recoveryStarted = Date.now(); - api('POST', '/account/recovery', { email, url: REDIRECT_URL }, headers, [201], 'account.recovery.create'); - const recoveryEmail = waitForEmail(email, (message) => { - return includes(message.subject, 'recovery') - || includes(message.subject, 'recover') - || includes(message.subject, 'reset') - || includes(message.html, 'recovery') - || includes(message.html, 'recover') - || includes(message.html, 'reset') - || includes(message.text, 'recovery') - || includes(message.text, 'recover') - || includes(message.text, 'reset'); - }, MAIL_TIMEOUT_MS); - mailsWorkerDuration.add(Date.now() - recoveryStarted, { job: 'password_recovery' }); - - const recovery = extractQueryParams(recoveryEmail); - if (recovery.userId && recovery.secret) { - api('DELETE', '/account/sessions/current', null, sessionHeaders, [204], 'account.sessions.current.delete'); - - api('PUT', '/account/recovery', { - userId: recovery.userId, - secret: recovery.secret, - password: `${PASSWORD}3`, - }, headers, [200], 'account.recovery.update'); - - const recoveredSession = api('POST', '/account/sessions/email', { - email, - password: `${PASSWORD}3`, - }, headers, [201], 'account.sessions.email.recovered'); - - ctx.sessionHeaders = { - ...headers, - Cookie: cookieHeader(recoveredSession), - }; - - const recoveredJwt = api('POST', '/account/jwts', null, ctx.sessionHeaders, [201], 'account.jwts.recovered'); - ctx.jwtHeaders = { - ...headers, - 'X-Appwrite-JWT': recoveredJwt.json('jwt'), - }; - } -} - -function databasesFlow(ctx) { - const databaseId = unique('db'); - const collectionId = unique('col'); - const documentId = unique('doc'); - const indexKey = unique('idx'); - - api('POST', '/databases', { databaseId, name: 'Benchmark DB' }, ctx.apiHeaders, [201], 'databases.create'); - api('POST', `/databases/${databaseId}/collections`, { - collectionId, - name: 'Benchmark Collection', - permissions: BASE_PERMISSIONS, - documentSecurity: false, - }, ctx.apiHeaders, [201], 'databases.collections.create'); - - const attributes = [ - ['string', 'title', { size: 128 }], - ['integer', 'count', { min: 0, max: 100000 }], - ['email', 'email', {}], - ['boolean', 'active', {}], - ['datetime', 'publishedAt', {}], - ['float', 'score', { min: 0, max: 1000 }], - ['url', 'url', {}], - ['ip', 'ip', {}], - ]; - - for (const [type, key, extra] of attributes) { - const started = Date.now(); - api('POST', `/databases/${databaseId}/collections/${collectionId}/attributes/${type}`, { - key, - required: false, - array: false, - ...extra, - }, ctx.apiHeaders, [202], `databases.attributes.${type}.create`); - waitForStatus(`/databases/${databaseId}/collections/${collectionId}/attributes/${key}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); - databaseWorkerDuration.add(Date.now() - started, { job: `attribute_${type}` }); - } - - const indexStarted = Date.now(); - api('POST', `/databases/${databaseId}/collections/${collectionId}/indexes`, { - key: indexKey, - type: 'key', - attributes: ['title'], - orders: ['asc'], - }, ctx.apiHeaders, [202], 'databases.indexes.create'); - waitForStatus(`/databases/${databaseId}/collections/${collectionId}/indexes/${indexKey}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); - databaseWorkerDuration.add(Date.now() - indexStarted, { job: 'index' }); - - api('POST', `/databases/${databaseId}/collections/${collectionId}/documents`, { - documentId, - data: documentPayload(), - permissions: ITEM_PERMISSIONS, - }, ctx.apiHeaders, [201], 'databases.documents.create'); - api('GET', `/databases/${databaseId}/collections/${collectionId}/documents`, null, ctx.apiHeaders, [200], 'databases.documents.list'); - api('GET', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}`, null, ctx.apiHeaders, [200], 'databases.documents.get'); - api('PATCH', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}`, { - data: { title: 'Benchmark Document Updated' }, - }, ctx.apiHeaders, [200], 'databases.documents.update'); - api('PATCH', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}/count/increment`, { - value: 1, - }, ctx.apiHeaders, [200], 'databases.documents.increment'); - api('PATCH', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}/count/decrement`, { - value: 1, - }, ctx.apiHeaders, [200], 'databases.documents.decrement'); - api('DELETE', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}`, null, ctx.apiHeaders, [204], 'databases.documents.delete'); - api('DELETE', `/databases/${databaseId}`, null, ctx.apiHeaders, [204], 'databases.delete'); -} - -function tablesDbFlow(ctx) { - const databaseId = unique('tdb'); - const tableId = unique('tbl'); - const rowId = unique('row'); - const indexKey = unique('tidx'); - - api('POST', '/tablesdb', { databaseId, name: 'Benchmark TablesDB' }, ctx.apiHeaders, [201], 'tablesdb.create'); - api('POST', `/tablesdb/${databaseId}/tables`, { - tableId, - name: 'Benchmark Table', - permissions: BASE_PERMISSIONS, - rowSecurity: false, - }, ctx.apiHeaders, [201], 'tablesdb.tables.create'); - - const columns = [ - ['string', 'title', { size: 128 }], - ['integer', 'count', { min: 0, max: 100000 }], - ['email', 'email', {}], - ['boolean', 'active', {}], - ]; - - for (const [type, key, extra] of columns) { - const started = Date.now(); - api('POST', `/tablesdb/${databaseId}/tables/${tableId}/columns/${type}`, { - key, - required: false, - array: false, - ...extra, - }, ctx.apiHeaders, [202], `tablesdb.columns.${type}.create`); - waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/columns/${key}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); - tablesWorkerDuration.add(Date.now() - started, { job: `column_${type}` }); - } - - const indexStarted = Date.now(); - api('POST', `/tablesdb/${databaseId}/tables/${tableId}/indexes`, { - key: indexKey, - type: 'key', - columns: ['title'], - orders: ['asc'], - }, ctx.apiHeaders, [202], 'tablesdb.indexes.create'); - waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/indexes/${indexKey}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); - tablesWorkerDuration.add(Date.now() - indexStarted, { job: 'index' }); - - api('POST', `/tablesdb/${databaseId}/tables/${tableId}/rows`, { - rowId, - data: tablePayload(), - permissions: ITEM_PERMISSIONS, - }, ctx.sessionHeaders, [201], 'tablesdb.rows.create'); - api('GET', `/tablesdb/${databaseId}/tables/${tableId}/rows`, null, ctx.sessionHeaders, [200], 'tablesdb.rows.list'); - api('GET', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, null, ctx.sessionHeaders, [200], 'tablesdb.rows.get'); - api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, { - data: { title: 'Benchmark Row Updated' }, - }, ctx.sessionHeaders, [200], 'tablesdb.rows.update'); - api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}/count/increment`, { - value: 1, - }, ctx.sessionHeaders, [200], 'tablesdb.rows.increment'); - api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}/count/decrement`, { - value: 1, - }, ctx.sessionHeaders, [200], 'tablesdb.rows.decrement'); - api('DELETE', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, null, ctx.sessionHeaders, [204], 'tablesdb.rows.delete'); - api('DELETE', `/tablesdb/${databaseId}`, null, ctx.apiHeaders, [204], 'tablesdb.delete'); -} - -function storageFlow(ctx) { - const bucketId = unique('bucket'); - const fileId = unique('file'); - - api('POST', '/storage/buckets', { - bucketId, - name: 'Benchmark Bucket', - permissions: BASE_PERMISSIONS, - fileSecurity: false, - enabled: true, - maximumFileSize: 30000000, - allowedFileExtensions: [], - compression: 'none', - encryption: false, - antivirus: false, - }, ctx.apiHeaders, [201], 'storage.buckets.create'); - - const multipartHeaders = { ...ctx.sessionHeaders }; - delete multipartHeaders['Content-Type']; - - const upload = http.post(`${ENDPOINT}/storage/buckets/${bucketId}/files`, { - fileId, - file: http.file(onePixelPng(), 'benchmark.png', 'image/png'), - ...flattenMultipartArray('permissions', ITEM_PERMISSIONS), - }, { - headers: multipartHeaders, - tags: { name: 'storage.files.create' }, - }); - - apiDuration.add(upload.timings.duration, { name: 'storage.files.create' }); - assertStatus(upload, [201], 'storage file created'); - - api('GET', `/storage/buckets/${bucketId}/files`, null, ctx.sessionHeaders, [200], 'storage.files.list'); - api('GET', `/storage/buckets/${bucketId}/files/${fileId}`, null, ctx.sessionHeaders, [200], 'storage.files.get'); - api('GET', `/storage/buckets/${bucketId}/files/${fileId}/view`, null, ctx.sessionHeaders, [200], 'storage.files.view'); - api('GET', `/storage/buckets/${bucketId}/files/${fileId}/download`, null, ctx.sessionHeaders, [200], 'storage.files.download'); - api('GET', `/storage/buckets/${bucketId}/files/${fileId}/preview`, null, ctx.sessionHeaders, [200], 'storage.files.preview'); - api('PUT', `/storage/buckets/${bucketId}/files/${fileId}`, { - name: 'benchmark-renamed.png', - permissions: ITEM_PERMISSIONS, - }, ctx.sessionHeaders, [200], 'storage.files.update'); - - const token = api('POST', `/tokens/buckets/${bucketId}/files/${fileId}`, {}, ctx.apiHeaders, [201], 'tokens.files.create'); - api('GET', `/tokens/buckets/${bucketId}/files/${fileId}`, null, ctx.apiHeaders, [200], 'tokens.files.list'); - api('GET', `/tokens/${token.json('$id')}`, null, ctx.apiHeaders, [200], 'tokens.get'); - api('PATCH', `/tokens/${token.json('$id')}`, { expire: null }, ctx.apiHeaders, [200], 'tokens.update'); - api('DELETE', `/tokens/${token.json('$id')}`, null, ctx.apiHeaders, [204], 'tokens.delete'); - - api('DELETE', `/storage/buckets/${bucketId}/files/${fileId}`, null, ctx.sessionHeaders, [204], 'storage.files.delete'); - api('DELETE', `/storage/buckets/${bucketId}`, null, ctx.apiHeaders, [204], 'storage.buckets.delete'); -} - -function messagingFlow(ctx) { - const providerId = unique('smtp'); - let targetId = unique('target'); - const topicId = unique('topic'); - const subscriberId = unique('sub'); - const messageId = unique('msg'); - - api('POST', '/messaging/providers/smtp', { - providerId, - name: 'Benchmark SMTP', - host: __ENV.APPWRITE_SMTP_HOST || 'maildev', - port: Number(__ENV.APPWRITE_SMTP_PORT || 1025), - username: __ENV.APPWRITE_SMTP_USERNAME || 'user', - password: __ENV.APPWRITE_SMTP_PASSWORD || 'password', - encryption: __ENV.APPWRITE_SMTP_ENCRYPTION || 'none', - autoTLS: false, - fromName: 'Benchmark', - fromEmail: 'benchmark@appwrite.io', - replyToName: 'Benchmark', - replyToEmail: 'benchmark@appwrite.io', - enabled: true, - }, ctx.apiHeaders, [201], 'messaging.providers.smtp.create'); - - const targets = api('GET', `/users/${ctx.userId}/targets`, null, ctx.apiHeaders, [200], 'users.targets.list'); - const existingTarget = (targets.json('targets') || []).find((target) => { - return target.providerType === 'email' && target.identifier === ctx.userEmail; - }); - - if (existingTarget) { - targetId = existingTarget.$id; - api('PATCH', `/users/${ctx.userId}/targets/${targetId}`, { - providerId, - name: 'Benchmark email target', - }, ctx.apiHeaders, [200], 'users.targets.update'); - } else { - api('POST', `/users/${ctx.userId}/targets`, { - targetId, - providerType: 'email', - identifier: ctx.userEmail, - providerId, - name: 'Benchmark email target', - }, ctx.apiHeaders, [201], 'users.targets.create'); - } - - api('POST', '/messaging/topics', { - topicId, - name: 'Benchmark Topic', - subscribe: ['users'], - }, ctx.apiHeaders, [201], 'messaging.topics.create'); - - api('POST', `/messaging/topics/${topicId}/subscribers`, { - subscriberId, - targetId, - }, ctx.sessionHeaders, [201], 'messaging.subscribers.create'); - - const started = Date.now(); - api('POST', '/messaging/messages/email', { - messageId, - subject: `Benchmark message ${ctx.runId}`, - content: `Benchmark messaging worker probe ${ctx.runId}`, - targets: [targetId], - draft: false, - html: false, - }, ctx.apiHeaders, [201], 'messaging.messages.email.create'); - - waitForMessage(messageId, ctx.apiHeaders, WORKER_TIMEOUT_MS); - waitForEmail(ctx.userEmail, (message) => includes(message.subject, `Benchmark message ${ctx.runId}`), MAIL_TIMEOUT_MS, true); - messagingWorkerDuration.add(Date.now() - started, { job: 'email_message' }); - - api('GET', '/messaging/messages', null, ctx.apiHeaders, [200], 'messaging.messages.list'); - api('GET', `/messaging/messages/${messageId}/logs`, null, ctx.apiHeaders, [200], 'messaging.messages.logs.list'); - api('GET', `/messaging/messages/${messageId}/targets`, null, ctx.apiHeaders, [200], 'messaging.messages.targets.list'); - api('GET', `/messaging/providers/${providerId}/logs`, null, ctx.apiHeaders, [200], 'messaging.providers.logs.list'); - api('GET', `/messaging/topics/${topicId}/logs`, null, ctx.apiHeaders, [200], 'messaging.topics.logs.list'); - api('GET', `/messaging/subscribers/${subscriberId}/logs`, null, ctx.apiHeaders, [200], 'messaging.subscribers.logs.list'); - api('DELETE', `/messaging/topics/${topicId}/subscribers/${subscriberId}`, null, ctx.sessionHeaders, [204], 'messaging.subscribers.delete'); - api('DELETE', `/messaging/topics/${topicId}`, null, ctx.apiHeaders, [204], 'messaging.topics.delete'); - api('DELETE', `/messaging/messages/${messageId}`, null, ctx.apiHeaders, [204], 'messaging.messages.delete'); - api('DELETE', `/messaging/providers/${providerId}`, null, ctx.apiHeaders, [204], 'messaging.providers.delete'); -} - -function computeFlow(ctx) { - const functionId = unique('fn'); - let functionVariableId; - const siteId = unique('site'); - let siteVariableId; - - api('POST', '/functions', { - functionId, - name: 'Benchmark Function', - runtime: __ENV.APPWRITE_BENCHMARK_RUNTIME || 'node-22', - execute: ['any'], - events: [], - schedule: '', - timeout: 15, - enabled: true, - logging: true, - entrypoint: 'index.js', - commands: 'npm install', - scopes: ['users.read'], - }, ctx.apiHeaders, [201], 'functions.create'); - api('GET', '/functions/runtimes', null, ctx.sessionHeaders, [200], 'functions.runtimes.list'); - api('GET', '/functions/specifications', null, ctx.apiHeaders, [200], 'functions.specifications.list'); - const functionVariable = api('POST', `/functions/${functionId}/variables`, { - key: 'BENCHMARK', - value: 'true', - secret: false, - }, ctx.apiHeaders, [201], 'functions.variables.create'); - functionVariableId = functionVariable.json('$id'); - - api('PUT', `/functions/${functionId}/variables/${functionVariableId}`, { - key: 'BENCHMARK', - value: 'updated', - secret: false, - }, ctx.apiHeaders, [200], 'functions.variables.update'); - api('GET', `/functions/${functionId}/variables/${functionVariableId}`, null, ctx.apiHeaders, [200], 'functions.variables.get'); - api('DELETE', `/functions/${functionId}/variables/${functionVariableId}`, null, ctx.apiHeaders, [204], 'functions.variables.delete'); - api('DELETE', `/functions/${functionId}`, null, ctx.apiHeaders, [204], 'functions.delete'); - - api('POST', '/sites', { - siteId, - name: 'Benchmark Site', - framework: 'other', - adapter: 'static', - buildRuntime: __ENV.APPWRITE_BENCHMARK_RUNTIME || 'node-22', - buildCommand: '', - outputDirectory: '.', - installCommand: '', - fallbackFile: 'index.html', - providerRootDirectory: '.', - specification: '', - }, ctx.apiHeaders, [201], 'sites.create'); - api('GET', '/sites/frameworks', null, ctx.sessionHeaders, [200], 'sites.frameworks.list'); - api('GET', '/sites/specifications', null, ctx.apiHeaders, [200], 'sites.specifications.list'); - const siteVariable = api('POST', `/sites/${siteId}/variables`, { - key: 'BENCHMARK', - value: 'true', - secret: false, - }, ctx.apiHeaders, [201], 'sites.variables.create'); - siteVariableId = siteVariable.json('$id'); - - api('PUT', `/sites/${siteId}/variables/${siteVariableId}`, { - key: 'BENCHMARK', - value: 'updated', - secret: false, - }, ctx.apiHeaders, [200], 'sites.variables.update'); - api('GET', `/sites/${siteId}/variables/${siteVariableId}`, null, ctx.apiHeaders, [200], 'sites.variables.get'); - api('DELETE', `/sites/${siteId}/variables/${siteVariableId}`, null, ctx.apiHeaders, [204], 'sites.variables.delete'); - api('DELETE', `/sites/${siteId}`, null, ctx.apiHeaders, [204], 'sites.delete'); -} - -function healthFlow(ctx) { - const probes = [ - '/health', - '/health/db', - '/health/cache', - '/health/pubsub', - '/health/storage', - '/health/storage/local', - '/health/time', - '/health/queue/databases', - '/health/queue/mails', - '/health/queue/messaging', - '/health/queue/functions', - '/health/queue/builds', - '/health/queue/deletes', - '/health/queue/webhooks', - '/health/queue/stats-resources', - '/health/queue/stats-usage', - '/health/queue/failed/v1-mails', - ]; - - for (const path of probes) { - api('GET', path, null, ctx.apiHeaders, [200], `health${path.replace(/\//g, '.')}`); - } -} - -function api(method, path, body, headers, expected, name) { - const response = rawRequest(method, path, body, headers, name); - apiDuration.add(response.timings.duration, { name }); - assertStatus(response, expected, name); - return response; -} - -function rawRequest(method, path, body, headers, name) { - const params = { - headers, - tags: { name }, - }; - const payload = body === null || body === undefined ? null : JSON.stringify(body); - return http.request(method, `${ENDPOINT}${path}`, payload, params); -} - -function waitForStatus(path, headers, wantedStatus, timeoutMs) { - const started = Date.now(); - - while (Date.now() - started < timeoutMs) { - const response = rawRequest('GET', path, null, headers, `wait${path}`); - if (response.status === 200 && response.json('status') === wantedStatus) { - return response; - } - sleep(0.5); - } - - throw new Error(`Timed out waiting for ${path} to become ${wantedStatus}`); -} - -function waitForMessage(messageId, headers, timeoutMs) { - const started = Date.now(); - - while (Date.now() - started < timeoutMs) { - const response = rawRequest('GET', `/messaging/messages/${messageId}`, null, headers, 'messaging.messages.poll'); - const status = response.status === 200 ? response.json('status') : null; - - if (['sent', 'failed'].includes(status)) { - if (status === 'failed') { - throw new Error(`Messaging worker marked message ${messageId} as failed`); - } - return response; - } - - sleep(0.5); - } - - throw new Error(`Timed out waiting for messaging worker to send message ${messageId}`); -} - -function waitForEmail(address, predicate, timeoutMs, allowMissingRecipient = false) { - const started = Date.now(); - - while (Date.now() - started < timeoutMs) { - const response = http.get(MAILDEV_ENDPOINT, { tags: { name: 'maildev.email.list' } }); - if (response.status === 200) { - const emails = response.json(); - for (let i = emails.length - 1; i >= 0; i--) { - const message = emails[i]; - if ((emailMatches(message, address) || (allowMissingRecipient && emailRecipientMissing(message))) && predicate(message)) { - return message; - } - } - } - sleep(0.5); - } - - throw new Error(`Timed out waiting for email to ${address}`); -} - -function emailMatches(message, address) { - const recipients = message.to || []; - return recipients.some((recipient) => recipient.address === address); -} - -function emailRecipientMissing(message) { - const recipients = message.to || []; - return recipients.length === 0 || recipients.every((recipient) => !recipient.address); -} - -function extractQueryParams(message) { - const content = `${message.html || ''}\n${message.text || ''}`; - const links = []; - const hrefPattern = /href="([^"]+)"/g; - let hrefMatch = hrefPattern.exec(content); - - while (hrefMatch !== null) { - links.push(hrefMatch[1]); - hrefMatch = hrefPattern.exec(content); - } - - if (links.length === 0) { - links.push(content); - } - - for (const link of links) { - const queryStart = link.indexOf('?'); - if (queryStart === -1) { - continue; - } - - const query = link.slice(queryStart + 1).split('#')[0].replace(/&/g, '&'); - const params = {}; - - for (const pair of query.split('&')) { - const [key, value] = pair.split('='); - params[decodeURIComponent(key)] = decodeURIComponent(value || ''); - } - - if (params.userId && params.secret) { - return params; - } - } - - return {}; -} - -function assertStatus(response, expected, name) { - const ok = check(response, { - [`${name} status ${expected.join('|')}`]: (r) => expected.includes(r.status), - }); - - if (!ok) { - failResponse(response, `${name} returned an unexpected status`); - } -} - -function failResponse(response, message) { - throw new Error(`${message}. Status: ${response.status}. Body: ${response.body}`); -} - -function cookieHeader(response) { - return response.headers['Set-Cookie'] || response.headers['set-cookie'] || ''; -} - -function projectHeaders(projectId) { - return { - 'Content-Type': 'application/json', - 'X-Appwrite-Project': projectId, - }; -} - -function documentPayload() { - return { - title: 'Benchmark Document', - count: 1, - email: 'document@example.com', - active: true, - publishedAt: new Date().toISOString(), - score: 10.5, - url: 'https://appwrite.io', - ip: '127.0.0.1', - }; -} - -function tablePayload() { - return { - title: 'Benchmark Row', - count: 1, - email: 'row@example.com', - active: true, - }; -} - -function onePixelPng() { - return encoding.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=', 'std', 'b'); -} - -function flattenMultipartArray(key, values) { - const output = {}; - values.forEach((value, index) => { - output[`${key}[${index}]`] = value; - }); - return output; -} - -function unique(prefix) { - return `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}` - .toLowerCase() - .replace(/[^a-z0-9-]/g, '-') - .slice(0, 36); -} - -function includes(value, needle) { - return String(value || '').toLowerCase().includes(String(needle).toLowerCase()); -} - -function hostnameFromUrl(value) { - return value.replace(/^https?:\/\//, '').split('/')[0].split(':')[0]; -} - -export function handleSummary(data) { - const lines = [ - 'Appwrite curated benchmark review', - '', - 'Before/after comparison', - '', - comparisonTable(PREVIOUS_SUMMARY, data), - '', - 'Current run details', - '', - metricLine(data, 'http_req_duration', 'HTTP total'), - metricLine(data, 'appwrite_api_duration', 'API endpoints'), - metricLine(data, 'appwrite_worker_database_duration', 'Database worker schema jobs'), - metricLine(data, 'appwrite_worker_tables_duration', 'TablesDB worker schema jobs'), - metricLine(data, 'appwrite_worker_mails_duration', 'Mail worker delivery'), - metricLine(data, 'appwrite_worker_messaging_duration', 'Messaging worker delivery'), - counterLine(data, 'appwrite_benchmark_flow_failures', 'Flow failures'), - '', - `Endpoint: ${ENDPOINT}`, - `Maildev API: ${MAILDEV_ENDPOINT}`, - '', - ]; - - return { - stdout: `${lines.filter(Boolean).join('\n')}\n`, - [SUMMARY_PATH]: JSON.stringify(data, null, 2), - }; -} - -function loadPreviousSummary() { - try { - return JSON.parse(open(PREVIOUS_SUMMARY_PATH)); - } catch (error) { - return null; - } -} - -function comparisonTable(before, after) { - const rows = [ - ['HTTP total p95', trendMetric(before, 'http_req_duration', 'p(95)'), trendMetric(after, 'http_req_duration', 'p(95)'), 'ms'], - ['API endpoints p95', trendMetric(before, 'appwrite_api_duration', 'p(95)'), trendMetric(after, 'appwrite_api_duration', 'p(95)'), 'ms'], - ['Database worker p95', trendMetric(before, 'appwrite_worker_database_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_database_duration', 'p(95)'), 'ms'], - ['TablesDB worker p95', trendMetric(before, 'appwrite_worker_tables_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_tables_duration', 'p(95)'), 'ms'], - ['Mail worker p95', trendMetric(before, 'appwrite_worker_mails_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_mails_duration', 'p(95)'), 'ms'], - ['Messaging worker p95', trendMetric(before, 'appwrite_worker_messaging_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_messaging_duration', 'p(95)'), 'ms'], - ['Flow failures', counterMetric(before, 'appwrite_benchmark_flow_failures'), counterMetric(after, 'appwrite_benchmark_flow_failures'), ''], - ['Check failures', checkFailures(before), checkFailures(after), ''], - ]; - - return [ - '| Metric | Before | After | Delta |', - '| --- | ---: | ---: | ---: |', - ...rows.map(([label, beforeValue, afterValue, unit]) => { - return `| ${label} | ${formatValue(beforeValue, unit)} | ${formatValue(afterValue, unit)} | ${formatDelta(beforeValue, afterValue, unit)} |`; - }), - ].join('\n'); -} - -function trendMetric(data, metric, stat) { - return data && data.metrics[metric] && data.metrics[metric].values - ? data.metrics[metric].values[stat] - : null; -} - -function counterMetric(data, metric) { - return data && data.metrics[metric] && data.metrics[metric].values - ? data.metrics[metric].values.count - : null; -} - -function checkFailures(data) { - return data && data.metrics.checks && data.metrics.checks.values - ? data.metrics.checks.values.fails - : null; -} - -function formatValue(value, unit) { - if (value === null || value === undefined || Number.isNaN(value)) { - return 'n/a'; - } - - return `${round(value)}${unit}`; -} - -function formatDelta(before, after, unit) { - if (before === null || before === undefined || after === null || after === undefined || Number.isNaN(before) || Number.isNaN(after)) { - return 'n/a'; - } - - const delta = round(after - before); - const sign = delta > 0 ? '+' : ''; - return `${sign}${delta}${unit}`; -} - -function metricLine(data, metric, label) { - const values = data.metrics[metric] && data.metrics[metric].values; - if (!values || values.count === 0) { - return `${label}: no samples`; - } - - return `${label}: avg=${round(values.avg)}ms p90=${round(values['p(90)'])}ms p95=${round(values['p(95)'])}ms max=${round(values.max)}ms`; -} - -function counterLine(data, metric, label) { - const values = data.metrics[metric] && data.metrics[metric].values; - return `${label}: ${values ? values.count : 0}`; -} - -function round(value) { - return Math.round((value || 0) * 100) / 100; -} diff --git a/tests/benchmarks/http.php b/tests/benchmarks/http.php new file mode 100644 index 0000000000..1d91246352 --- /dev/null +++ b/tests/benchmarks/http.php @@ -0,0 +1,1299 @@ +jsonParsed) { + $this->json = json_decode($this->body, true); + $this->jsonParsed = true; + } + + if ($key === null) { + return $this->json; + } + + return is_array($this->json) ? ($this->json[$key] ?? null) : null; + } + + public function header(string $name): string + { + $key = strtolower($name); + return isset($this->headers[$key]) ? implode(', ', $this->headers[$key]) : ''; + } + + public function cookieHeader(): string + { + $cookies = []; + + foreach ($this->headers['set-cookie'] ?? [] as $cookie) { + $cookies[] = explode(';', $cookie, 2)[0]; + } + + return implode('; ', $cookies); + } +} + +final class BenchmarkMetrics +{ + private array $trends = []; + private array $counters = [ + 'appwrite_benchmark_flow_failures' => 0, + ]; + private int $checksPassed = 0; + private int $checksFailed = 0; + + public function addTrend(string $name, float $value): void + { + $this->trends[$name] ??= []; + $this->trends[$name][] = $value; + } + + public function addCounter(string $name, int $value = 1): void + { + $this->counters[$name] ??= 0; + $this->counters[$name] += $value; + } + + public function addCheck(bool $passed): void + { + if ($passed) { + $this->checksPassed++; + return; + } + + $this->checksFailed++; + } + + public function summary(): array + { + $metrics = []; + + foreach ($this->trends as $name => $values) { + $metrics[$name] = [ + 'type' => 'trend', + 'contains' => 'time', + 'values' => $this->trendValues($values), + ]; + } + + foreach ($this->counters as $name => $count) { + $metrics[$name] = [ + 'type' => 'counter', + 'contains' => 'default', + 'values' => [ + 'count' => $count, + ], + ]; + } + + $totalChecks = $this->checksPassed + $this->checksFailed; + $metrics['checks'] = [ + 'type' => 'rate', + 'contains' => 'default', + 'values' => [ + 'rate' => $totalChecks > 0 ? $this->checksPassed / $totalChecks : 1, + 'passes' => $this->checksPassed, + 'fails' => $this->checksFailed, + ], + ]; + + return ['metrics' => $metrics]; + } + + public function failedChecks(): int + { + return $this->checksFailed; + } + + public function flowFailures(): int + { + return $this->counters['appwrite_benchmark_flow_failures'] ?? 0; + } + + private function trendValues(array $values): array + { + sort($values, SORT_NUMERIC); + $count = count($values); + + if ($count === 0) { + return [ + 'count' => 0, + 'min' => null, + 'avg' => null, + 'med' => null, + 'max' => null, + 'p(90)' => null, + 'p(95)' => null, + ]; + } + + return [ + 'count' => $count, + 'min' => $values[0], + 'avg' => array_sum($values) / $count, + 'med' => $this->percentile($values, 50), + 'max' => $values[$count - 1], + 'p(90)' => $this->percentile($values, 90), + 'p(95)' => $this->percentile($values, 95), + ]; + } + + private function percentile(array $sortedValues, int $percentile): float + { + $count = count($sortedValues); + + if ($count === 1) { + return (float) $sortedValues[0]; + } + + $rank = ($percentile / 100) * ($count - 1); + $lower = (int) floor($rank); + $upper = (int) ceil($rank); + + if ($lower === $upper) { + return (float) $sortedValues[$lower]; + } + + $weight = $rank - $lower; + return (float) ($sortedValues[$lower] + (($sortedValues[$upper] - $sortedValues[$lower]) * $weight)); + } +} + +final class HttpBenchmark +{ + private const API_SCOPES = [ + 'sessions.write', + 'users.read', + 'users.write', + 'teams.read', + 'teams.write', + 'databases.read', + 'databases.write', + 'collections.read', + 'collections.write', + 'tables.read', + 'tables.write', + 'attributes.read', + 'attributes.write', + 'columns.read', + 'columns.write', + 'indexes.read', + 'indexes.write', + 'documents.read', + 'documents.write', + 'rows.read', + 'rows.write', + 'files.read', + 'files.write', + 'buckets.read', + 'buckets.write', + 'functions.read', + 'functions.write', + 'sites.read', + 'sites.write', + 'log.read', + 'log.write', + 'execution.read', + 'execution.write', + 'locale.read', + 'avatars.read', + 'health.read', + 'providers.read', + 'providers.write', + 'messages.read', + 'messages.write', + 'topics.read', + 'topics.write', + 'subscribers.read', + 'subscribers.write', + 'targets.read', + 'targets.write', + 'rules.read', + 'rules.write', + 'migrations.read', + 'migrations.write', + 'vcs.read', + 'vcs.write', + 'assistant.read', + 'tokens.read', + 'tokens.write', + 'platforms.read', + 'platforms.write', + ]; + + private const BASE_PERMISSIONS = [ + 'read("any")', + 'create("any")', + 'update("any")', + 'delete("any")', + ]; + + private const ITEM_PERMISSIONS = [ + 'read("any")', + 'update("any")', + 'delete("any")', + ]; + + private const PNG_1X1 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII='; + + private BenchmarkMetrics $metrics; + private string $endpoint; + private string $maildevEndpoint; + private string $consoleProject; + private string $region; + private string $redirectUrl; + private string $password; + private int $mailTimeoutMs; + private int $workerTimeoutMs; + private int $iterations; + private int $vus; + private string $summaryPath; + private ?array $previousSummary; + + public function __construct() + { + $this->metrics = new BenchmarkMetrics(); + $this->endpoint = rtrim($this->env('APPWRITE_ENDPOINT', 'http://localhost/v1'), '/'); + $this->maildevEndpoint = $this->env('APPWRITE_MAILDEV_ENDPOINT', 'http://localhost:9503/email'); + $this->consoleProject = $this->env('APPWRITE_CONSOLE_PROJECT', 'console'); + $this->region = $this->env('APPWRITE_REGION', 'default'); + $this->redirectUrl = $this->env('APPWRITE_BENCHMARK_REDIRECT_URL', 'http://localhost'); + $this->password = $this->env('APPWRITE_BENCHMARK_PASSWORD', 'Password123!'); + $this->mailTimeoutMs = (int) $this->env('APPWRITE_MAIL_TIMEOUT_MS', '20000'); + $this->workerTimeoutMs = (int) $this->env('APPWRITE_WORKER_TIMEOUT_MS', '60000'); + $this->iterations = max(1, (int) $this->env('APPWRITE_BENCHMARK_ITERATIONS', '1')); + $this->vus = max(1, (int) $this->env('APPWRITE_BENCHMARK_VUS', '1')); + $this->summaryPath = $this->env('APPWRITE_BENCHMARK_SUMMARY_PATH', 'tests/benchmarks/http-summary.json'); + $this->previousSummary = $this->loadPreviousSummary($this->env('APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH', $this->summaryPath)); + } + + public function run(): int + { + $context = null; + $exitCode = 0; + + try { + $context = $this->setup(); + + for ($i = 0; $i < $this->iterations * $this->vus; $i++) { + $this->curatedFlows($context); + } + } catch (Throwable $error) { + $exitCode = 1; + fwrite(STDERR, $error->getMessage() . PHP_EOL); + } finally { + if (is_array($context)) { + $this->teardown($context); + } + + $summary = $this->metrics->summary(); + $this->writeSummary($summary); + echo $this->renderSummary($summary); + } + + if ($this->metrics->failedChecks() > 0 || $this->metrics->flowFailures() > 0) { + $exitCode = 1; + } + + return $exitCode; + } + + private function setup(): array + { + $runId = $this->unique('run'); + $consoleEmail = $this->env('APPWRITE_ADMIN_EMAIL', "bench-admin-{$runId}@example.com"); + $consolePassword = $this->env('APPWRITE_ADMIN_PASSWORD', $this->password); + $consoleHeaders = [ + 'Content-Type' => 'application/json', + 'X-Appwrite-Project' => $this->consoleProject, + ]; + + $account = $this->rawRequest('POST', '/account', [ + 'userId' => $this->unique('admin'), + 'email' => $consoleEmail, + 'password' => $consolePassword, + 'name' => 'Benchmark Admin', + ], $consoleHeaders, 'setup.account.create'); + + if (!in_array($account->status, [201, 409], true)) { + $this->failResponse($account, 'Unable to create or reuse the benchmark console account'); + } + + $session = $this->rawRequest('POST', '/account/sessions/email', [ + 'email' => $consoleEmail, + 'password' => $consolePassword, + ], $consoleHeaders, 'setup.account.session'); + $this->assertStatus($session, [201], 'console session created'); + + $consoleSessionHeaders = [ + ...$consoleHeaders, + 'Cookie' => $session->cookieHeader(), + ]; + + $team = $this->api('POST', '/teams', [ + 'teamId' => $this->unique('team'), + 'name' => "Benchmark Team {$runId}", + ], $consoleSessionHeaders, [201], 'setup.teams.create'); + + $teamId = (string) $team->json('$id'); + $project = $this->api('POST', '/projects', [ + 'projectId' => $this->unique('project'), + 'name' => "Benchmark Project {$runId}", + 'teamId' => $teamId, + 'region' => $this->region, + ], $consoleSessionHeaders, [201], 'setup.projects.create'); + + $projectId = (string) $project->json('$id'); + $key = $this->api('POST', "/projects/{$projectId}/keys", [ + 'keyId' => $this->unique('key'), + 'name' => 'Benchmark API key', + 'scopes' => self::API_SCOPES, + ], $consoleSessionHeaders, [201], 'setup.projects.keys.create'); + + $apiHeaders = [ + 'Content-Type' => 'application/json', + 'X-Appwrite-Project' => $projectId, + 'X-Appwrite-Key' => (string) $key->json('secret'), + ]; + + $platform = $this->api('POST', '/project/platforms/web', [ + 'platformId' => $this->unique('web'), + 'name' => 'Benchmark web', + 'hostname' => $this->hostnameFromUrl($this->redirectUrl), + ], $apiHeaders, [201, 409], 'setup.project.platforms.web.create'); + + $smtpBody = [ + 'enabled' => true, + 'senderName' => 'Benchmark', + 'senderEmail' => 'benchmark@appwrite.io', + 'replyTo' => 'benchmark@appwrite.io', + 'host' => $this->env('APPWRITE_SMTP_HOST', 'maildev'), + 'port' => (int) $this->env('APPWRITE_SMTP_PORT', '1025'), + 'username' => $this->env('APPWRITE_SMTP_USERNAME', 'user'), + 'password' => $this->env('APPWRITE_SMTP_PASSWORD', 'password'), + ]; + + if ($this->env('APPWRITE_SMTP_SECURE', '') !== '') { + $smtpBody['secure'] = $this->env('APPWRITE_SMTP_SECURE', ''); + } + + $smtp = $this->rawRequest('PATCH', "/projects/{$projectId}/smtp", $smtpBody, $consoleSessionHeaders, 'setup.projects.smtp.update'); + if ($smtp->status !== 200) { + fwrite(STDERR, "Custom SMTP was not enabled ({$smtp->status}). Mail worker timings may be unavailable." . PHP_EOL); + } + + return [ + 'runId' => $runId, + 'teamId' => $teamId, + 'projectId' => $projectId, + 'consoleSessionHeaders' => $consoleSessionHeaders, + 'apiHeaders' => $apiHeaders, + 'platformStatus' => $platform->status, + ]; + } + + private function curatedFlows(array &$context): void + { + try { + $this->accountFlow($context); + $this->databasesFlow($context); + $this->tablesDbFlow($context); + $this->storageFlow($context); + $this->messagingFlow($context); + $this->computeFlow($context); + $this->healthFlow($context); + } catch (Throwable $error) { + $this->metrics->addCounter('appwrite_benchmark_flow_failures'); + throw $error; + } + } + + private function teardown(array $context): void + { + if (($context['projectId'] ?? null) && ($context['consoleSessionHeaders'] ?? null)) { + $this->rawRequest('DELETE', "/projects/{$context['projectId']}", null, $context['consoleSessionHeaders'], 'teardown.projects.delete'); + } + + if (($context['teamId'] ?? null) && ($context['consoleSessionHeaders'] ?? null)) { + $this->rawRequest('DELETE', "/teams/{$context['teamId']}", null, $context['consoleSessionHeaders'], 'teardown.teams.delete'); + } + } + + private function accountFlow(array &$context): void + { + $userId = $this->unique('user'); + $email = 'bench-user-' . $this->unique('mail') . '@example.com'; + $headers = $this->projectHeaders($context['projectId']); + + $this->api('POST', '/account', [ + 'userId' => $userId, + 'email' => $email, + 'password' => $this->password, + 'name' => 'Benchmark User', + ], $headers, [201], 'account.create'); + + $session = $this->api('POST', '/account/sessions/email', [ + 'email' => $email, + 'password' => $this->password, + ], $headers, [201], 'account.sessions.email.create'); + + $sessionHeaders = [ + ...$headers, + 'Cookie' => $session->cookieHeader(), + ]; + + $context['userId'] = $userId; + $context['userEmail'] = $email; + $context['sessionHeaders'] = $sessionHeaders; + + $jwt = $this->api('POST', '/account/jwts', null, $sessionHeaders, [201], 'account.jwts.create'); + $context['jwtHeaders'] = [ + ...$headers, + 'X-Appwrite-JWT' => (string) $jwt->json('jwt'), + ]; + + $this->api('GET', '/account', null, $sessionHeaders, [200], 'account.get'); + $this->api('GET', '/account/logs', null, $sessionHeaders, [200], 'account.logs.list'); + $this->api('PATCH', '/account/prefs', ['prefs' => ['benchmark' => true, 'runId' => $context['runId']]], $sessionHeaders, [200], 'account.prefs.update'); + $this->api('PATCH', '/account/name', ['name' => 'Benchmark User Updated'], $sessionHeaders, [200], 'account.name.update'); + $this->api('PATCH', '/account/password', ['password' => $this->password . '2', 'oldPassword' => $this->password], $sessionHeaders, [200], 'account.password.update'); + + $verificationStarted = $this->nowMs(); + $this->api('POST', '/account/verifications/email', ['url' => $this->redirectUrl], $sessionHeaders, [201], 'account.emailVerification.create'); + $verificationEmail = $this->waitForEmail($email, fn (array $message): bool => $this->messageIncludes($message, ['verify', 'verification']), $this->mailTimeoutMs); + $this->metrics->addTrend('appwrite_worker_mails_duration', $this->nowMs() - $verificationStarted); + + $verification = $this->extractQueryParams($verificationEmail); + if (($verification['userId'] ?? null) && ($verification['secret'] ?? null)) { + $this->api('PUT', '/account/verifications/email', [ + 'userId' => $verification['userId'], + 'secret' => $verification['secret'], + ], $sessionHeaders, [200], 'account.emailVerification.update'); + } + + $recoveryStarted = $this->nowMs(); + $this->api('POST', '/account/recovery', ['email' => $email, 'url' => $this->redirectUrl], $headers, [201], 'account.recovery.create'); + $recoveryEmail = $this->waitForEmail($email, fn (array $message): bool => $this->messageIncludes($message, ['recovery', 'recover', 'reset']), $this->mailTimeoutMs); + $this->metrics->addTrend('appwrite_worker_mails_duration', $this->nowMs() - $recoveryStarted); + + $recovery = $this->extractQueryParams($recoveryEmail); + if (($recovery['userId'] ?? null) && ($recovery['secret'] ?? null)) { + $this->api('DELETE', '/account/sessions/current', null, $sessionHeaders, [204], 'account.sessions.current.delete'); + $this->api('PUT', '/account/recovery', [ + 'userId' => $recovery['userId'], + 'secret' => $recovery['secret'], + 'password' => $this->password . '3', + ], $headers, [200], 'account.recovery.update'); + + $recoveredSession = $this->api('POST', '/account/sessions/email', [ + 'email' => $email, + 'password' => $this->password . '3', + ], $headers, [201], 'account.sessions.email.recovered'); + + $context['sessionHeaders'] = [ + ...$headers, + 'Cookie' => $recoveredSession->cookieHeader(), + ]; + + $recoveredJwt = $this->api('POST', '/account/jwts', null, $context['sessionHeaders'], [201], 'account.jwts.recovered'); + $context['jwtHeaders'] = [ + ...$headers, + 'X-Appwrite-JWT' => (string) $recoveredJwt->json('jwt'), + ]; + } + } + + private function databasesFlow(array $context): void + { + $databaseId = $this->unique('db'); + $collectionId = $this->unique('col'); + $documentId = $this->unique('doc'); + $indexKey = $this->unique('idx'); + + $this->api('POST', '/databases', ['databaseId' => $databaseId, 'name' => 'Benchmark DB'], $context['apiHeaders'], [201], 'databases.create'); + $this->api('POST', "/databases/{$databaseId}/collections", [ + 'collectionId' => $collectionId, + 'name' => 'Benchmark Collection', + 'permissions' => self::BASE_PERMISSIONS, + 'documentSecurity' => false, + ], $context['apiHeaders'], [201], 'databases.collections.create'); + + $attributes = [ + ['string', 'title', ['size' => 128]], + ['integer', 'count', ['min' => 0, 'max' => 100000]], + ['email', 'email', []], + ['boolean', 'active', []], + ['datetime', 'publishedAt', []], + ['float', 'score', ['min' => 0, 'max' => 1000]], + ['url', 'url', []], + ['ip', 'ip', []], + ]; + + foreach ($attributes as [$type, $key, $extra]) { + $started = $this->nowMs(); + $this->api('POST', "/databases/{$databaseId}/collections/{$collectionId}/attributes/{$type}", [ + 'key' => $key, + 'required' => false, + 'array' => false, + ...$extra, + ], $context['apiHeaders'], [202], "databases.attributes.{$type}.create"); + $this->waitForStatus("/databases/{$databaseId}/collections/{$collectionId}/attributes/{$key}", $context['apiHeaders'], 'available', $this->workerTimeoutMs); + $this->metrics->addTrend('appwrite_worker_database_duration', $this->nowMs() - $started); + } + + $indexStarted = $this->nowMs(); + $this->api('POST', "/databases/{$databaseId}/collections/{$collectionId}/indexes", [ + 'key' => $indexKey, + 'type' => 'key', + 'attributes' => ['title'], + 'orders' => ['asc'], + ], $context['apiHeaders'], [202], 'databases.indexes.create'); + $this->waitForStatus("/databases/{$databaseId}/collections/{$collectionId}/indexes/{$indexKey}", $context['apiHeaders'], 'available', $this->workerTimeoutMs); + $this->metrics->addTrend('appwrite_worker_database_duration', $this->nowMs() - $indexStarted); + + $this->api('POST', "/databases/{$databaseId}/collections/{$collectionId}/documents", [ + 'documentId' => $documentId, + 'data' => $this->documentPayload(), + 'permissions' => self::ITEM_PERMISSIONS, + ], $context['apiHeaders'], [201], 'databases.documents.create'); + $this->api('GET', "/databases/{$databaseId}/collections/{$collectionId}/documents", null, $context['apiHeaders'], [200], 'databases.documents.list'); + $this->api('GET', "/databases/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", null, $context['apiHeaders'], [200], 'databases.documents.get'); + $this->api('PATCH', "/databases/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", ['data' => ['title' => 'Benchmark Document Updated']], $context['apiHeaders'], [200], 'databases.documents.update'); + $this->api('PATCH', "/databases/{$databaseId}/collections/{$collectionId}/documents/{$documentId}/count/increment", ['value' => 1], $context['apiHeaders'], [200], 'databases.documents.increment'); + $this->api('PATCH', "/databases/{$databaseId}/collections/{$collectionId}/documents/{$documentId}/count/decrement", ['value' => 1], $context['apiHeaders'], [200], 'databases.documents.decrement'); + $this->api('DELETE', "/databases/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", null, $context['apiHeaders'], [204], 'databases.documents.delete'); + $this->api('DELETE', "/databases/{$databaseId}", null, $context['apiHeaders'], [204], 'databases.delete'); + } + + private function tablesDbFlow(array $context): void + { + $databaseId = $this->unique('tdb'); + $tableId = $this->unique('tbl'); + $rowId = $this->unique('row'); + $indexKey = $this->unique('tidx'); + + $this->api('POST', '/tablesdb', ['databaseId' => $databaseId, 'name' => 'Benchmark TablesDB'], $context['apiHeaders'], [201], 'tablesdb.create'); + $this->api('POST', "/tablesdb/{$databaseId}/tables", [ + 'tableId' => $tableId, + 'name' => 'Benchmark Table', + 'permissions' => self::BASE_PERMISSIONS, + 'rowSecurity' => false, + ], $context['apiHeaders'], [201], 'tablesdb.tables.create'); + + $columns = [ + ['string', 'title', ['size' => 128]], + ['integer', 'count', ['min' => 0, 'max' => 100000]], + ['email', 'email', []], + ['boolean', 'active', []], + ]; + + foreach ($columns as [$type, $key, $extra]) { + $started = $this->nowMs(); + $this->api('POST', "/tablesdb/{$databaseId}/tables/{$tableId}/columns/{$type}", [ + 'key' => $key, + 'required' => false, + 'array' => false, + ...$extra, + ], $context['apiHeaders'], [202], "tablesdb.columns.{$type}.create"); + $this->waitForStatus("/tablesdb/{$databaseId}/tables/{$tableId}/columns/{$key}", $context['apiHeaders'], 'available', $this->workerTimeoutMs); + $this->metrics->addTrend('appwrite_worker_tables_duration', $this->nowMs() - $started); + } + + $indexStarted = $this->nowMs(); + $this->api('POST', "/tablesdb/{$databaseId}/tables/{$tableId}/indexes", [ + 'key' => $indexKey, + 'type' => 'key', + 'columns' => ['title'], + 'orders' => ['asc'], + ], $context['apiHeaders'], [202], 'tablesdb.indexes.create'); + $this->waitForStatus("/tablesdb/{$databaseId}/tables/{$tableId}/indexes/{$indexKey}", $context['apiHeaders'], 'available', $this->workerTimeoutMs); + $this->metrics->addTrend('appwrite_worker_tables_duration', $this->nowMs() - $indexStarted); + + $this->api('POST', "/tablesdb/{$databaseId}/tables/{$tableId}/rows", [ + 'rowId' => $rowId, + 'data' => $this->tablePayload(), + 'permissions' => self::ITEM_PERMISSIONS, + ], $context['sessionHeaders'], [201], 'tablesdb.rows.create'); + $this->api('GET', "/tablesdb/{$databaseId}/tables/{$tableId}/rows", null, $context['sessionHeaders'], [200], 'tablesdb.rows.list'); + $this->api('GET', "/tablesdb/{$databaseId}/tables/{$tableId}/rows/{$rowId}", null, $context['sessionHeaders'], [200], 'tablesdb.rows.get'); + $this->api('PATCH', "/tablesdb/{$databaseId}/tables/{$tableId}/rows/{$rowId}", ['data' => ['title' => 'Benchmark Row Updated']], $context['sessionHeaders'], [200], 'tablesdb.rows.update'); + $this->api('PATCH', "/tablesdb/{$databaseId}/tables/{$tableId}/rows/{$rowId}/count/increment", ['value' => 1], $context['sessionHeaders'], [200], 'tablesdb.rows.increment'); + $this->api('PATCH', "/tablesdb/{$databaseId}/tables/{$tableId}/rows/{$rowId}/count/decrement", ['value' => 1], $context['sessionHeaders'], [200], 'tablesdb.rows.decrement'); + $this->api('DELETE', "/tablesdb/{$databaseId}/tables/{$tableId}/rows/{$rowId}", null, $context['sessionHeaders'], [204], 'tablesdb.rows.delete'); + $this->api('DELETE', "/tablesdb/{$databaseId}", null, $context['apiHeaders'], [204], 'tablesdb.delete'); + } + + private function storageFlow(array $context): void + { + $bucketId = $this->unique('bucket'); + $fileId = $this->unique('file'); + + $this->api('POST', '/storage/buckets', [ + 'bucketId' => $bucketId, + 'name' => 'Benchmark Bucket', + 'permissions' => self::BASE_PERMISSIONS, + 'fileSecurity' => false, + 'enabled' => true, + 'maximumFileSize' => 30000000, + 'allowedFileExtensions' => [], + 'compression' => 'none', + 'encryption' => false, + 'antivirus' => false, + ], $context['apiHeaders'], [201], 'storage.buckets.create'); + + $tmpFile = tempnam(sys_get_temp_dir(), 'appwrite-benchmark-'); + if ($tmpFile === false) { + throw new RuntimeException('Unable to create temporary PNG fixture'); + } + + file_put_contents($tmpFile, base64_decode(self::PNG_1X1, true)); + + try { + $fields = [ + 'fileId' => $fileId, + 'file' => new CURLFile($tmpFile, 'image/png', 'benchmark.png'), + ...$this->flattenMultipartArray('permissions', self::ITEM_PERMISSIONS), + ]; + $multipartHeaders = $context['sessionHeaders']; + unset($multipartHeaders['Content-Type']); + + $upload = $this->rawMultipartRequest('POST', "/storage/buckets/{$bucketId}/files", $fields, $multipartHeaders, 'storage.files.create'); + $this->metrics->addTrend('appwrite_api_duration', $upload->duration); + $this->assertStatus($upload, [201], 'storage file created'); + } finally { + @unlink($tmpFile); + } + + $this->api('GET', "/storage/buckets/{$bucketId}/files", null, $context['sessionHeaders'], [200], 'storage.files.list'); + $this->api('GET', "/storage/buckets/{$bucketId}/files/{$fileId}", null, $context['sessionHeaders'], [200], 'storage.files.get'); + $this->api('GET', "/storage/buckets/{$bucketId}/files/{$fileId}/view", null, $context['sessionHeaders'], [200], 'storage.files.view'); + $this->api('GET', "/storage/buckets/{$bucketId}/files/{$fileId}/download", null, $context['sessionHeaders'], [200], 'storage.files.download'); + $this->api('GET', "/storage/buckets/{$bucketId}/files/{$fileId}/preview", null, $context['sessionHeaders'], [200], 'storage.files.preview'); + $this->api('PUT', "/storage/buckets/{$bucketId}/files/{$fileId}", [ + 'name' => 'benchmark-renamed.png', + 'permissions' => self::ITEM_PERMISSIONS, + ], $context['sessionHeaders'], [200], 'storage.files.update'); + + $token = $this->api('POST', "/tokens/buckets/{$bucketId}/files/{$fileId}", (object) [], $context['apiHeaders'], [201], 'tokens.files.create'); + $tokenId = (string) $token->json('$id'); + $this->api('GET', "/tokens/buckets/{$bucketId}/files/{$fileId}", null, $context['apiHeaders'], [200], 'tokens.files.list'); + $this->api('GET', "/tokens/{$tokenId}", null, $context['apiHeaders'], [200], 'tokens.get'); + $this->api('PATCH', "/tokens/{$tokenId}", ['expire' => null], $context['apiHeaders'], [200], 'tokens.update'); + $this->api('DELETE', "/tokens/{$tokenId}", null, $context['apiHeaders'], [204], 'tokens.delete'); + + $this->api('DELETE', "/storage/buckets/{$bucketId}/files/{$fileId}", null, $context['sessionHeaders'], [204], 'storage.files.delete'); + $this->api('DELETE', "/storage/buckets/{$bucketId}", null, $context['apiHeaders'], [204], 'storage.buckets.delete'); + } + + private function messagingFlow(array $context): void + { + $providerId = $this->unique('smtp'); + $targetId = $this->unique('target'); + $existingTarget = false; + $topicId = $this->unique('topic'); + $subscriberId = $this->unique('sub'); + $messageId = $this->unique('msg'); + + $this->api('POST', '/messaging/providers/smtp', [ + 'providerId' => $providerId, + 'name' => 'Benchmark SMTP', + 'host' => $this->env('APPWRITE_SMTP_HOST', 'maildev'), + 'port' => (int) $this->env('APPWRITE_SMTP_PORT', '1025'), + 'username' => $this->env('APPWRITE_SMTP_USERNAME', 'user'), + 'password' => $this->env('APPWRITE_SMTP_PASSWORD', 'password'), + 'encryption' => $this->env('APPWRITE_SMTP_ENCRYPTION', 'none'), + 'autoTLS' => false, + 'fromName' => 'Benchmark', + 'fromEmail' => 'benchmark@appwrite.io', + 'replyToName' => 'Benchmark', + 'replyToEmail' => 'benchmark@appwrite.io', + 'enabled' => true, + ], $context['apiHeaders'], [201], 'messaging.providers.smtp.create'); + + $targets = $this->api('GET', "/users/{$context['userId']}/targets", null, $context['apiHeaders'], [200], 'users.targets.list'); + foreach ($targets->json('targets') ?? [] as $target) { + if (($target['providerType'] ?? '') === 'email' && ($target['identifier'] ?? '') === $context['userEmail']) { + $targetId = (string) $target['$id']; + $existingTarget = true; + break; + } + } + + if ($existingTarget) { + $this->api('PATCH', "/users/{$context['userId']}/targets/{$targetId}", [ + 'providerId' => $providerId, + 'name' => 'Benchmark email target', + ], $context['apiHeaders'], [200], 'users.targets.update'); + } else { + $this->api('POST', "/users/{$context['userId']}/targets", [ + 'targetId' => $targetId, + 'providerType' => 'email', + 'identifier' => $context['userEmail'], + 'providerId' => $providerId, + 'name' => 'Benchmark email target', + ], $context['apiHeaders'], [201], 'users.targets.create'); + } + + $this->api('POST', '/messaging/topics', [ + 'topicId' => $topicId, + 'name' => 'Benchmark Topic', + 'subscribe' => ['users'], + ], $context['apiHeaders'], [201], 'messaging.topics.create'); + + $this->api('POST', "/messaging/topics/{$topicId}/subscribers", [ + 'subscriberId' => $subscriberId, + 'targetId' => $targetId, + ], $context['sessionHeaders'], [201], 'messaging.subscribers.create'); + + $started = $this->nowMs(); + $this->api('POST', '/messaging/messages/email', [ + 'messageId' => $messageId, + 'subject' => "Benchmark message {$context['runId']}", + 'content' => "Benchmark messaging worker probe {$context['runId']}", + 'targets' => [$targetId], + 'draft' => false, + 'html' => false, + ], $context['apiHeaders'], [201], 'messaging.messages.email.create'); + + $this->waitForMessage($messageId, $context['apiHeaders'], $this->workerTimeoutMs); + $this->waitForEmail($context['userEmail'], fn (array $message): bool => $this->includes($message['subject'] ?? '', "Benchmark message {$context['runId']}"), $this->mailTimeoutMs, true); + $this->metrics->addTrend('appwrite_worker_messaging_duration', $this->nowMs() - $started); + + $this->api('GET', '/messaging/messages', null, $context['apiHeaders'], [200], 'messaging.messages.list'); + $this->api('GET', "/messaging/messages/{$messageId}/logs", null, $context['apiHeaders'], [200], 'messaging.messages.logs.list'); + $this->api('GET', "/messaging/messages/{$messageId}/targets", null, $context['apiHeaders'], [200], 'messaging.messages.targets.list'); + $this->api('GET', "/messaging/providers/{$providerId}/logs", null, $context['apiHeaders'], [200], 'messaging.providers.logs.list'); + $this->api('GET', "/messaging/topics/{$topicId}/logs", null, $context['apiHeaders'], [200], 'messaging.topics.logs.list'); + $this->api('GET', "/messaging/subscribers/{$subscriberId}/logs", null, $context['apiHeaders'], [200], 'messaging.subscribers.logs.list'); + $this->api('DELETE', "/messaging/topics/{$topicId}/subscribers/{$subscriberId}", null, $context['sessionHeaders'], [204], 'messaging.subscribers.delete'); + $this->api('DELETE', "/messaging/topics/{$topicId}", null, $context['apiHeaders'], [204], 'messaging.topics.delete'); + $this->api('DELETE', "/messaging/messages/{$messageId}", null, $context['apiHeaders'], [204], 'messaging.messages.delete'); + $this->api('DELETE', "/messaging/providers/{$providerId}", null, $context['apiHeaders'], [204], 'messaging.providers.delete'); + } + + private function computeFlow(array $context): void + { + $functionId = $this->unique('fn'); + $siteId = $this->unique('site'); + $runtime = $this->env('APPWRITE_BENCHMARK_RUNTIME', 'node-22'); + + $this->api('POST', '/functions', [ + 'functionId' => $functionId, + 'name' => 'Benchmark Function', + 'runtime' => $runtime, + 'execute' => ['any'], + 'events' => [], + 'schedule' => '', + 'timeout' => 15, + 'enabled' => true, + 'logging' => true, + 'entrypoint' => 'index.js', + 'commands' => 'npm install', + 'scopes' => ['users.read'], + ], $context['apiHeaders'], [201], 'functions.create'); + $this->api('GET', '/functions/runtimes', null, $context['sessionHeaders'], [200], 'functions.runtimes.list'); + $this->api('GET', '/functions/specifications', null, $context['apiHeaders'], [200], 'functions.specifications.list'); + + $functionVariable = $this->api('POST', "/functions/{$functionId}/variables", [ + 'key' => 'BENCHMARK', + 'value' => 'true', + 'secret' => false, + ], $context['apiHeaders'], [201], 'functions.variables.create'); + $functionVariableId = (string) $functionVariable->json('$id'); + $this->api('PUT', "/functions/{$functionId}/variables/{$functionVariableId}", ['key' => 'BENCHMARK', 'value' => 'updated', 'secret' => false], $context['apiHeaders'], [200], 'functions.variables.update'); + $this->api('GET', "/functions/{$functionId}/variables/{$functionVariableId}", null, $context['apiHeaders'], [200], 'functions.variables.get'); + $this->api('DELETE', "/functions/{$functionId}/variables/{$functionVariableId}", null, $context['apiHeaders'], [204], 'functions.variables.delete'); + $this->api('DELETE', "/functions/{$functionId}", null, $context['apiHeaders'], [204], 'functions.delete'); + + $this->api('POST', '/sites', [ + 'siteId' => $siteId, + 'name' => 'Benchmark Site', + 'framework' => 'other', + 'adapter' => 'static', + 'buildRuntime' => $runtime, + 'buildCommand' => '', + 'outputDirectory' => '.', + 'installCommand' => '', + 'fallbackFile' => 'index.html', + 'providerRootDirectory' => '.', + 'specification' => '', + ], $context['apiHeaders'], [201], 'sites.create'); + $this->api('GET', '/sites/frameworks', null, $context['sessionHeaders'], [200], 'sites.frameworks.list'); + $this->api('GET', '/sites/specifications', null, $context['apiHeaders'], [200], 'sites.specifications.list'); + + $siteVariable = $this->api('POST', "/sites/{$siteId}/variables", ['key' => 'BENCHMARK', 'value' => 'true', 'secret' => false], $context['apiHeaders'], [201], 'sites.variables.create'); + $siteVariableId = (string) $siteVariable->json('$id'); + $this->api('PUT', "/sites/{$siteId}/variables/{$siteVariableId}", ['key' => 'BENCHMARK', 'value' => 'updated', 'secret' => false], $context['apiHeaders'], [200], 'sites.variables.update'); + $this->api('GET', "/sites/{$siteId}/variables/{$siteVariableId}", null, $context['apiHeaders'], [200], 'sites.variables.get'); + $this->api('DELETE', "/sites/{$siteId}/variables/{$siteVariableId}", null, $context['apiHeaders'], [204], 'sites.variables.delete'); + $this->api('DELETE', "/sites/{$siteId}", null, $context['apiHeaders'], [204], 'sites.delete'); + } + + private function healthFlow(array $context): void + { + $probes = [ + '/health', + '/health/db', + '/health/cache', + '/health/pubsub', + '/health/storage', + '/health/storage/local', + '/health/time', + '/health/queue/databases', + '/health/queue/mails', + '/health/queue/messaging', + '/health/queue/functions', + '/health/queue/builds', + '/health/queue/deletes', + '/health/queue/webhooks', + '/health/queue/stats-resources', + '/health/queue/stats-usage', + '/health/queue/failed/v1-mails', + ]; + + foreach ($probes as $path) { + $this->api('GET', $path, null, $context['apiHeaders'], [200], 'health' . str_replace('/', '.', $path)); + } + } + + private function api(string $method, string $path, mixed $body, array $headers, array $expected, string $name): BenchmarkResponse + { + $response = $this->rawRequest($method, $path, $body, $headers, $name); + $this->metrics->addTrend('appwrite_api_duration', $response->duration); + $this->assertStatus($response, $expected, $name); + return $response; + } + + private function rawRequest(string $method, string $path, mixed $body, array $headers, string $name): BenchmarkResponse + { + return $this->send($method, str_starts_with($path, 'http') ? $path : $this->endpoint . $path, $body, $headers, $name, false); + } + + private function rawMultipartRequest(string $method, string $path, array $fields, array $headers, string $name): BenchmarkResponse + { + return $this->send($method, $this->endpoint . $path, $fields, $headers, $name, true); + } + + private function send(string $method, string $url, mixed $body, array $headers, string $name, bool $multipart): BenchmarkResponse + { + $handle = curl_init($url); + if ($handle === false) { + throw new RuntimeException("Unable to initialize curl for {$url}"); + } + + $headerLines = []; + foreach ($headers as $key => $value) { + $headerLines[] = "{$key}: {$value}"; + } + + curl_setopt_array($handle, [ + CURLOPT_CUSTOMREQUEST => $method, + CURLOPT_RETURNTRANSFER => true, + CURLOPT_HEADER => true, + CURLOPT_HTTPHEADER => $headerLines, + CURLOPT_TIMEOUT => 120, + ]); + + if ($body !== null) { + curl_setopt($handle, CURLOPT_POSTFIELDS, $multipart ? $body : json_encode($body, JSON_UNESCAPED_SLASHES)); + } elseif (in_array($method, ['POST', 'PUT', 'PATCH'], true)) { + curl_setopt($handle, CURLOPT_POSTFIELDS, ''); + } + + $started = hrtime(true); + $raw = curl_exec($handle); + $duration = (hrtime(true) - $started) / 1_000_000; + $this->metrics->addTrend('http_req_duration', $duration); + + if ($raw === false) { + $error = curl_error($handle); + throw new RuntimeException("{$name} curl error: {$error}"); + } + + $status = (int) curl_getinfo($handle, CURLINFO_RESPONSE_CODE); + $headerSize = (int) curl_getinfo($handle, CURLINFO_HEADER_SIZE); + + return new BenchmarkResponse( + $status, + substr($raw, $headerSize), + $this->parseHeaders(substr($raw, 0, $headerSize)), + $duration, + ); + } + + private function waitForStatus(string $path, array $headers, string $wantedStatus, int $timeoutMs): BenchmarkResponse + { + $started = $this->nowMs(); + + while ($this->nowMs() - $started < $timeoutMs) { + $response = $this->rawRequest('GET', $path, null, $headers, "wait{$path}"); + if ($response->status === 200 && $response->json('status') === $wantedStatus) { + return $response; + } + + usleep(500_000); + } + + throw new RuntimeException("Timed out waiting for {$path} to become {$wantedStatus}"); + } + + private function waitForMessage(string $messageId, array $headers, int $timeoutMs): BenchmarkResponse + { + $started = $this->nowMs(); + + while ($this->nowMs() - $started < $timeoutMs) { + $response = $this->rawRequest('GET', "/messaging/messages/{$messageId}", null, $headers, 'messaging.messages.poll'); + $status = $response->status === 200 ? $response->json('status') : null; + + if (in_array($status, ['sent', 'failed'], true)) { + if ($status === 'failed') { + throw new RuntimeException("Messaging worker marked message {$messageId} as failed"); + } + + return $response; + } + + usleep(500_000); + } + + throw new RuntimeException("Timed out waiting for messaging worker to send message {$messageId}"); + } + + private function waitForEmail(string $address, callable $predicate, int $timeoutMs, bool $allowMissingRecipient = false): array + { + $started = $this->nowMs(); + + while ($this->nowMs() - $started < $timeoutMs) { + $response = $this->rawRequest('GET', $this->maildevEndpoint, null, [], 'maildev.email.list'); + + if ($response->status === 200) { + $emails = $response->json(); + if (is_array($emails)) { + for ($i = count($emails) - 1; $i >= 0; $i--) { + $message = $emails[$i]; + if (!is_array($message)) { + continue; + } + + if (($this->emailMatches($message, $address) || ($allowMissingRecipient && $this->emailRecipientMissing($message))) && $predicate($message)) { + return $message; + } + } + } + } + + usleep(500_000); + } + + throw new RuntimeException("Timed out waiting for email to {$address}"); + } + + private function assertStatus(BenchmarkResponse $response, array $expected, string $name): void + { + $passed = in_array($response->status, $expected, true); + $this->metrics->addCheck($passed); + + if (!$passed) { + $this->failResponse($response, "{$name} returned an unexpected status"); + } + } + + private function failResponse(BenchmarkResponse $response, string $message): never + { + throw new RuntimeException("{$message}. Status: {$response->status}. Body: {$response->body}"); + } + + private function parseHeaders(string $rawHeaders): array + { + $blocks = preg_split("/\r\n\r\n|\n\n/", trim($rawHeaders)) ?: []; + $headerBlock = end($blocks) ?: ''; + $headers = []; + + foreach (preg_split("/\r\n|\n|\r/", $headerBlock) ?: [] as $line) { + if (!str_contains($line, ':')) { + continue; + } + + [$name, $value] = explode(':', $line, 2); + $headers[strtolower(trim($name))][] = trim($value); + } + + return $headers; + } + + private function emailMatches(array $message, string $address): bool + { + foreach ($message['to'] ?? [] as $recipient) { + if (($recipient['address'] ?? null) === $address) { + return true; + } + } + + return false; + } + + private function emailRecipientMissing(array $message): bool + { + $recipients = $message['to'] ?? []; + if ($recipients === []) { + return true; + } + + foreach ($recipients as $recipient) { + if ($recipient['address'] ?? null) { + return false; + } + } + + return true; + } + + private function extractQueryParams(array $message): array + { + $content = ($message['html'] ?? '') . "\n" . ($message['text'] ?? ''); + preg_match_all('/href="([^"]+)"/', $content, $matches); + $links = $matches[1] ?: [$content]; + + foreach ($links as $link) { + $query = parse_url(html_entity_decode($link), PHP_URL_QUERY); + if (!is_string($query)) { + continue; + } + + parse_str($query, $params); + if (($params['userId'] ?? null) && ($params['secret'] ?? null)) { + return $params; + } + } + + return []; + } + + private function projectHeaders(string $projectId): array + { + return [ + 'Content-Type' => 'application/json', + 'X-Appwrite-Project' => $projectId, + ]; + } + + private function documentPayload(): array + { + return [ + 'title' => 'Benchmark Document', + 'count' => 1, + 'email' => 'document@example.com', + 'active' => true, + 'publishedAt' => gmdate('c'), + 'score' => 10.5, + 'url' => 'https://appwrite.io', + 'ip' => '127.0.0.1', + ]; + } + + private function tablePayload(): array + { + return [ + 'title' => 'Benchmark Row', + 'count' => 1, + 'email' => 'row@example.com', + 'active' => true, + ]; + } + + private function flattenMultipartArray(string $key, array $values): array + { + $output = []; + + foreach (array_values($values) as $index => $value) { + $output["{$key}[{$index}]"] = $value; + } + + return $output; + } + + private function messageIncludes(array $message, array $needles): bool + { + $content = implode("\n", [ + (string) ($message['subject'] ?? ''), + (string) ($message['html'] ?? ''), + (string) ($message['text'] ?? ''), + ]); + + foreach ($needles as $needle) { + if ($this->includes($content, $needle)) { + return true; + } + } + + return false; + } + + private function includes(string $value, string $needle): bool + { + return str_contains(strtolower($value), strtolower($needle)); + } + + private function hostnameFromUrl(string $value): string + { + $host = parse_url($value, PHP_URL_HOST); + if (is_string($host) && $host !== '') { + return $host; + } + + return explode(':', explode('/', preg_replace('/^https?:\/\//', '', $value) ?? '')[0])[0]; + } + + private function unique(string $prefix): string + { + $id = strtolower($prefix . '-' . base_convert((string) ((int) (microtime(true) * 1000)), 10, 36) . '-' . bin2hex(random_bytes(4))); + return substr(preg_replace('/[^a-z0-9-]/', '-', $id) ?? $id, 0, 36); + } + + private function nowMs(): float + { + return hrtime(true) / 1_000_000; + } + + private function env(string $name, string $default): string + { + $value = getenv($name); + return $value === false || $value === '' ? $default : $value; + } + + private function loadPreviousSummary(string $path): ?array + { + if (!is_file($path)) { + return null; + } + + $summary = json_decode((string) file_get_contents($path), true); + return is_array($summary) ? $summary : null; + } + + private function writeSummary(array $summary): void + { + $directory = dirname($this->summaryPath); + if ($directory !== '.' && !is_dir($directory)) { + mkdir($directory, 0777, true); + } + + file_put_contents($this->summaryPath, json_encode($summary, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); + } + + private function renderSummary(array $summary): string + { + $lines = [ + 'Appwrite curated benchmark review', + '', + 'Before/after comparison', + '', + $this->comparisonTable($this->previousSummary, $summary), + '', + 'Current run details', + '', + $this->metricLine($summary, 'http_req_duration', 'HTTP total'), + $this->metricLine($summary, 'appwrite_api_duration', 'API endpoints'), + $this->metricLine($summary, 'appwrite_worker_database_duration', 'Database worker schema jobs'), + $this->metricLine($summary, 'appwrite_worker_tables_duration', 'TablesDB worker schema jobs'), + $this->metricLine($summary, 'appwrite_worker_mails_duration', 'Mail worker delivery'), + $this->metricLine($summary, 'appwrite_worker_messaging_duration', 'Messaging worker delivery'), + $this->counterLine($summary, 'appwrite_benchmark_flow_failures', 'Flow failures'), + '', + ]; + + return implode(PHP_EOL, array_filter($lines, fn (string $line): bool => $line !== '')) . PHP_EOL; + } + + private function comparisonTable(?array $before, array $after): string + { + $rows = [ + ['HTTP total p95', $this->trendMetric($before, 'http_req_duration', 'p(95)'), $this->trendMetric($after, 'http_req_duration', 'p(95)'), 'ms'], + ['API endpoints p95', $this->trendMetric($before, 'appwrite_api_duration', 'p(95)'), $this->trendMetric($after, 'appwrite_api_duration', 'p(95)'), 'ms'], + ['Database worker p95', $this->trendMetric($before, 'appwrite_worker_database_duration', 'p(95)'), $this->trendMetric($after, 'appwrite_worker_database_duration', 'p(95)'), 'ms'], + ['TablesDB worker p95', $this->trendMetric($before, 'appwrite_worker_tables_duration', 'p(95)'), $this->trendMetric($after, 'appwrite_worker_tables_duration', 'p(95)'), 'ms'], + ['Mail worker p95', $this->trendMetric($before, 'appwrite_worker_mails_duration', 'p(95)'), $this->trendMetric($after, 'appwrite_worker_mails_duration', 'p(95)'), 'ms'], + ['Messaging worker p95', $this->trendMetric($before, 'appwrite_worker_messaging_duration', 'p(95)'), $this->trendMetric($after, 'appwrite_worker_messaging_duration', 'p(95)'), 'ms'], + ['Flow failures', $this->counterMetric($before, 'appwrite_benchmark_flow_failures'), $this->counterMetric($after, 'appwrite_benchmark_flow_failures'), ''], + ['Check failures', $this->checkFailures($before), $this->checkFailures($after), ''], + ]; + + $table = [ + '| Metric | Before | After | Delta |', + '| --- | ---: | ---: | ---: |', + ]; + + foreach ($rows as [$label, $beforeValue, $afterValue, $unit]) { + $table[] = "| {$label} | {$this->formatValue($beforeValue, $unit)} | {$this->formatValue($afterValue, $unit)} | {$this->formatDelta($beforeValue, $afterValue, $unit)} |"; + } + + return implode(PHP_EOL, $table); + } + + private function trendMetric(?array $data, string $metric, string $stat): ?float + { + return $data['metrics'][$metric]['values'][$stat] ?? null; + } + + private function counterMetric(?array $data, string $metric): ?float + { + return $data['metrics'][$metric]['values']['count'] ?? null; + } + + private function checkFailures(?array $data): ?float + { + return $data['metrics']['checks']['values']['fails'] ?? null; + } + + private function metricLine(array $data, string $metric, string $label): string + { + $values = $data['metrics'][$metric]['values'] ?? null; + if (!is_array($values) || ($values['count'] ?? 0) === 0) { + return "{$label}: no samples"; + } + + return "{$label}: avg={$this->round($values['avg'])}ms p90={$this->round($values['p(90)'])}ms p95={$this->round($values['p(95)'])}ms max={$this->round($values['max'])}ms"; + } + + private function counterLine(array $data, string $metric, string $label): string + { + return "{$label}: " . ($data['metrics'][$metric]['values']['count'] ?? 0); + } + + private function formatValue(?float $value, string $unit): string + { + return $value === null || is_nan($value) ? 'n/a' : $this->round($value) . $unit; + } + + private function formatDelta(?float $before, ?float $after, string $unit): string + { + if ($before === null || $after === null || is_nan($before) || is_nan($after)) { + return 'n/a'; + } + + $delta = $this->round($after - $before); + return ($delta > 0 ? '+' : '') . $delta . $unit; + } + + private function round(float|int|null $value): string + { + $rounded = round((float) ($value ?? 0), 2); + return rtrim(rtrim(number_format($rounded, 2, '.', ''), '0'), '.'); + } +} + +exit((new HttpBenchmark())->run()); From 566eebfaecf5e81b0afc5933f7d4460d3705524a Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 16:31:57 +0530 Subject: [PATCH 09/39] Fix benchmark PNG fixture --- tests/benchmarks/http.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/benchmarks/http.php b/tests/benchmarks/http.php index 1d91246352..3d7556e8d7 100644 --- a/tests/benchmarks/http.php +++ b/tests/benchmarks/http.php @@ -248,7 +248,7 @@ final class HttpBenchmark 'delete("any")', ]; - private const PNG_1X1 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII='; + private const PNG_1X1 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII='; private BenchmarkMetrics $metrics; private string $endpoint; From ef08d5a04c2c6a2b7ac2e00de837dcdc54d65088 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 16:33:09 +0530 Subject: [PATCH 10/39] Run before benchmark with base image --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78d959779c..595f496e1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -710,7 +710,7 @@ jobs: -e APPWRITE_BENCHMARK_ITERATIONS=1 \ -e APPWRITE_BENCHMARK_VUS=1 \ -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-before-summary.json \ - ${{ env.IMAGE }}:after php tests/benchmarks/http.php | tee benchmark-before.txt + ${{ env.IMAGE }}:before php tests/benchmarks/http.php | tee benchmark-before.txt - name: Stop before Appwrite if: always() From 30cfbb2d992de8bd926a338f234698a7c46ea1d4 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 16:58:31 +0530 Subject: [PATCH 11/39] Polish benchmark reporting --- .github/workflows/ci.yml | 26 ++++++++++++++++++++++++-- tests/benchmarks/http.php | 9 +++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 595f496e1d..23aa1f21de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -746,8 +746,30 @@ jobs: docker run --rm -i -v "$PWD:/scripts" -w /scripts ${{ env.IMAGE }}:after php <<'PHP' > benchmark-comment.txt getMessage()}\n"); + exit(1); + } + + if (!is_array($summary)) { + fwrite(STDERR, "Invalid benchmark summary {$path}: expected JSON object\n"); + exit(1); + } + + return $summary; + } + + $before = read_summary('benchmark-before-summary.json'); + $after = read_summary('benchmark-after-summary.json'); function metric_value(?array $data, string $metric, string $stat): mixed { diff --git a/tests/benchmarks/http.php b/tests/benchmarks/http.php index 3d7556e8d7..1c28ef33ea 100644 --- a/tests/benchmarks/http.php +++ b/tests/benchmarks/http.php @@ -290,7 +290,12 @@ final class HttpBenchmark $context = $this->setup(); for ($i = 0; $i < $this->iterations * $this->vus; $i++) { - $this->curatedFlows($context); + try { + $this->curatedFlows($context); + } catch (Throwable $error) { + $exitCode = 1; + fwrite(STDERR, 'Iteration ' . ($i + 1) . ' failed: ' . $error->getMessage() . PHP_EOL); + } } } catch (Throwable $error) { $exitCode = 1; @@ -1216,7 +1221,7 @@ final class HttpBenchmark '', ]; - return implode(PHP_EOL, array_filter($lines, fn (string $line): bool => $line !== '')) . PHP_EOL; + return implode(PHP_EOL, $lines) . PHP_EOL; } private function comparisonTable(?array $before, array $after): string From 63b2a1fb7fff0c7c22c9105585e8b95a26f37be7 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 17:07:05 +0530 Subject: [PATCH 12/39] Harden benchmark baseline reporting --- .github/workflows/ci.yml | 41 ++++++++++++++++++++++++++++----------- tests/benchmarks/http.php | 6 +++--- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23aa1f21de..403160a8aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -683,10 +683,13 @@ jobs: docker tag ${{ env.IMAGE }} ${{ env.IMAGE }}:after - name: Prepare benchmark before + id: benchmark_before_prepare + continue-on-error: true run: | git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} git worktree add --detach /tmp/appwrite-benchmark-before ${{ github.event.pull_request.base.sha }} docker build \ + --cache-from ${{ env.IMAGE }}:after \ --target development \ --build-arg DEBUG=false \ --build-arg TESTING=true \ @@ -695,6 +698,9 @@ jobs: /tmp/appwrite-benchmark-before - name: Start before Appwrite + id: benchmark_before_start + if: steps.benchmark_before_prepare.outcome == 'success' + continue-on-error: true working-directory: /tmp/appwrite-benchmark-before run: | docker tag ${{ env.IMAGE }}:before ${{ env.IMAGE }} @@ -702,13 +708,15 @@ jobs: docker compose up -d --wait --no-build - name: Benchmark before + if: steps.benchmark_before_start.outcome == 'success' + continue-on-error: true run: | rm -f benchmark-before-summary.json benchmark-after-summary.json benchmark-before.txt benchmark.txt docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts \ -e APPWRITE_ENDPOINT=http://localhost/v1 \ -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ -e APPWRITE_BENCHMARK_ITERATIONS=1 \ - -e APPWRITE_BENCHMARK_VUS=1 \ + -e APPWRITE_BENCHMARK_RUNS=1 \ -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-before-summary.json \ ${{ env.IMAGE }}:before php tests/benchmarks/http.php | tee benchmark-before.txt @@ -717,7 +725,7 @@ jobs: run: | if [ -d /tmp/appwrite-benchmark-before ]; then cd /tmp/appwrite-benchmark-before - docker compose down -v + docker compose down -v || true fi - name: Start after Appwrite @@ -732,7 +740,7 @@ jobs: -e APPWRITE_ENDPOINT=http://localhost/v1 \ -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ -e APPWRITE_BENCHMARK_ITERATIONS=1 \ - -e APPWRITE_BENCHMARK_VUS=1 \ + -e APPWRITE_BENCHMARK_RUNS=1 \ -e APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH=benchmark-before-summary.json \ -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-after-summary.json \ ${{ env.IMAGE }}:after php tests/benchmarks/http.php | tee benchmark.txt @@ -746,29 +754,37 @@ jobs: docker run --rm -i -v "$PWD:/scripts" -w /scripts ${{ env.IMAGE }}:after php <<'PHP' > benchmark-comment.txt getMessage()}\n"); - exit(1); + fail_summary("Invalid benchmark summary {$path}: {$error->getMessage()}", $required); + return null; } if (!is_array($summary)) { - fwrite(STDERR, "Invalid benchmark summary {$path}: expected JSON object\n"); - exit(1); + fail_summary("Invalid benchmark summary {$path}: expected JSON object", $required); + return null; } return $summary; } - $before = read_summary('benchmark-before-summary.json'); + $before = read_summary('benchmark-before-summary.json', false); $after = read_summary('benchmark-after-summary.json'); function metric_value(?array $data, string $metric, string $stat): mixed @@ -829,6 +845,9 @@ jobs: echo "\n"; echo "## :sparkles: Benchmark results\n\n"; echo 'Comparing `${{ github.event.pull_request.base.ref }}` (before) to `${{ github.event.pull_request.head.ref }}` (after).' . "\n\n"; + if ($before === null) { + echo "> Before benchmark did not complete; showing current branch metrics only.\n\n"; + } echo "| Metric | Before | After | Delta |\n"; echo "| --- | ---: | ---: | ---: |\n"; echo implode("\n", $rows) . "\n\n"; diff --git a/tests/benchmarks/http.php b/tests/benchmarks/http.php index 1c28ef33ea..c44677c4ec 100644 --- a/tests/benchmarks/http.php +++ b/tests/benchmarks/http.php @@ -260,7 +260,7 @@ final class HttpBenchmark private int $mailTimeoutMs; private int $workerTimeoutMs; private int $iterations; - private int $vus; + private int $runs; private string $summaryPath; private ?array $previousSummary; @@ -276,7 +276,7 @@ final class HttpBenchmark $this->mailTimeoutMs = (int) $this->env('APPWRITE_MAIL_TIMEOUT_MS', '20000'); $this->workerTimeoutMs = (int) $this->env('APPWRITE_WORKER_TIMEOUT_MS', '60000'); $this->iterations = max(1, (int) $this->env('APPWRITE_BENCHMARK_ITERATIONS', '1')); - $this->vus = max(1, (int) $this->env('APPWRITE_BENCHMARK_VUS', '1')); + $this->runs = max(1, (int) $this->env('APPWRITE_BENCHMARK_RUNS', $this->env('APPWRITE_BENCHMARK_VUS', '1'))); $this->summaryPath = $this->env('APPWRITE_BENCHMARK_SUMMARY_PATH', 'tests/benchmarks/http-summary.json'); $this->previousSummary = $this->loadPreviousSummary($this->env('APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH', $this->summaryPath)); } @@ -289,7 +289,7 @@ final class HttpBenchmark try { $context = $this->setup(); - for ($i = 0; $i < $this->iterations * $this->vus; $i++) { + for ($i = 0; $i < $this->iterations * $this->runs; $i++) { try { $this->curatedFlows($context); } catch (Throwable $error) { From 9c65609d730c549fcbd84fac0b903d6d6e7f6a10 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 17:13:00 +0530 Subject: [PATCH 13/39] Check benchmark summary writes --- tests/benchmarks/http.php | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/benchmarks/http.php b/tests/benchmarks/http.php index c44677c4ec..73fa6ea9ec 100644 --- a/tests/benchmarks/http.php +++ b/tests/benchmarks/http.php @@ -1193,11 +1193,18 @@ final class HttpBenchmark private function writeSummary(array $summary): void { $directory = dirname($this->summaryPath); - if ($directory !== '.' && !is_dir($directory)) { - mkdir($directory, 0777, true); + if ($directory !== '.' && !is_dir($directory) && !mkdir($directory, 0777, true) && !is_dir($directory)) { + throw new RuntimeException("Unable to create benchmark summary directory: {$directory}"); } - file_put_contents($this->summaryPath, json_encode($summary, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); + $json = json_encode($summary, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + if ($json === false) { + throw new RuntimeException('Unable to encode benchmark summary: ' . json_last_error_msg()); + } + + if (file_put_contents($this->summaryPath, $json) === false) { + throw new RuntimeException("Unable to write benchmark summary: {$this->summaryPath}"); + } } private function renderSummary(array $summary): string From b9d01617a42f01b7b2c758aa55b07164d57f5f6f Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 17:24:26 +0530 Subject: [PATCH 14/39] Address benchmark review hardening --- .github/workflows/ci.yml | 23 +++++++++++++++++++++-- tests/benchmarks/http.php | 18 ++++++++++++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 403160a8aa..5db20064dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -750,10 +750,27 @@ jobs: run: docker compose down -v - name: Prepare comment + env: + BENCHMARK_BASE_REF: ${{ github.event.pull_request.base.ref }} + BENCHMARK_HEAD_REF: ${{ github.event.pull_request.head.ref }} run: | - docker run --rm -i -v "$PWD:/scripts" -w /scripts ${{ env.IMAGE }}:after php <<'PHP' > benchmark-comment.txt + docker run --rm -i -v "$PWD:/scripts" -w /scripts \ + -e BENCHMARK_BASE_REF \ + -e BENCHMARK_HEAD_REF \ + ${{ env.IMAGE }}:after php <<'PHP' > benchmark-comment.txt \n"; echo "## :sparkles: Benchmark results\n\n"; - echo 'Comparing `${{ github.event.pull_request.base.ref }}` (before) to `${{ github.event.pull_request.head.ref }}` (after).' . "\n\n"; + echo "Comparing {$baseRef} (before) to {$headRef} (after).\n\n"; if ($before === null) { echo "> Before benchmark did not complete; showing current branch metrics only.\n\n"; } diff --git a/tests/benchmarks/http.php b/tests/benchmarks/http.php index 73fa6ea9ec..c71991a062 100644 --- a/tests/benchmarks/http.php +++ b/tests/benchmarks/http.php @@ -302,12 +302,22 @@ final class HttpBenchmark fwrite(STDERR, $error->getMessage() . PHP_EOL); } finally { if (is_array($context)) { - $this->teardown($context); + try { + $this->teardown($context); + } catch (Throwable $error) { + $exitCode = 1; + fwrite(STDERR, 'Teardown failed: ' . $error->getMessage() . PHP_EOL); + } } $summary = $this->metrics->summary(); - $this->writeSummary($summary); echo $this->renderSummary($summary); + try { + $this->writeSummary($summary); + } catch (Throwable $error) { + $exitCode = 1; + fwrite(STDERR, $error->getMessage() . PHP_EOL); + } } if ($this->metrics->failedChecks() > 0 || $this->metrics->flowFailures() > 0) { @@ -586,6 +596,10 @@ final class HttpBenchmark private function tablesDbFlow(array $context): void { + if (!isset($context['sessionHeaders']) || !is_array($context['sessionHeaders'])) { + throw new RuntimeException('accountFlow must run before tablesDbFlow'); + } + $databaseId = $this->unique('tdb'); $tableId = $this->unique('tbl'); $rowId = $this->unique('row'); From 196b04a39c9d4a23bf97ce169f7da80cd7b242a5 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 17:49:50 +0530 Subject: [PATCH 15/39] Polish benchmark comment details --- .github/workflows/ci.yml | 38 +++++++++++++++++++++++--------------- tests/benchmarks/http.php | 25 +++++++++++++++++-------- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5db20064dc..9e1a79cb3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -837,17 +837,24 @@ jobs: return '| ' . $label . ' | ' . format_value($beforeValue, $suffix) . ' | ' . format_value($afterValue, $suffix) . ' | ' . delta($beforeValue, $afterValue, $suffix) . ' |'; } - function detail(array $after, string $label, string $metric, string $suffix = 'ms'): string + function format_detail_value(mixed $value, string $suffix = ''): string + { + return $value === null ? 'n/a' : number_format((float) $value, 2, '.', '') . $suffix; + } + + function detail_row(array $after, string $label, string $metric, string $suffix = 'ms'): string { $values = $after['metrics'][$metric]['values'] ?? null; if (!is_array($values)) { - return '- **' . $label . ':** no samples'; + return '| ' . $label . ' | n/a | n/a | n/a | n/a |'; } - return '- **' . $label . ':** avg=' . format_value($values['avg'] ?? null, $suffix) - . ' p90=' . format_value($values['p(90)'] ?? null, $suffix) - . ' p95=' . format_value($values['p(95)'] ?? null, $suffix) - . ' max=' . format_value($values['max'] ?? null, $suffix); + return '| ' . $label + . ' | ' . format_detail_value($values['avg'] ?? null, $suffix) + . ' | ' . format_detail_value($values['p(90)'] ?? null, $suffix) + . ' | ' . format_detail_value($values['p(95)'] ?? null, $suffix) + . ' | ' . format_detail_value($values['max'] ?? null, $suffix) + . ' |'; } $rows = [ @@ -857,8 +864,6 @@ jobs: row('TablesDB worker p95', metric_value($before, 'appwrite_worker_tables_duration', 'p(95)'), metric_value($after, 'appwrite_worker_tables_duration', 'p(95)'), 'ms'), row('Mail worker p95', metric_value($before, 'appwrite_worker_mails_duration', 'p(95)'), metric_value($after, 'appwrite_worker_mails_duration', 'p(95)'), 'ms'), row('Messaging worker p95', metric_value($before, 'appwrite_worker_messaging_duration', 'p(95)'), metric_value($after, 'appwrite_worker_messaging_duration', 'p(95)'), 'ms'), - row('Flow failures', metric_value($before, 'appwrite_benchmark_flow_failures', 'count'), metric_value($after, 'appwrite_benchmark_flow_failures', 'count')), - row('Check failures', metric_value($before, 'checks', 'fails'), metric_value($after, 'checks', 'fails')), ]; echo "\n"; @@ -871,13 +876,16 @@ jobs: echo "| --- | ---: | ---: | ---: |\n"; echo implode("\n", $rows) . "\n\n"; echo "
\n"; - echo "Current run details\n\n"; - echo detail($after, 'HTTP total', 'http_req_duration') . "\n"; - echo detail($after, 'API endpoints', 'appwrite_api_duration') . "\n"; - echo detail($after, 'Database worker schema jobs', 'appwrite_worker_database_duration') . "\n"; - echo detail($after, 'TablesDB worker schema jobs', 'appwrite_worker_tables_duration') . "\n"; - echo detail($after, 'Mail worker delivery', 'appwrite_worker_mails_duration') . "\n"; - echo detail($after, 'Messaging worker delivery', 'appwrite_worker_messaging_duration') . "\n\n"; + echo "Current run details\n\n"; + echo "
\n\n"; + echo "| Scenario | Avg | P90 | P95 | Max |\n"; + echo "| --- | ---: | ---: | ---: | ---: |\n"; + echo detail_row($after, 'HTTP total', 'http_req_duration') . "\n"; + echo detail_row($after, 'API endpoints', 'appwrite_api_duration') . "\n"; + echo detail_row($after, 'Database worker schema jobs', 'appwrite_worker_database_duration') . "\n"; + echo detail_row($after, 'TablesDB worker schema jobs', 'appwrite_worker_tables_duration') . "\n"; + echo detail_row($after, 'Mail worker delivery', 'appwrite_worker_mails_duration') . "\n"; + echo detail_row($after, 'Messaging worker delivery', 'appwrite_worker_messaging_duration') . "\n\n"; echo "
\n"; PHP diff --git a/tests/benchmarks/http.php b/tests/benchmarks/http.php index c71991a062..7924351409 100644 --- a/tests/benchmarks/http.php +++ b/tests/benchmarks/http.php @@ -897,17 +897,17 @@ final class HttpBenchmark return $response; } - private function rawRequest(string $method, string $path, mixed $body, array $headers, string $name): BenchmarkResponse + private function rawRequest(string $method, string $path, mixed $body, array $headers, string $name, bool $recordHttpDuration = true): BenchmarkResponse { - return $this->send($method, str_starts_with($path, 'http') ? $path : $this->endpoint . $path, $body, $headers, $name, false); + return $this->send($method, str_starts_with($path, 'http') ? $path : $this->endpoint . $path, $body, $headers, $name, false, $recordHttpDuration); } private function rawMultipartRequest(string $method, string $path, array $fields, array $headers, string $name): BenchmarkResponse { - return $this->send($method, $this->endpoint . $path, $fields, $headers, $name, true); + return $this->send($method, $this->endpoint . $path, $fields, $headers, $name, true, true); } - private function send(string $method, string $url, mixed $body, array $headers, string $name, bool $multipart): BenchmarkResponse + private function send(string $method, string $url, mixed $body, array $headers, string $name, bool $multipart, bool $recordHttpDuration): BenchmarkResponse { $handle = curl_init($url); if ($handle === false) { @@ -936,7 +936,9 @@ final class HttpBenchmark $started = hrtime(true); $raw = curl_exec($handle); $duration = (hrtime(true) - $started) / 1_000_000; - $this->metrics->addTrend('http_req_duration', $duration); + if ($recordHttpDuration) { + $this->metrics->addTrend('http_req_duration', $duration); + } if ($raw === false) { $error = curl_error($handle); @@ -960,8 +962,15 @@ final class HttpBenchmark while ($this->nowMs() - $started < $timeoutMs) { $response = $this->rawRequest('GET', $path, null, $headers, "wait{$path}"); - if ($response->status === 200 && $response->json('status') === $wantedStatus) { - return $response; + if ($response->status === 200) { + $status = $response->json('status'); + if ($status === $wantedStatus) { + return $response; + } + + if ($status === 'failed') { + throw new RuntimeException("Resource {$path} failed while waiting for {$wantedStatus}"); + } } usleep(500_000); @@ -997,7 +1006,7 @@ final class HttpBenchmark $started = $this->nowMs(); while ($this->nowMs() - $started < $timeoutMs) { - $response = $this->rawRequest('GET', $this->maildevEndpoint, null, [], 'maildev.email.list'); + $response = $this->rawRequest('GET', $this->maildevEndpoint, null, [], 'maildev.email.list', false); if ($response->status === 200) { $emails = $response->json(); From 30bf9deae0574154f1c7583d1f84cba04bfe9c5f Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 17:54:36 +0530 Subject: [PATCH 16/39] Remove benchmark failure rows from output --- tests/benchmarks/http.php | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/benchmarks/http.php b/tests/benchmarks/http.php index 7924351409..016fedb383 100644 --- a/tests/benchmarks/http.php +++ b/tests/benchmarks/http.php @@ -1247,7 +1247,6 @@ final class HttpBenchmark $this->metricLine($summary, 'appwrite_worker_tables_duration', 'TablesDB worker schema jobs'), $this->metricLine($summary, 'appwrite_worker_mails_duration', 'Mail worker delivery'), $this->metricLine($summary, 'appwrite_worker_messaging_duration', 'Messaging worker delivery'), - $this->counterLine($summary, 'appwrite_benchmark_flow_failures', 'Flow failures'), '', ]; @@ -1263,8 +1262,6 @@ final class HttpBenchmark ['TablesDB worker p95', $this->trendMetric($before, 'appwrite_worker_tables_duration', 'p(95)'), $this->trendMetric($after, 'appwrite_worker_tables_duration', 'p(95)'), 'ms'], ['Mail worker p95', $this->trendMetric($before, 'appwrite_worker_mails_duration', 'p(95)'), $this->trendMetric($after, 'appwrite_worker_mails_duration', 'p(95)'), 'ms'], ['Messaging worker p95', $this->trendMetric($before, 'appwrite_worker_messaging_duration', 'p(95)'), $this->trendMetric($after, 'appwrite_worker_messaging_duration', 'p(95)'), 'ms'], - ['Flow failures', $this->counterMetric($before, 'appwrite_benchmark_flow_failures'), $this->counterMetric($after, 'appwrite_benchmark_flow_failures'), ''], - ['Check failures', $this->checkFailures($before), $this->checkFailures($after), ''], ]; $table = [ @@ -1284,16 +1281,6 @@ final class HttpBenchmark return $data['metrics'][$metric]['values'][$stat] ?? null; } - private function counterMetric(?array $data, string $metric): ?float - { - return $data['metrics'][$metric]['values']['count'] ?? null; - } - - private function checkFailures(?array $data): ?float - { - return $data['metrics']['checks']['values']['fails'] ?? null; - } - private function metricLine(array $data, string $metric, string $label): string { $values = $data['metrics'][$metric]['values'] ?? null; @@ -1304,11 +1291,6 @@ final class HttpBenchmark return "{$label}: avg={$this->round($values['avg'])}ms p90={$this->round($values['p(90)'])}ms p95={$this->round($values['p(95)'])}ms max={$this->round($values['max'])}ms"; } - private function counterLine(array $data, string $metric, string $label): string - { - return "{$label}: " . ($data['metrics'][$metric]['values']['count'] ?? 0); - } - private function formatValue(?float $value, string $unit): string { return $value === null || is_nan($value) ? 'n/a' : $this->round($value) . $unit; From 211ac32080a78f8c20cc95bb05851028385db835 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Tue, 21 Apr 2026 17:59:09 +0530 Subject: [PATCH 17/39] Guard benchmark account-dependent flows --- tests/benchmarks/http.php | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/benchmarks/http.php b/tests/benchmarks/http.php index 016fedb383..9bddf57327 100644 --- a/tests/benchmarks/http.php +++ b/tests/benchmarks/http.php @@ -658,6 +658,10 @@ final class HttpBenchmark private function storageFlow(array $context): void { + if (!isset($context['sessionHeaders']) || !is_array($context['sessionHeaders'])) { + throw new RuntimeException('accountFlow must run before storageFlow'); + } + $bucketId = $this->unique('bucket'); $fileId = $this->unique('file'); @@ -720,6 +724,13 @@ final class HttpBenchmark private function messagingFlow(array $context): void { + if ( + !isset($context['sessionHeaders']) || !is_array($context['sessionHeaders']) + || !isset($context['userId'], $context['userEmail']) + ) { + throw new RuntimeException('accountFlow must run before messagingFlow'); + } + $providerId = $this->unique('smtp'); $targetId = $this->unique('target'); $existingTarget = false; @@ -806,6 +817,10 @@ final class HttpBenchmark private function computeFlow(array $context): void { + if (!isset($context['sessionHeaders']) || !is_array($context['sessionHeaders'])) { + throw new RuntimeException('accountFlow must run before computeFlow'); + } + $functionId = $this->unique('fn'); $siteId = $this->unique('site'); $runtime = $this->env('APPWRITE_BENCHMARK_RUNTIME', 'node-22'); From 9ca84a56c9bff1c03194bf98026eba48e05604ba Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 08:51:51 +0530 Subject: [PATCH 18/39] Switch HTTP benchmark back to k6 --- .github/workflows/ci.yml | 202 +++--- tests/benchmarks/http.js | 1017 ++++++++++++++++++++++++++++ tests/benchmarks/http.php | 1331 ------------------------------------- 3 files changed, 1104 insertions(+), 1446 deletions(-) create mode 100644 tests/benchmarks/http.js delete mode 100644 tests/benchmarks/http.php diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e1a79cb3d..327f32daa8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,7 @@ concurrency: env: COMPOSE_FILE: docker-compose.yml IMAGE: appwrite-dev + K6_IMAGE: grafana/k6:0.53.0 on: pull_request: @@ -712,13 +713,13 @@ jobs: continue-on-error: true run: | rm -f benchmark-before-summary.json benchmark-after-summary.json benchmark-before.txt benchmark.txt - docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts \ + docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts ${{ env.K6_IMAGE }} run --quiet \ -e APPWRITE_ENDPOINT=http://localhost/v1 \ -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ -e APPWRITE_BENCHMARK_ITERATIONS=1 \ - -e APPWRITE_BENCHMARK_RUNS=1 \ + -e APPWRITE_BENCHMARK_VUS=1 \ -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-before-summary.json \ - ${{ env.IMAGE }}:before php tests/benchmarks/http.php | tee benchmark-before.txt + tests/benchmarks/http.js | tee benchmark-before.txt - name: Stop before Appwrite if: always() @@ -736,14 +737,14 @@ jobs: - name: Benchmark after run: | - docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts \ + docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts ${{ env.K6_IMAGE }} run --quiet \ -e APPWRITE_ENDPOINT=http://localhost/v1 \ -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ -e APPWRITE_BENCHMARK_ITERATIONS=1 \ - -e APPWRITE_BENCHMARK_RUNS=1 \ + -e APPWRITE_BENCHMARK_VUS=1 \ -e APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH=benchmark-before-summary.json \ -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-after-summary.json \ - ${{ env.IMAGE }}:after php tests/benchmarks/http.php | tee benchmark.txt + tests/benchmarks/http.js | tee benchmark.txt - name: Stop after Appwrite if: always() @@ -754,140 +755,111 @@ jobs: BENCHMARK_BASE_REF: ${{ github.event.pull_request.base.ref }} BENCHMARK_HEAD_REF: ${{ github.event.pull_request.head.ref }} run: | - docker run --rm -i -v "$PWD:/scripts" -w /scripts \ - -e BENCHMARK_BASE_REF \ - -e BENCHMARK_HEAD_REF \ - ${{ env.IMAGE }}:after php <<'PHP' > benchmark-comment.txt - benchmark-comment.txt + const fs = require('fs'); - function env_value(string $name, string $default): string - { - $value = getenv($name); - return $value === false || $value === '' ? $default : $value; - } - - function markdown_text(string $value): string - { - return htmlspecialchars(str_replace(["\r", "\n"], ' ', $value), ENT_QUOTES | ENT_SUBSTITUTE, 'UTF-8'); - } - - function fail_summary(string $message, bool $required): void - { - fwrite(STDERR, $message . "\n"); - if ($required) { - exit(1); + function readSummary(path, required = true) { + if (!fs.existsSync(path)) { + if (required) { + throw new Error(`Missing benchmark summary: ${path}`); } + return null; + } + + try { + return JSON.parse(fs.readFileSync(path, 'utf8')); + } catch (error) { + throw new Error(`Invalid benchmark summary ${path}: ${error.message}`); + } } - function read_summary(string $path, bool $required = true): ?array - { - if (!is_file($path)) { - fail_summary("Missing benchmark summary: {$path}", $required); - return null; - } - - try { - $summary = json_decode(file_get_contents($path), true, 512, JSON_THROW_ON_ERROR); - } catch (JsonException $error) { - fail_summary("Invalid benchmark summary {$path}: {$error->getMessage()}", $required); - return null; - } - - if (!is_array($summary)) { - fail_summary("Invalid benchmark summary {$path}: expected JSON object", $required); - return null; - } - - return $summary; + function markdownText(value) { + return String(value || '').replace(/[\r\n]/g, ' ').replace(/[&<>"']/g, (char) => { + return ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[char]; + }); } - $before = read_summary('benchmark-before-summary.json', false); - $after = read_summary('benchmark-after-summary.json'); - $baseRef = markdown_text(env_value('BENCHMARK_BASE_REF', 'base')); - $headRef = markdown_text(env_value('BENCHMARK_HEAD_REF', 'head')); - - function metric_value(?array $data, string $metric, string $stat): mixed - { - return $data['metrics'][$metric]['values'][$stat] ?? null; + function metricValue(data, metric, stat) { + return data?.metrics?.[metric]?.values?.[stat] ?? null; } - function format_number(mixed $value): string - { - $value = round((float) $value, 2); - return rtrim(rtrim(number_format($value, 2, '.', ''), '0'), '.'); + function formatNumber(value) { + return Number(value).toFixed(2).replace(/\.?0+$/, ''); } - function format_value(mixed $value, string $suffix = ''): string - { - return $value === null ? 'n/a' : format_number($value) . $suffix; + function formatValue(value, suffix = '') { + return value === null ? 'n/a' : `${formatNumber(value)}${suffix}`; } - function delta(mixed $beforeValue, mixed $afterValue, string $suffix = ''): string - { - if ($beforeValue === null || $afterValue === null) { - return 'n/a'; - } + function delta(beforeValue, afterValue, suffix = '') { + if (beforeValue === null || afterValue === null) { + return 'n/a'; + } - $difference = round((float) $afterValue - (float) $beforeValue, 2); - return ($difference > 0 ? '+' : '') . format_number($difference) . $suffix; + const difference = Number((afterValue - beforeValue).toFixed(2)); + return `${difference > 0 ? '+' : ''}${formatNumber(difference)}${suffix}`; } - function row(string $label, mixed $beforeValue, mixed $afterValue, string $suffix = ''): string - { - return '| ' . $label . ' | ' . format_value($beforeValue, $suffix) . ' | ' . format_value($afterValue, $suffix) . ' | ' . delta($beforeValue, $afterValue, $suffix) . ' |'; + function row(label, beforeValue, afterValue, suffix = '') { + return `| ${label} | ${formatValue(beforeValue, suffix)} | ${formatValue(afterValue, suffix)} | ${delta(beforeValue, afterValue, suffix)} |`; } - function format_detail_value(mixed $value, string $suffix = ''): string - { - return $value === null ? 'n/a' : number_format((float) $value, 2, '.', '') . $suffix; + function detailValue(value, suffix = '') { + return value === null ? 'n/a' : `${Number(value).toFixed(2)}${suffix}`; } - function detail_row(array $after, string $label, string $metric, string $suffix = 'ms'): string - { - $values = $after['metrics'][$metric]['values'] ?? null; - if (!is_array($values)) { - return '| ' . $label . ' | n/a | n/a | n/a | n/a |'; - } + function detailRow(after, label, metric, suffix = 'ms') { + const values = after.metrics?.[metric]?.values; + if (!values) { + return `| ${label} | n/a | n/a | n/a | n/a |`; + } - return '| ' . $label - . ' | ' . format_detail_value($values['avg'] ?? null, $suffix) - . ' | ' . format_detail_value($values['p(90)'] ?? null, $suffix) - . ' | ' . format_detail_value($values['p(95)'] ?? null, $suffix) - . ' | ' . format_detail_value($values['max'] ?? null, $suffix) - . ' |'; + return `| ${label} | ${detailValue(values.avg ?? null, suffix)} | ${detailValue(values['p(90)'] ?? null, suffix)} | ${detailValue(values['p(95)'] ?? null, suffix)} | ${detailValue(values.max ?? null, suffix)} |`; } - $rows = [ - row('HTTP total p95', metric_value($before, 'http_req_duration', 'p(95)'), metric_value($after, 'http_req_duration', 'p(95)'), 'ms'), - row('API endpoints p95', metric_value($before, 'appwrite_api_duration', 'p(95)'), metric_value($after, 'appwrite_api_duration', 'p(95)'), 'ms'), - row('Database worker p95', metric_value($before, 'appwrite_worker_database_duration', 'p(95)'), metric_value($after, 'appwrite_worker_database_duration', 'p(95)'), 'ms'), - row('TablesDB worker p95', metric_value($before, 'appwrite_worker_tables_duration', 'p(95)'), metric_value($after, 'appwrite_worker_tables_duration', 'p(95)'), 'ms'), - row('Mail worker p95', metric_value($before, 'appwrite_worker_mails_duration', 'p(95)'), metric_value($after, 'appwrite_worker_mails_duration', 'p(95)'), 'ms'), - row('Messaging worker p95', metric_value($before, 'appwrite_worker_messaging_duration', 'p(95)'), metric_value($after, 'appwrite_worker_messaging_duration', 'p(95)'), 'ms'), + const before = readSummary('benchmark-before-summary.json', false); + const after = readSummary('benchmark-after-summary.json'); + const baseRef = markdownText(process.env.BENCHMARK_BASE_REF || 'base'); + const headRef = markdownText(process.env.BENCHMARK_HEAD_REF || 'head'); + + const rows = [ + row('HTTP total p95', metricValue(before, 'appwrite_http_duration', 'p(95)'), metricValue(after, 'appwrite_http_duration', 'p(95)'), 'ms'), + row('API endpoints p95', metricValue(before, 'appwrite_api_duration', 'p(95)'), metricValue(after, 'appwrite_api_duration', 'p(95)'), 'ms'), + row('Database worker p95', metricValue(before, 'appwrite_worker_database_duration', 'p(95)'), metricValue(after, 'appwrite_worker_database_duration', 'p(95)'), 'ms'), + row('TablesDB worker p95', metricValue(before, 'appwrite_worker_tables_duration', 'p(95)'), metricValue(after, 'appwrite_worker_tables_duration', 'p(95)'), 'ms'), + row('Mail worker p95', metricValue(before, 'appwrite_worker_mails_duration', 'p(95)'), metricValue(after, 'appwrite_worker_mails_duration', 'p(95)'), 'ms'), + row('Messaging worker p95', metricValue(before, 'appwrite_worker_messaging_duration', 'p(95)'), metricValue(after, 'appwrite_worker_messaging_duration', 'p(95)'), 'ms'), ]; - echo "\n"; - echo "## :sparkles: Benchmark results\n\n"; - echo "Comparing {$baseRef} (before) to {$headRef} (after).\n\n"; - if ($before === null) { - echo "> Before benchmark did not complete; showing current branch metrics only.\n\n"; + console.log(''); + console.log('## :sparkles: Benchmark results'); + console.log(); + console.log(`Comparing ${baseRef} (before) to ${headRef} (after).`); + console.log(); + if (before === null) { + console.log('> Before benchmark did not complete; showing current branch metrics only.'); + console.log(); } - echo "| Metric | Before | After | Delta |\n"; - echo "| --- | ---: | ---: | ---: |\n"; - echo implode("\n", $rows) . "\n\n"; - echo "
\n"; - echo "Current run details\n\n"; - echo "
\n\n"; - echo "| Scenario | Avg | P90 | P95 | Max |\n"; - echo "| --- | ---: | ---: | ---: | ---: |\n"; - echo detail_row($after, 'HTTP total', 'http_req_duration') . "\n"; - echo detail_row($after, 'API endpoints', 'appwrite_api_duration') . "\n"; - echo detail_row($after, 'Database worker schema jobs', 'appwrite_worker_database_duration') . "\n"; - echo detail_row($after, 'TablesDB worker schema jobs', 'appwrite_worker_tables_duration') . "\n"; - echo detail_row($after, 'Mail worker delivery', 'appwrite_worker_mails_duration') . "\n"; - echo detail_row($after, 'Messaging worker delivery', 'appwrite_worker_messaging_duration') . "\n\n"; - echo "
\n"; - PHP + console.log('| Metric | Before | After | Delta |'); + console.log('| --- | ---: | ---: | ---: |'); + console.log(rows.join('\n')); + console.log(); + console.log('
'); + console.log('Current run details'); + console.log(); + console.log('
'); + console.log(); + console.log('| Scenario | Avg | P90 | P95 | Max |'); + console.log('| --- | ---: | ---: | ---: | ---: |'); + console.log(detailRow(after, 'HTTP total', 'appwrite_http_duration')); + console.log(detailRow(after, 'API endpoints', 'appwrite_api_duration')); + console.log(detailRow(after, 'Database worker schema jobs', 'appwrite_worker_database_duration')); + console.log(detailRow(after, 'TablesDB worker schema jobs', 'appwrite_worker_tables_duration')); + console.log(detailRow(after, 'Mail worker delivery', 'appwrite_worker_mails_duration')); + console.log(detailRow(after, 'Messaging worker delivery', 'appwrite_worker_messaging_duration')); + console.log(); + console.log('
'); + NODE - name: Save results uses: actions/upload-artifact@v7 diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js new file mode 100644 index 0000000000..e852794e3b --- /dev/null +++ b/tests/benchmarks/http.js @@ -0,0 +1,1017 @@ +import http from 'k6/http'; +import { check, group, sleep } from 'k6'; +import encoding from 'k6/encoding'; +import { Counter, Trend } from 'k6/metrics'; + +const ENDPOINT = (__ENV.APPWRITE_ENDPOINT || 'http://localhost/v1').replace(/\/+$/, ''); +const MAILDEV_ENDPOINT = __ENV.APPWRITE_MAILDEV_ENDPOINT || 'http://localhost:9503/email'; +const CONSOLE_PROJECT = __ENV.APPWRITE_CONSOLE_PROJECT || 'console'; +const REGION = __ENV.APPWRITE_REGION || 'default'; +const REDIRECT_URL = __ENV.APPWRITE_BENCHMARK_REDIRECT_URL || 'http://localhost'; +const PASSWORD = __ENV.APPWRITE_BENCHMARK_PASSWORD || 'Password123!'; +const MAIL_TIMEOUT_MS = Number(__ENV.APPWRITE_MAIL_TIMEOUT_MS || 20000); +const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 60000); +const ITERATIONS = Number(__ENV.APPWRITE_BENCHMARK_ITERATIONS || 1); +const VUS = Number(__ENV.APPWRITE_BENCHMARK_VUS || 1); +const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || 'tests/benchmarks/http-summary.json'; +const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || SUMMARY_PATH; +const PREVIOUS_SUMMARY = loadPreviousSummary(); + +export const httpDuration = new Trend('appwrite_http_duration', true); +export const apiDuration = new Trend('appwrite_api_duration', true); +export const databaseWorkerDuration = new Trend('appwrite_worker_database_duration', true); +export const tablesWorkerDuration = new Trend('appwrite_worker_tables_duration', true); +export const mailsWorkerDuration = new Trend('appwrite_worker_mails_duration', true); +export const messagingWorkerDuration = new Trend('appwrite_worker_messaging_duration', true); +export const flowFailures = new Counter('appwrite_benchmark_flow_failures'); + +export const options = { + scenarios: { + curated_flows: { + executor: 'shared-iterations', + exec: 'curatedFlows', + vus: VUS, + iterations: ITERATIONS, + maxDuration: __ENV.APPWRITE_BENCHMARK_MAX_DURATION || '30m', + }, + }, + thresholds: { + http_req_failed: ['rate<0.05'], + appwrite_api_duration: ['p(95)<2000'], + appwrite_benchmark_flow_failures: ['count<1'], + }, +}; + +const API_SCOPES = [ + 'sessions.write', + 'users.read', + 'users.write', + 'teams.read', + 'teams.write', + 'databases.read', + 'databases.write', + 'collections.read', + 'collections.write', + 'tables.read', + 'tables.write', + 'attributes.read', + 'attributes.write', + 'columns.read', + 'columns.write', + 'indexes.read', + 'indexes.write', + 'documents.read', + 'documents.write', + 'rows.read', + 'rows.write', + 'files.read', + 'files.write', + 'buckets.read', + 'buckets.write', + 'functions.read', + 'functions.write', + 'sites.read', + 'sites.write', + 'log.read', + 'log.write', + 'execution.read', + 'execution.write', + 'locale.read', + 'avatars.read', + 'health.read', + 'providers.read', + 'providers.write', + 'messages.read', + 'messages.write', + 'topics.read', + 'topics.write', + 'subscribers.read', + 'subscribers.write', + 'targets.read', + 'targets.write', + 'rules.read', + 'rules.write', + 'migrations.read', + 'migrations.write', + 'vcs.read', + 'vcs.write', + 'assistant.read', + 'tokens.read', + 'tokens.write', + 'platforms.read', + 'platforms.write', +]; + +const BASE_PERMISSIONS = [ + 'read("any")', + 'create("any")', + 'update("any")', + 'delete("any")', +]; + +const ITEM_PERMISSIONS = [ + 'read("any")', + 'update("any")', + 'delete("any")', +]; + +export function setup() { + const runId = unique('run'); + const consoleEmail = __ENV.APPWRITE_ADMIN_EMAIL || `bench-admin-${runId}@example.com`; + const consolePassword = __ENV.APPWRITE_ADMIN_PASSWORD || PASSWORD; + + const consoleHeaders = { + 'Content-Type': 'application/json', + 'X-Appwrite-Project': CONSOLE_PROJECT, + }; + + const account = rawRequest('POST', '/account', { + userId: unique('admin'), + email: consoleEmail, + password: consolePassword, + name: 'Benchmark Admin', + }, consoleHeaders, 'setup.account.create'); + + if (![201, 409].includes(account.status)) { + failResponse(account, 'Unable to create or reuse the benchmark console account'); + } + + const session = rawRequest('POST', '/account/sessions/email', { + email: consoleEmail, + password: consolePassword, + }, consoleHeaders, 'setup.account.session'); + + assertStatus(session, [201], 'console session created'); + + const consoleSessionHeaders = { + ...consoleHeaders, + Cookie: cookieHeader(session), + }; + + const team = api('POST', '/teams', { + teamId: unique('team'), + name: `Benchmark Team ${runId}`, + }, consoleSessionHeaders, [201], 'setup.teams.create'); + + const teamId = team.json('$id'); + const project = api('POST', '/projects', { + projectId: unique('project'), + name: `Benchmark Project ${runId}`, + teamId, + region: REGION, + }, consoleSessionHeaders, [201], 'setup.projects.create'); + + const projectId = project.json('$id'); + const key = api('POST', `/projects/${projectId}/keys`, { + keyId: unique('key'), + name: 'Benchmark API key', + scopes: API_SCOPES, + }, consoleSessionHeaders, [201], 'setup.projects.keys.create'); + + const apiHeaders = { + 'Content-Type': 'application/json', + 'X-Appwrite-Project': projectId, + 'X-Appwrite-Key': key.json('secret'), + }; + + const platform = api('POST', '/project/platforms/web', { + platformId: unique('web'), + name: 'Benchmark web', + hostname: hostnameFromUrl(REDIRECT_URL), + }, apiHeaders, [201, 409], 'setup.project.platforms.web.create'); + + const smtp = rawRequest('PATCH', `/projects/${projectId}/smtp`, { + enabled: true, + senderName: 'Benchmark', + senderEmail: 'benchmark@appwrite.io', + replyTo: 'benchmark@appwrite.io', + host: __ENV.APPWRITE_SMTP_HOST || 'maildev', + port: Number(__ENV.APPWRITE_SMTP_PORT || 1025), + username: __ENV.APPWRITE_SMTP_USERNAME || 'user', + password: __ENV.APPWRITE_SMTP_PASSWORD || 'password', + ...(String(__ENV.APPWRITE_SMTP_SECURE || '') !== '' ? { secure: __ENV.APPWRITE_SMTP_SECURE } : {}), + }, consoleSessionHeaders, 'setup.projects.smtp.update'); + + if (smtp.status !== 200) { + console.warn(`Custom SMTP was not enabled (${smtp.status}). Mail worker timings may be unavailable.`); + } + + return { + runId, + teamId, + projectId, + consoleSessionHeaders, + apiHeaders, + platformStatus: platform.status, + }; +} + +export function curatedFlows(data) { + const ctx = { ...data }; + + try { + group('account and mail worker', () => accountFlow(ctx)); + group('databases documents flow', () => databasesFlow(ctx)); + group('tablesdb rows flow', () => tablesDbFlow(ctx)); + group('storage files and tokens flow', () => storageFlow(ctx)); + group('messaging worker flow', () => messagingFlow(ctx)); + group('functions and sites control-plane flow', () => computeFlow(ctx)); + group('health and queue probes', () => healthFlow(ctx)); + } catch (error) { + flowFailures.add(1); + throw error; + } +} + +export function teardown(data) { + if (data && data.projectId && data.consoleSessionHeaders) { + rawRequest('DELETE', `/projects/${data.projectId}`, null, data.consoleSessionHeaders, 'teardown.projects.delete'); + } + + if (data && data.teamId && data.consoleSessionHeaders) { + rawRequest('DELETE', `/teams/${data.teamId}`, null, data.consoleSessionHeaders, 'teardown.teams.delete'); + } +} + +function accountFlow(ctx) { + const userId = unique('user'); + const email = `bench-user-${unique('mail')}@example.com`; + const headers = projectHeaders(ctx.projectId); + + api('POST', '/account', { + userId, + email, + password: PASSWORD, + name: 'Benchmark User', + }, headers, [201], 'account.create'); + + const session = api('POST', '/account/sessions/email', { + email, + password: PASSWORD, + }, headers, [201], 'account.sessions.email.create'); + + const sessionHeaders = { + ...headers, + Cookie: cookieHeader(session), + }; + + ctx.userId = userId; + ctx.userEmail = email; + ctx.sessionHeaders = sessionHeaders; + + const jwt = api('POST', '/account/jwts', null, sessionHeaders, [201], 'account.jwts.create'); + ctx.jwtHeaders = { + ...headers, + 'X-Appwrite-JWT': jwt.json('jwt'), + }; + + api('GET', '/account', null, sessionHeaders, [200], 'account.get'); + api('GET', '/account/logs', null, sessionHeaders, [200], 'account.logs.list'); + api('PATCH', '/account/prefs', { prefs: { benchmark: true, runId: ctx.runId } }, sessionHeaders, [200], 'account.prefs.update'); + api('PATCH', '/account/name', { name: 'Benchmark User Updated' }, sessionHeaders, [200], 'account.name.update'); + api('PATCH', '/account/password', { password: `${PASSWORD}2`, oldPassword: PASSWORD }, sessionHeaders, [200], 'account.password.update'); + + const verificationStarted = Date.now(); + api('POST', '/account/verifications/email', { url: REDIRECT_URL }, sessionHeaders, [201], 'account.emailVerification.create'); + const verificationEmail = waitForEmail(email, (message) => { + return includes(message.subject, 'verify') + || includes(message.subject, 'verification') + || includes(message.html, 'verify') + || includes(message.html, 'verification') + || includes(message.text, 'verify') + || includes(message.text, 'verification'); + }, MAIL_TIMEOUT_MS); + mailsWorkerDuration.add(Date.now() - verificationStarted, { job: 'email_verification' }); + + const verification = extractQueryParams(verificationEmail); + if (verification.userId && verification.secret) { + api('PUT', '/account/verifications/email', { + userId: verification.userId, + secret: verification.secret, + }, sessionHeaders, [200], 'account.emailVerification.update'); + } + + const recoveryStarted = Date.now(); + api('POST', '/account/recovery', { email, url: REDIRECT_URL }, headers, [201], 'account.recovery.create'); + const recoveryEmail = waitForEmail(email, (message) => { + return includes(message.subject, 'recovery') + || includes(message.subject, 'recover') + || includes(message.subject, 'reset') + || includes(message.html, 'recovery') + || includes(message.html, 'recover') + || includes(message.html, 'reset') + || includes(message.text, 'recovery') + || includes(message.text, 'recover') + || includes(message.text, 'reset'); + }, MAIL_TIMEOUT_MS); + mailsWorkerDuration.add(Date.now() - recoveryStarted, { job: 'password_recovery' }); + + const recovery = extractQueryParams(recoveryEmail); + if (recovery.userId && recovery.secret) { + api('DELETE', '/account/sessions/current', null, sessionHeaders, [204], 'account.sessions.current.delete'); + + api('PUT', '/account/recovery', { + userId: recovery.userId, + secret: recovery.secret, + password: `${PASSWORD}3`, + }, headers, [200], 'account.recovery.update'); + + const recoveredSession = api('POST', '/account/sessions/email', { + email, + password: `${PASSWORD}3`, + }, headers, [201], 'account.sessions.email.recovered'); + + ctx.sessionHeaders = { + ...headers, + Cookie: cookieHeader(recoveredSession), + }; + + const recoveredJwt = api('POST', '/account/jwts', null, ctx.sessionHeaders, [201], 'account.jwts.recovered'); + ctx.jwtHeaders = { + ...headers, + 'X-Appwrite-JWT': recoveredJwt.json('jwt'), + }; + } +} + +function databasesFlow(ctx) { + const databaseId = unique('db'); + const collectionId = unique('col'); + const documentId = unique('doc'); + const indexKey = unique('idx'); + + api('POST', '/databases', { databaseId, name: 'Benchmark DB' }, ctx.apiHeaders, [201], 'databases.create'); + api('POST', `/databases/${databaseId}/collections`, { + collectionId, + name: 'Benchmark Collection', + permissions: BASE_PERMISSIONS, + documentSecurity: false, + }, ctx.apiHeaders, [201], 'databases.collections.create'); + + const attributes = [ + ['string', 'title', { size: 128 }], + ['integer', 'count', { min: 0, max: 100000 }], + ['email', 'email', {}], + ['boolean', 'active', {}], + ['datetime', 'publishedAt', {}], + ['float', 'score', { min: 0, max: 1000 }], + ['url', 'url', {}], + ['ip', 'ip', {}], + ]; + + for (const [type, key, extra] of attributes) { + const started = Date.now(); + api('POST', `/databases/${databaseId}/collections/${collectionId}/attributes/${type}`, { + key, + required: false, + array: false, + ...extra, + }, ctx.apiHeaders, [202], `databases.attributes.${type}.create`); + waitForStatus(`/databases/${databaseId}/collections/${collectionId}/attributes/${key}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); + databaseWorkerDuration.add(Date.now() - started, { job: `attribute_${type}` }); + } + + const indexStarted = Date.now(); + api('POST', `/databases/${databaseId}/collections/${collectionId}/indexes`, { + key: indexKey, + type: 'key', + attributes: ['title'], + orders: ['asc'], + }, ctx.apiHeaders, [202], 'databases.indexes.create'); + waitForStatus(`/databases/${databaseId}/collections/${collectionId}/indexes/${indexKey}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); + databaseWorkerDuration.add(Date.now() - indexStarted, { job: 'index' }); + + api('POST', `/databases/${databaseId}/collections/${collectionId}/documents`, { + documentId, + data: documentPayload(), + permissions: ITEM_PERMISSIONS, + }, ctx.apiHeaders, [201], 'databases.documents.create'); + api('GET', `/databases/${databaseId}/collections/${collectionId}/documents`, null, ctx.apiHeaders, [200], 'databases.documents.list'); + api('GET', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}`, null, ctx.apiHeaders, [200], 'databases.documents.get'); + api('PATCH', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}`, { + data: { title: 'Benchmark Document Updated' }, + }, ctx.apiHeaders, [200], 'databases.documents.update'); + api('PATCH', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}/count/increment`, { + value: 1, + }, ctx.apiHeaders, [200], 'databases.documents.increment'); + api('PATCH', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}/count/decrement`, { + value: 1, + }, ctx.apiHeaders, [200], 'databases.documents.decrement'); + api('DELETE', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}`, null, ctx.apiHeaders, [204], 'databases.documents.delete'); + api('DELETE', `/databases/${databaseId}`, null, ctx.apiHeaders, [204], 'databases.delete'); +} + +function tablesDbFlow(ctx) { + requireSession(ctx, 'tablesDbFlow'); + + const databaseId = unique('tdb'); + const tableId = unique('tbl'); + const rowId = unique('row'); + const indexKey = unique('tidx'); + + api('POST', '/tablesdb', { databaseId, name: 'Benchmark TablesDB' }, ctx.apiHeaders, [201], 'tablesdb.create'); + api('POST', `/tablesdb/${databaseId}/tables`, { + tableId, + name: 'Benchmark Table', + permissions: BASE_PERMISSIONS, + rowSecurity: false, + }, ctx.apiHeaders, [201], 'tablesdb.tables.create'); + + const columns = [ + ['string', 'title', { size: 128 }], + ['integer', 'count', { min: 0, max: 100000 }], + ['email', 'email', {}], + ['boolean', 'active', {}], + ]; + + for (const [type, key, extra] of columns) { + const started = Date.now(); + api('POST', `/tablesdb/${databaseId}/tables/${tableId}/columns/${type}`, { + key, + required: false, + array: false, + ...extra, + }, ctx.apiHeaders, [202], `tablesdb.columns.${type}.create`); + waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/columns/${key}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); + tablesWorkerDuration.add(Date.now() - started, { job: `column_${type}` }); + } + + const indexStarted = Date.now(); + api('POST', `/tablesdb/${databaseId}/tables/${tableId}/indexes`, { + key: indexKey, + type: 'key', + columns: ['title'], + orders: ['asc'], + }, ctx.apiHeaders, [202], 'tablesdb.indexes.create'); + waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/indexes/${indexKey}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); + tablesWorkerDuration.add(Date.now() - indexStarted, { job: 'index' }); + + api('POST', `/tablesdb/${databaseId}/tables/${tableId}/rows`, { + rowId, + data: tablePayload(), + permissions: ITEM_PERMISSIONS, + }, ctx.sessionHeaders, [201], 'tablesdb.rows.create'); + api('GET', `/tablesdb/${databaseId}/tables/${tableId}/rows`, null, ctx.sessionHeaders, [200], 'tablesdb.rows.list'); + api('GET', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, null, ctx.sessionHeaders, [200], 'tablesdb.rows.get'); + api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, { + data: { title: 'Benchmark Row Updated' }, + }, ctx.sessionHeaders, [200], 'tablesdb.rows.update'); + api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}/count/increment`, { + value: 1, + }, ctx.sessionHeaders, [200], 'tablesdb.rows.increment'); + api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}/count/decrement`, { + value: 1, + }, ctx.sessionHeaders, [200], 'tablesdb.rows.decrement'); + api('DELETE', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, null, ctx.sessionHeaders, [204], 'tablesdb.rows.delete'); + api('DELETE', `/tablesdb/${databaseId}`, null, ctx.apiHeaders, [204], 'tablesdb.delete'); +} + +function storageFlow(ctx) { + requireSession(ctx, 'storageFlow'); + + const bucketId = unique('bucket'); + const fileId = unique('file'); + + api('POST', '/storage/buckets', { + bucketId, + name: 'Benchmark Bucket', + permissions: BASE_PERMISSIONS, + fileSecurity: false, + enabled: true, + maximumFileSize: 30000000, + allowedFileExtensions: [], + compression: 'none', + encryption: false, + antivirus: false, + }, ctx.apiHeaders, [201], 'storage.buckets.create'); + + const multipartHeaders = { ...ctx.sessionHeaders }; + delete multipartHeaders['Content-Type']; + + const upload = http.post(`${ENDPOINT}/storage/buckets/${bucketId}/files`, { + fileId, + file: http.file(onePixelPng(), 'benchmark.png', 'image/png'), + ...flattenMultipartArray('permissions', ITEM_PERMISSIONS), + }, { + headers: multipartHeaders, + tags: { name: 'storage.files.create' }, + }); + + httpDuration.add(upload.timings.duration, { name: 'storage.files.create' }); + apiDuration.add(upload.timings.duration, { name: 'storage.files.create' }); + assertStatus(upload, [201], 'storage file created'); + + api('GET', `/storage/buckets/${bucketId}/files`, null, ctx.sessionHeaders, [200], 'storage.files.list'); + api('GET', `/storage/buckets/${bucketId}/files/${fileId}`, null, ctx.sessionHeaders, [200], 'storage.files.get'); + api('GET', `/storage/buckets/${bucketId}/files/${fileId}/view`, null, ctx.sessionHeaders, [200], 'storage.files.view'); + api('GET', `/storage/buckets/${bucketId}/files/${fileId}/download`, null, ctx.sessionHeaders, [200], 'storage.files.download'); + api('GET', `/storage/buckets/${bucketId}/files/${fileId}/preview`, null, ctx.sessionHeaders, [200], 'storage.files.preview'); + api('PUT', `/storage/buckets/${bucketId}/files/${fileId}`, { + name: 'benchmark-renamed.png', + permissions: ITEM_PERMISSIONS, + }, ctx.sessionHeaders, [200], 'storage.files.update'); + + const token = api('POST', `/tokens/buckets/${bucketId}/files/${fileId}`, {}, ctx.apiHeaders, [201], 'tokens.files.create'); + api('GET', `/tokens/buckets/${bucketId}/files/${fileId}`, null, ctx.apiHeaders, [200], 'tokens.files.list'); + api('GET', `/tokens/${token.json('$id')}`, null, ctx.apiHeaders, [200], 'tokens.get'); + api('PATCH', `/tokens/${token.json('$id')}`, { expire: null }, ctx.apiHeaders, [200], 'tokens.update'); + api('DELETE', `/tokens/${token.json('$id')}`, null, ctx.apiHeaders, [204], 'tokens.delete'); + + api('DELETE', `/storage/buckets/${bucketId}/files/${fileId}`, null, ctx.sessionHeaders, [204], 'storage.files.delete'); + api('DELETE', `/storage/buckets/${bucketId}`, null, ctx.apiHeaders, [204], 'storage.buckets.delete'); +} + +function messagingFlow(ctx) { + requireSession(ctx, 'messagingFlow'); + if (!ctx.userId || !ctx.userEmail) { + throw new Error('accountFlow must run before messagingFlow'); + } + + const providerId = unique('smtp'); + let targetId = unique('target'); + const topicId = unique('topic'); + const subscriberId = unique('sub'); + const messageId = unique('msg'); + + api('POST', '/messaging/providers/smtp', { + providerId, + name: 'Benchmark SMTP', + host: __ENV.APPWRITE_SMTP_HOST || 'maildev', + port: Number(__ENV.APPWRITE_SMTP_PORT || 1025), + username: __ENV.APPWRITE_SMTP_USERNAME || 'user', + password: __ENV.APPWRITE_SMTP_PASSWORD || 'password', + encryption: __ENV.APPWRITE_SMTP_ENCRYPTION || 'none', + autoTLS: false, + fromName: 'Benchmark', + fromEmail: 'benchmark@appwrite.io', + replyToName: 'Benchmark', + replyToEmail: 'benchmark@appwrite.io', + enabled: true, + }, ctx.apiHeaders, [201], 'messaging.providers.smtp.create'); + + const targets = api('GET', `/users/${ctx.userId}/targets`, null, ctx.apiHeaders, [200], 'users.targets.list'); + const existingTarget = (targets.json('targets') || []).find((target) => { + return target.providerType === 'email' && target.identifier === ctx.userEmail; + }); + + if (existingTarget) { + targetId = existingTarget.$id; + api('PATCH', `/users/${ctx.userId}/targets/${targetId}`, { + providerId, + name: 'Benchmark email target', + }, ctx.apiHeaders, [200], 'users.targets.update'); + } else { + api('POST', `/users/${ctx.userId}/targets`, { + targetId, + providerType: 'email', + identifier: ctx.userEmail, + providerId, + name: 'Benchmark email target', + }, ctx.apiHeaders, [201], 'users.targets.create'); + } + + api('POST', '/messaging/topics', { + topicId, + name: 'Benchmark Topic', + subscribe: ['users'], + }, ctx.apiHeaders, [201], 'messaging.topics.create'); + + api('POST', `/messaging/topics/${topicId}/subscribers`, { + subscriberId, + targetId, + }, ctx.sessionHeaders, [201], 'messaging.subscribers.create'); + + const started = Date.now(); + api('POST', '/messaging/messages/email', { + messageId, + subject: `Benchmark message ${ctx.runId}`, + content: `Benchmark messaging worker probe ${ctx.runId}`, + targets: [targetId], + draft: false, + html: false, + }, ctx.apiHeaders, [201], 'messaging.messages.email.create'); + + waitForMessage(messageId, ctx.apiHeaders, WORKER_TIMEOUT_MS); + waitForEmail(ctx.userEmail, (message) => includes(message.subject, `Benchmark message ${ctx.runId}`), MAIL_TIMEOUT_MS, true); + messagingWorkerDuration.add(Date.now() - started, { job: 'email_message' }); + + api('GET', '/messaging/messages', null, ctx.apiHeaders, [200], 'messaging.messages.list'); + api('GET', `/messaging/messages/${messageId}/logs`, null, ctx.apiHeaders, [200], 'messaging.messages.logs.list'); + api('GET', `/messaging/messages/${messageId}/targets`, null, ctx.apiHeaders, [200], 'messaging.messages.targets.list'); + api('GET', `/messaging/providers/${providerId}/logs`, null, ctx.apiHeaders, [200], 'messaging.providers.logs.list'); + api('GET', `/messaging/topics/${topicId}/logs`, null, ctx.apiHeaders, [200], 'messaging.topics.logs.list'); + api('GET', `/messaging/subscribers/${subscriberId}/logs`, null, ctx.apiHeaders, [200], 'messaging.subscribers.logs.list'); + api('DELETE', `/messaging/topics/${topicId}/subscribers/${subscriberId}`, null, ctx.sessionHeaders, [204], 'messaging.subscribers.delete'); + api('DELETE', `/messaging/topics/${topicId}`, null, ctx.apiHeaders, [204], 'messaging.topics.delete'); + api('DELETE', `/messaging/messages/${messageId}`, null, ctx.apiHeaders, [204], 'messaging.messages.delete'); + api('DELETE', `/messaging/providers/${providerId}`, null, ctx.apiHeaders, [204], 'messaging.providers.delete'); +} + +function computeFlow(ctx) { + requireSession(ctx, 'computeFlow'); + + const functionId = unique('fn'); + let functionVariableId; + const siteId = unique('site'); + let siteVariableId; + + api('POST', '/functions', { + functionId, + name: 'Benchmark Function', + runtime: __ENV.APPWRITE_BENCHMARK_RUNTIME || 'node-22', + execute: ['any'], + events: [], + schedule: '', + timeout: 15, + enabled: true, + logging: true, + entrypoint: 'index.js', + commands: 'npm install', + scopes: ['users.read'], + }, ctx.apiHeaders, [201], 'functions.create'); + api('GET', '/functions/runtimes', null, ctx.sessionHeaders, [200], 'functions.runtimes.list'); + api('GET', '/functions/specifications', null, ctx.apiHeaders, [200], 'functions.specifications.list'); + const functionVariable = api('POST', `/functions/${functionId}/variables`, { + key: 'BENCHMARK', + value: 'true', + secret: false, + }, ctx.apiHeaders, [201], 'functions.variables.create'); + functionVariableId = functionVariable.json('$id'); + + api('PUT', `/functions/${functionId}/variables/${functionVariableId}`, { + key: 'BENCHMARK', + value: 'updated', + secret: false, + }, ctx.apiHeaders, [200], 'functions.variables.update'); + api('GET', `/functions/${functionId}/variables/${functionVariableId}`, null, ctx.apiHeaders, [200], 'functions.variables.get'); + api('DELETE', `/functions/${functionId}/variables/${functionVariableId}`, null, ctx.apiHeaders, [204], 'functions.variables.delete'); + api('DELETE', `/functions/${functionId}`, null, ctx.apiHeaders, [204], 'functions.delete'); + + api('POST', '/sites', { + siteId, + name: 'Benchmark Site', + framework: 'other', + adapter: 'static', + buildRuntime: __ENV.APPWRITE_BENCHMARK_RUNTIME || 'node-22', + buildCommand: '', + outputDirectory: '.', + installCommand: '', + fallbackFile: 'index.html', + providerRootDirectory: '.', + specification: '', + }, ctx.apiHeaders, [201], 'sites.create'); + api('GET', '/sites/frameworks', null, ctx.sessionHeaders, [200], 'sites.frameworks.list'); + api('GET', '/sites/specifications', null, ctx.apiHeaders, [200], 'sites.specifications.list'); + const siteVariable = api('POST', `/sites/${siteId}/variables`, { + key: 'BENCHMARK', + value: 'true', + secret: false, + }, ctx.apiHeaders, [201], 'sites.variables.create'); + siteVariableId = siteVariable.json('$id'); + + api('PUT', `/sites/${siteId}/variables/${siteVariableId}`, { + key: 'BENCHMARK', + value: 'updated', + secret: false, + }, ctx.apiHeaders, [200], 'sites.variables.update'); + api('GET', `/sites/${siteId}/variables/${siteVariableId}`, null, ctx.apiHeaders, [200], 'sites.variables.get'); + api('DELETE', `/sites/${siteId}/variables/${siteVariableId}`, null, ctx.apiHeaders, [204], 'sites.variables.delete'); + api('DELETE', `/sites/${siteId}`, null, ctx.apiHeaders, [204], 'sites.delete'); +} + +function healthFlow(ctx) { + const probes = [ + '/health', + '/health/db', + '/health/cache', + '/health/pubsub', + '/health/storage', + '/health/storage/local', + '/health/time', + '/health/queue/databases', + '/health/queue/mails', + '/health/queue/messaging', + '/health/queue/functions', + '/health/queue/builds', + '/health/queue/deletes', + '/health/queue/webhooks', + '/health/queue/stats-resources', + '/health/queue/stats-usage', + '/health/queue/failed/v1-mails', + ]; + + for (const path of probes) { + api('GET', path, null, ctx.apiHeaders, [200], `health${path.replace(/\//g, '.')}`); + } +} + +function api(method, path, body, headers, expected, name) { + const response = rawRequest(method, path, body, headers, name); + apiDuration.add(response.timings.duration, { name }); + assertStatus(response, expected, name); + return response; +} + +function rawRequest(method, path, body, headers, name) { + const params = { + headers, + tags: { name }, + }; + const payload = body === null || body === undefined ? null : JSON.stringify(body); + const response = http.request(method, `${ENDPOINT}${path}`, payload, params); + httpDuration.add(response.timings.duration, { name }); + + return response; +} + +function waitForStatus(path, headers, wantedStatus, timeoutMs) { + const started = Date.now(); + + while (Date.now() - started < timeoutMs) { + const response = rawRequest('GET', path, null, headers, `wait${path}`); + if (response.status === 200) { + const status = response.json('status'); + if (status === wantedStatus) { + return response; + } + if (status === 'failed') { + throw new Error(`${path} failed while waiting for ${wantedStatus}`); + } + } + sleep(0.5); + } + + throw new Error(`Timed out waiting for ${path} to become ${wantedStatus}`); +} + +function waitForMessage(messageId, headers, timeoutMs) { + const started = Date.now(); + + while (Date.now() - started < timeoutMs) { + const response = rawRequest('GET', `/messaging/messages/${messageId}`, null, headers, 'messaging.messages.poll'); + const status = response.status === 200 ? response.json('status') : null; + + if (['sent', 'failed'].includes(status)) { + if (status === 'failed') { + throw new Error(`Messaging worker marked message ${messageId} as failed`); + } + return response; + } + + sleep(0.5); + } + + throw new Error(`Timed out waiting for messaging worker to send message ${messageId}`); +} + +function waitForEmail(address, predicate, timeoutMs, allowMissingRecipient = false) { + const started = Date.now(); + + while (Date.now() - started < timeoutMs) { + const response = http.get(MAILDEV_ENDPOINT, { tags: { name: 'maildev.email.list' } }); + if (response.status === 200) { + const emails = response.json(); + for (let i = emails.length - 1; i >= 0; i--) { + const message = emails[i]; + if ((emailMatches(message, address) || (allowMissingRecipient && emailRecipientMissing(message))) && predicate(message)) { + return message; + } + } + } + sleep(0.5); + } + + throw new Error(`Timed out waiting for email to ${address}`); +} + +function emailMatches(message, address) { + const recipients = message.to || []; + return recipients.some((recipient) => recipient.address === address); +} + +function emailRecipientMissing(message) { + const recipients = message.to || []; + return recipients.length === 0 || recipients.every((recipient) => !recipient.address); +} + +function extractQueryParams(message) { + const content = `${message.html || ''}\n${message.text || ''}`; + const links = []; + const hrefPattern = /href="([^"]+)"/g; + let hrefMatch = hrefPattern.exec(content); + + while (hrefMatch !== null) { + links.push(hrefMatch[1]); + hrefMatch = hrefPattern.exec(content); + } + + if (links.length === 0) { + links.push(content); + } + + for (const link of links) { + const queryStart = link.indexOf('?'); + if (queryStart === -1) { + continue; + } + + const query = link.slice(queryStart + 1).split('#')[0].replace(/&/g, '&'); + const params = {}; + + for (const pair of query.split('&')) { + const [key, value] = pair.split('='); + params[decodeURIComponent(key)] = decodeURIComponent(value || ''); + } + + if (params.userId && params.secret) { + return params; + } + } + + return {}; +} + +function assertStatus(response, expected, name) { + const ok = check(response, { + [`${name} status ${expected.join('|')}`]: (r) => expected.includes(r.status), + }); + + if (!ok) { + failResponse(response, `${name} returned an unexpected status`); + } +} + +function failResponse(response, message) { + throw new Error(`${message}. Status: ${response.status}. Body: ${response.body}`); +} + +function cookieHeader(response) { + return response.headers['Set-Cookie'] || response.headers['set-cookie'] || ''; +} + +function projectHeaders(projectId) { + return { + 'Content-Type': 'application/json', + 'X-Appwrite-Project': projectId, + }; +} + +function requireSession(ctx, flow) { + if (!ctx.sessionHeaders || typeof ctx.sessionHeaders !== 'object') { + throw new Error(`accountFlow must run before ${flow}`); + } +} + +function documentPayload() { + return { + title: 'Benchmark Document', + count: 1, + email: 'document@example.com', + active: true, + publishedAt: new Date().toISOString(), + score: 10.5, + url: 'https://appwrite.io', + ip: '127.0.0.1', + }; +} + +function tablePayload() { + return { + title: 'Benchmark Row', + count: 1, + email: 'row@example.com', + active: true, + }; +} + +function onePixelPng() { + return encoding.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=', 'std', 'b'); +} + +function flattenMultipartArray(key, values) { + const output = {}; + values.forEach((value, index) => { + output[`${key}[${index}]`] = value; + }); + return output; +} + +function unique(prefix) { + return `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}` + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .slice(0, 36); +} + +function includes(value, needle) { + return String(value || '').toLowerCase().includes(String(needle).toLowerCase()); +} + +function hostnameFromUrl(value) { + return value.replace(/^https?:\/\//, '').split('/')[0].split(':')[0]; +} + +export function handleSummary(data) { + const lines = [ + 'Appwrite curated benchmark review', + '', + 'Before/after comparison', + '', + comparisonTable(PREVIOUS_SUMMARY, data), + '', + 'Current run details', + '', + detailsTable(data), + '', + ]; + + return { + stdout: `${lines.join('\n')}\n`, + [SUMMARY_PATH]: JSON.stringify(data, null, 2), + }; +} + +function detailsTable(data) { + return [ + '| Scenario | Avg | P90 | P95 | Max |', + '| --- | ---: | ---: | ---: | ---: |', + detailRow(data, 'HTTP total', 'appwrite_http_duration'), + detailRow(data, 'API endpoints', 'appwrite_api_duration'), + detailRow(data, 'Database worker schema jobs', 'appwrite_worker_database_duration'), + detailRow(data, 'TablesDB worker schema jobs', 'appwrite_worker_tables_duration'), + detailRow(data, 'Mail worker delivery', 'appwrite_worker_mails_duration'), + detailRow(data, 'Messaging worker delivery', 'appwrite_worker_messaging_duration'), + ].join('\n'); +} + +function detailRow(data, label, metric, unit = 'ms') { + const values = data.metrics[metric] && data.metrics[metric].values; + if (!values || values.count === 0) { + return `| ${label} | n/a | n/a | n/a | n/a |`; + } + + return `| ${label} | ${formatDetailValue(values.avg, unit)} | ${formatDetailValue(values['p(90)'], unit)} | ${formatDetailValue(values['p(95)'], unit)} | ${formatDetailValue(values.max, unit)} |`; +} + +function loadPreviousSummary() { + try { + return JSON.parse(open(PREVIOUS_SUMMARY_PATH)); + } catch (error) { + return null; + } +} + +function comparisonTable(before, after) { + const rows = [ + ['HTTP total p95', trendMetric(before, 'appwrite_http_duration', 'p(95)'), trendMetric(after, 'appwrite_http_duration', 'p(95)'), 'ms'], + ['API endpoints p95', trendMetric(before, 'appwrite_api_duration', 'p(95)'), trendMetric(after, 'appwrite_api_duration', 'p(95)'), 'ms'], + ['Database worker p95', trendMetric(before, 'appwrite_worker_database_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_database_duration', 'p(95)'), 'ms'], + ['TablesDB worker p95', trendMetric(before, 'appwrite_worker_tables_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_tables_duration', 'p(95)'), 'ms'], + ['Mail worker p95', trendMetric(before, 'appwrite_worker_mails_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_mails_duration', 'p(95)'), 'ms'], + ['Messaging worker p95', trendMetric(before, 'appwrite_worker_messaging_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_messaging_duration', 'p(95)'), 'ms'], + ]; + + return [ + '| Metric | Before | After | Delta |', + '| --- | ---: | ---: | ---: |', + ...rows.map(([label, beforeValue, afterValue, unit]) => { + return `| ${label} | ${formatValue(beforeValue, unit)} | ${formatValue(afterValue, unit)} | ${formatDelta(beforeValue, afterValue, unit)} |`; + }), + ].join('\n'); +} + +function trendMetric(data, metric, stat) { + return data && data.metrics[metric] && data.metrics[metric].values + ? data.metrics[metric].values[stat] + : null; +} + +function formatValue(value, unit) { + if (value === null || value === undefined || Number.isNaN(value)) { + return 'n/a'; + } + + return `${round(value)}${unit}`; +} + +function formatDetailValue(value, unit) { + if (value === null || value === undefined || Number.isNaN(value)) { + return 'n/a'; + } + + return `${Number(value).toFixed(2)}${unit}`; +} + +function formatDelta(before, after, unit) { + if (before === null || before === undefined || after === null || after === undefined || Number.isNaN(before) || Number.isNaN(after)) { + return 'n/a'; + } + + const delta = round(after - before); + const sign = delta > 0 ? '+' : ''; + return `${sign}${delta}${unit}`; +} + +function round(value) { + return Math.round((value || 0) * 100) / 100; +} diff --git a/tests/benchmarks/http.php b/tests/benchmarks/http.php deleted file mode 100644 index 9bddf57327..0000000000 --- a/tests/benchmarks/http.php +++ /dev/null @@ -1,1331 +0,0 @@ -jsonParsed) { - $this->json = json_decode($this->body, true); - $this->jsonParsed = true; - } - - if ($key === null) { - return $this->json; - } - - return is_array($this->json) ? ($this->json[$key] ?? null) : null; - } - - public function header(string $name): string - { - $key = strtolower($name); - return isset($this->headers[$key]) ? implode(', ', $this->headers[$key]) : ''; - } - - public function cookieHeader(): string - { - $cookies = []; - - foreach ($this->headers['set-cookie'] ?? [] as $cookie) { - $cookies[] = explode(';', $cookie, 2)[0]; - } - - return implode('; ', $cookies); - } -} - -final class BenchmarkMetrics -{ - private array $trends = []; - private array $counters = [ - 'appwrite_benchmark_flow_failures' => 0, - ]; - private int $checksPassed = 0; - private int $checksFailed = 0; - - public function addTrend(string $name, float $value): void - { - $this->trends[$name] ??= []; - $this->trends[$name][] = $value; - } - - public function addCounter(string $name, int $value = 1): void - { - $this->counters[$name] ??= 0; - $this->counters[$name] += $value; - } - - public function addCheck(bool $passed): void - { - if ($passed) { - $this->checksPassed++; - return; - } - - $this->checksFailed++; - } - - public function summary(): array - { - $metrics = []; - - foreach ($this->trends as $name => $values) { - $metrics[$name] = [ - 'type' => 'trend', - 'contains' => 'time', - 'values' => $this->trendValues($values), - ]; - } - - foreach ($this->counters as $name => $count) { - $metrics[$name] = [ - 'type' => 'counter', - 'contains' => 'default', - 'values' => [ - 'count' => $count, - ], - ]; - } - - $totalChecks = $this->checksPassed + $this->checksFailed; - $metrics['checks'] = [ - 'type' => 'rate', - 'contains' => 'default', - 'values' => [ - 'rate' => $totalChecks > 0 ? $this->checksPassed / $totalChecks : 1, - 'passes' => $this->checksPassed, - 'fails' => $this->checksFailed, - ], - ]; - - return ['metrics' => $metrics]; - } - - public function failedChecks(): int - { - return $this->checksFailed; - } - - public function flowFailures(): int - { - return $this->counters['appwrite_benchmark_flow_failures'] ?? 0; - } - - private function trendValues(array $values): array - { - sort($values, SORT_NUMERIC); - $count = count($values); - - if ($count === 0) { - return [ - 'count' => 0, - 'min' => null, - 'avg' => null, - 'med' => null, - 'max' => null, - 'p(90)' => null, - 'p(95)' => null, - ]; - } - - return [ - 'count' => $count, - 'min' => $values[0], - 'avg' => array_sum($values) / $count, - 'med' => $this->percentile($values, 50), - 'max' => $values[$count - 1], - 'p(90)' => $this->percentile($values, 90), - 'p(95)' => $this->percentile($values, 95), - ]; - } - - private function percentile(array $sortedValues, int $percentile): float - { - $count = count($sortedValues); - - if ($count === 1) { - return (float) $sortedValues[0]; - } - - $rank = ($percentile / 100) * ($count - 1); - $lower = (int) floor($rank); - $upper = (int) ceil($rank); - - if ($lower === $upper) { - return (float) $sortedValues[$lower]; - } - - $weight = $rank - $lower; - return (float) ($sortedValues[$lower] + (($sortedValues[$upper] - $sortedValues[$lower]) * $weight)); - } -} - -final class HttpBenchmark -{ - private const API_SCOPES = [ - 'sessions.write', - 'users.read', - 'users.write', - 'teams.read', - 'teams.write', - 'databases.read', - 'databases.write', - 'collections.read', - 'collections.write', - 'tables.read', - 'tables.write', - 'attributes.read', - 'attributes.write', - 'columns.read', - 'columns.write', - 'indexes.read', - 'indexes.write', - 'documents.read', - 'documents.write', - 'rows.read', - 'rows.write', - 'files.read', - 'files.write', - 'buckets.read', - 'buckets.write', - 'functions.read', - 'functions.write', - 'sites.read', - 'sites.write', - 'log.read', - 'log.write', - 'execution.read', - 'execution.write', - 'locale.read', - 'avatars.read', - 'health.read', - 'providers.read', - 'providers.write', - 'messages.read', - 'messages.write', - 'topics.read', - 'topics.write', - 'subscribers.read', - 'subscribers.write', - 'targets.read', - 'targets.write', - 'rules.read', - 'rules.write', - 'migrations.read', - 'migrations.write', - 'vcs.read', - 'vcs.write', - 'assistant.read', - 'tokens.read', - 'tokens.write', - 'platforms.read', - 'platforms.write', - ]; - - private const BASE_PERMISSIONS = [ - 'read("any")', - 'create("any")', - 'update("any")', - 'delete("any")', - ]; - - private const ITEM_PERMISSIONS = [ - 'read("any")', - 'update("any")', - 'delete("any")', - ]; - - private const PNG_1X1 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII='; - - private BenchmarkMetrics $metrics; - private string $endpoint; - private string $maildevEndpoint; - private string $consoleProject; - private string $region; - private string $redirectUrl; - private string $password; - private int $mailTimeoutMs; - private int $workerTimeoutMs; - private int $iterations; - private int $runs; - private string $summaryPath; - private ?array $previousSummary; - - public function __construct() - { - $this->metrics = new BenchmarkMetrics(); - $this->endpoint = rtrim($this->env('APPWRITE_ENDPOINT', 'http://localhost/v1'), '/'); - $this->maildevEndpoint = $this->env('APPWRITE_MAILDEV_ENDPOINT', 'http://localhost:9503/email'); - $this->consoleProject = $this->env('APPWRITE_CONSOLE_PROJECT', 'console'); - $this->region = $this->env('APPWRITE_REGION', 'default'); - $this->redirectUrl = $this->env('APPWRITE_BENCHMARK_REDIRECT_URL', 'http://localhost'); - $this->password = $this->env('APPWRITE_BENCHMARK_PASSWORD', 'Password123!'); - $this->mailTimeoutMs = (int) $this->env('APPWRITE_MAIL_TIMEOUT_MS', '20000'); - $this->workerTimeoutMs = (int) $this->env('APPWRITE_WORKER_TIMEOUT_MS', '60000'); - $this->iterations = max(1, (int) $this->env('APPWRITE_BENCHMARK_ITERATIONS', '1')); - $this->runs = max(1, (int) $this->env('APPWRITE_BENCHMARK_RUNS', $this->env('APPWRITE_BENCHMARK_VUS', '1'))); - $this->summaryPath = $this->env('APPWRITE_BENCHMARK_SUMMARY_PATH', 'tests/benchmarks/http-summary.json'); - $this->previousSummary = $this->loadPreviousSummary($this->env('APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH', $this->summaryPath)); - } - - public function run(): int - { - $context = null; - $exitCode = 0; - - try { - $context = $this->setup(); - - for ($i = 0; $i < $this->iterations * $this->runs; $i++) { - try { - $this->curatedFlows($context); - } catch (Throwable $error) { - $exitCode = 1; - fwrite(STDERR, 'Iteration ' . ($i + 1) . ' failed: ' . $error->getMessage() . PHP_EOL); - } - } - } catch (Throwable $error) { - $exitCode = 1; - fwrite(STDERR, $error->getMessage() . PHP_EOL); - } finally { - if (is_array($context)) { - try { - $this->teardown($context); - } catch (Throwable $error) { - $exitCode = 1; - fwrite(STDERR, 'Teardown failed: ' . $error->getMessage() . PHP_EOL); - } - } - - $summary = $this->metrics->summary(); - echo $this->renderSummary($summary); - try { - $this->writeSummary($summary); - } catch (Throwable $error) { - $exitCode = 1; - fwrite(STDERR, $error->getMessage() . PHP_EOL); - } - } - - if ($this->metrics->failedChecks() > 0 || $this->metrics->flowFailures() > 0) { - $exitCode = 1; - } - - return $exitCode; - } - - private function setup(): array - { - $runId = $this->unique('run'); - $consoleEmail = $this->env('APPWRITE_ADMIN_EMAIL', "bench-admin-{$runId}@example.com"); - $consolePassword = $this->env('APPWRITE_ADMIN_PASSWORD', $this->password); - $consoleHeaders = [ - 'Content-Type' => 'application/json', - 'X-Appwrite-Project' => $this->consoleProject, - ]; - - $account = $this->rawRequest('POST', '/account', [ - 'userId' => $this->unique('admin'), - 'email' => $consoleEmail, - 'password' => $consolePassword, - 'name' => 'Benchmark Admin', - ], $consoleHeaders, 'setup.account.create'); - - if (!in_array($account->status, [201, 409], true)) { - $this->failResponse($account, 'Unable to create or reuse the benchmark console account'); - } - - $session = $this->rawRequest('POST', '/account/sessions/email', [ - 'email' => $consoleEmail, - 'password' => $consolePassword, - ], $consoleHeaders, 'setup.account.session'); - $this->assertStatus($session, [201], 'console session created'); - - $consoleSessionHeaders = [ - ...$consoleHeaders, - 'Cookie' => $session->cookieHeader(), - ]; - - $team = $this->api('POST', '/teams', [ - 'teamId' => $this->unique('team'), - 'name' => "Benchmark Team {$runId}", - ], $consoleSessionHeaders, [201], 'setup.teams.create'); - - $teamId = (string) $team->json('$id'); - $project = $this->api('POST', '/projects', [ - 'projectId' => $this->unique('project'), - 'name' => "Benchmark Project {$runId}", - 'teamId' => $teamId, - 'region' => $this->region, - ], $consoleSessionHeaders, [201], 'setup.projects.create'); - - $projectId = (string) $project->json('$id'); - $key = $this->api('POST', "/projects/{$projectId}/keys", [ - 'keyId' => $this->unique('key'), - 'name' => 'Benchmark API key', - 'scopes' => self::API_SCOPES, - ], $consoleSessionHeaders, [201], 'setup.projects.keys.create'); - - $apiHeaders = [ - 'Content-Type' => 'application/json', - 'X-Appwrite-Project' => $projectId, - 'X-Appwrite-Key' => (string) $key->json('secret'), - ]; - - $platform = $this->api('POST', '/project/platforms/web', [ - 'platformId' => $this->unique('web'), - 'name' => 'Benchmark web', - 'hostname' => $this->hostnameFromUrl($this->redirectUrl), - ], $apiHeaders, [201, 409], 'setup.project.platforms.web.create'); - - $smtpBody = [ - 'enabled' => true, - 'senderName' => 'Benchmark', - 'senderEmail' => 'benchmark@appwrite.io', - 'replyTo' => 'benchmark@appwrite.io', - 'host' => $this->env('APPWRITE_SMTP_HOST', 'maildev'), - 'port' => (int) $this->env('APPWRITE_SMTP_PORT', '1025'), - 'username' => $this->env('APPWRITE_SMTP_USERNAME', 'user'), - 'password' => $this->env('APPWRITE_SMTP_PASSWORD', 'password'), - ]; - - if ($this->env('APPWRITE_SMTP_SECURE', '') !== '') { - $smtpBody['secure'] = $this->env('APPWRITE_SMTP_SECURE', ''); - } - - $smtp = $this->rawRequest('PATCH', "/projects/{$projectId}/smtp", $smtpBody, $consoleSessionHeaders, 'setup.projects.smtp.update'); - if ($smtp->status !== 200) { - fwrite(STDERR, "Custom SMTP was not enabled ({$smtp->status}). Mail worker timings may be unavailable." . PHP_EOL); - } - - return [ - 'runId' => $runId, - 'teamId' => $teamId, - 'projectId' => $projectId, - 'consoleSessionHeaders' => $consoleSessionHeaders, - 'apiHeaders' => $apiHeaders, - 'platformStatus' => $platform->status, - ]; - } - - private function curatedFlows(array &$context): void - { - try { - $this->accountFlow($context); - $this->databasesFlow($context); - $this->tablesDbFlow($context); - $this->storageFlow($context); - $this->messagingFlow($context); - $this->computeFlow($context); - $this->healthFlow($context); - } catch (Throwable $error) { - $this->metrics->addCounter('appwrite_benchmark_flow_failures'); - throw $error; - } - } - - private function teardown(array $context): void - { - if (($context['projectId'] ?? null) && ($context['consoleSessionHeaders'] ?? null)) { - $this->rawRequest('DELETE', "/projects/{$context['projectId']}", null, $context['consoleSessionHeaders'], 'teardown.projects.delete'); - } - - if (($context['teamId'] ?? null) && ($context['consoleSessionHeaders'] ?? null)) { - $this->rawRequest('DELETE', "/teams/{$context['teamId']}", null, $context['consoleSessionHeaders'], 'teardown.teams.delete'); - } - } - - private function accountFlow(array &$context): void - { - $userId = $this->unique('user'); - $email = 'bench-user-' . $this->unique('mail') . '@example.com'; - $headers = $this->projectHeaders($context['projectId']); - - $this->api('POST', '/account', [ - 'userId' => $userId, - 'email' => $email, - 'password' => $this->password, - 'name' => 'Benchmark User', - ], $headers, [201], 'account.create'); - - $session = $this->api('POST', '/account/sessions/email', [ - 'email' => $email, - 'password' => $this->password, - ], $headers, [201], 'account.sessions.email.create'); - - $sessionHeaders = [ - ...$headers, - 'Cookie' => $session->cookieHeader(), - ]; - - $context['userId'] = $userId; - $context['userEmail'] = $email; - $context['sessionHeaders'] = $sessionHeaders; - - $jwt = $this->api('POST', '/account/jwts', null, $sessionHeaders, [201], 'account.jwts.create'); - $context['jwtHeaders'] = [ - ...$headers, - 'X-Appwrite-JWT' => (string) $jwt->json('jwt'), - ]; - - $this->api('GET', '/account', null, $sessionHeaders, [200], 'account.get'); - $this->api('GET', '/account/logs', null, $sessionHeaders, [200], 'account.logs.list'); - $this->api('PATCH', '/account/prefs', ['prefs' => ['benchmark' => true, 'runId' => $context['runId']]], $sessionHeaders, [200], 'account.prefs.update'); - $this->api('PATCH', '/account/name', ['name' => 'Benchmark User Updated'], $sessionHeaders, [200], 'account.name.update'); - $this->api('PATCH', '/account/password', ['password' => $this->password . '2', 'oldPassword' => $this->password], $sessionHeaders, [200], 'account.password.update'); - - $verificationStarted = $this->nowMs(); - $this->api('POST', '/account/verifications/email', ['url' => $this->redirectUrl], $sessionHeaders, [201], 'account.emailVerification.create'); - $verificationEmail = $this->waitForEmail($email, fn (array $message): bool => $this->messageIncludes($message, ['verify', 'verification']), $this->mailTimeoutMs); - $this->metrics->addTrend('appwrite_worker_mails_duration', $this->nowMs() - $verificationStarted); - - $verification = $this->extractQueryParams($verificationEmail); - if (($verification['userId'] ?? null) && ($verification['secret'] ?? null)) { - $this->api('PUT', '/account/verifications/email', [ - 'userId' => $verification['userId'], - 'secret' => $verification['secret'], - ], $sessionHeaders, [200], 'account.emailVerification.update'); - } - - $recoveryStarted = $this->nowMs(); - $this->api('POST', '/account/recovery', ['email' => $email, 'url' => $this->redirectUrl], $headers, [201], 'account.recovery.create'); - $recoveryEmail = $this->waitForEmail($email, fn (array $message): bool => $this->messageIncludes($message, ['recovery', 'recover', 'reset']), $this->mailTimeoutMs); - $this->metrics->addTrend('appwrite_worker_mails_duration', $this->nowMs() - $recoveryStarted); - - $recovery = $this->extractQueryParams($recoveryEmail); - if (($recovery['userId'] ?? null) && ($recovery['secret'] ?? null)) { - $this->api('DELETE', '/account/sessions/current', null, $sessionHeaders, [204], 'account.sessions.current.delete'); - $this->api('PUT', '/account/recovery', [ - 'userId' => $recovery['userId'], - 'secret' => $recovery['secret'], - 'password' => $this->password . '3', - ], $headers, [200], 'account.recovery.update'); - - $recoveredSession = $this->api('POST', '/account/sessions/email', [ - 'email' => $email, - 'password' => $this->password . '3', - ], $headers, [201], 'account.sessions.email.recovered'); - - $context['sessionHeaders'] = [ - ...$headers, - 'Cookie' => $recoveredSession->cookieHeader(), - ]; - - $recoveredJwt = $this->api('POST', '/account/jwts', null, $context['sessionHeaders'], [201], 'account.jwts.recovered'); - $context['jwtHeaders'] = [ - ...$headers, - 'X-Appwrite-JWT' => (string) $recoveredJwt->json('jwt'), - ]; - } - } - - private function databasesFlow(array $context): void - { - $databaseId = $this->unique('db'); - $collectionId = $this->unique('col'); - $documentId = $this->unique('doc'); - $indexKey = $this->unique('idx'); - - $this->api('POST', '/databases', ['databaseId' => $databaseId, 'name' => 'Benchmark DB'], $context['apiHeaders'], [201], 'databases.create'); - $this->api('POST', "/databases/{$databaseId}/collections", [ - 'collectionId' => $collectionId, - 'name' => 'Benchmark Collection', - 'permissions' => self::BASE_PERMISSIONS, - 'documentSecurity' => false, - ], $context['apiHeaders'], [201], 'databases.collections.create'); - - $attributes = [ - ['string', 'title', ['size' => 128]], - ['integer', 'count', ['min' => 0, 'max' => 100000]], - ['email', 'email', []], - ['boolean', 'active', []], - ['datetime', 'publishedAt', []], - ['float', 'score', ['min' => 0, 'max' => 1000]], - ['url', 'url', []], - ['ip', 'ip', []], - ]; - - foreach ($attributes as [$type, $key, $extra]) { - $started = $this->nowMs(); - $this->api('POST', "/databases/{$databaseId}/collections/{$collectionId}/attributes/{$type}", [ - 'key' => $key, - 'required' => false, - 'array' => false, - ...$extra, - ], $context['apiHeaders'], [202], "databases.attributes.{$type}.create"); - $this->waitForStatus("/databases/{$databaseId}/collections/{$collectionId}/attributes/{$key}", $context['apiHeaders'], 'available', $this->workerTimeoutMs); - $this->metrics->addTrend('appwrite_worker_database_duration', $this->nowMs() - $started); - } - - $indexStarted = $this->nowMs(); - $this->api('POST', "/databases/{$databaseId}/collections/{$collectionId}/indexes", [ - 'key' => $indexKey, - 'type' => 'key', - 'attributes' => ['title'], - 'orders' => ['asc'], - ], $context['apiHeaders'], [202], 'databases.indexes.create'); - $this->waitForStatus("/databases/{$databaseId}/collections/{$collectionId}/indexes/{$indexKey}", $context['apiHeaders'], 'available', $this->workerTimeoutMs); - $this->metrics->addTrend('appwrite_worker_database_duration', $this->nowMs() - $indexStarted); - - $this->api('POST', "/databases/{$databaseId}/collections/{$collectionId}/documents", [ - 'documentId' => $documentId, - 'data' => $this->documentPayload(), - 'permissions' => self::ITEM_PERMISSIONS, - ], $context['apiHeaders'], [201], 'databases.documents.create'); - $this->api('GET', "/databases/{$databaseId}/collections/{$collectionId}/documents", null, $context['apiHeaders'], [200], 'databases.documents.list'); - $this->api('GET', "/databases/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", null, $context['apiHeaders'], [200], 'databases.documents.get'); - $this->api('PATCH', "/databases/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", ['data' => ['title' => 'Benchmark Document Updated']], $context['apiHeaders'], [200], 'databases.documents.update'); - $this->api('PATCH', "/databases/{$databaseId}/collections/{$collectionId}/documents/{$documentId}/count/increment", ['value' => 1], $context['apiHeaders'], [200], 'databases.documents.increment'); - $this->api('PATCH', "/databases/{$databaseId}/collections/{$collectionId}/documents/{$documentId}/count/decrement", ['value' => 1], $context['apiHeaders'], [200], 'databases.documents.decrement'); - $this->api('DELETE', "/databases/{$databaseId}/collections/{$collectionId}/documents/{$documentId}", null, $context['apiHeaders'], [204], 'databases.documents.delete'); - $this->api('DELETE', "/databases/{$databaseId}", null, $context['apiHeaders'], [204], 'databases.delete'); - } - - private function tablesDbFlow(array $context): void - { - if (!isset($context['sessionHeaders']) || !is_array($context['sessionHeaders'])) { - throw new RuntimeException('accountFlow must run before tablesDbFlow'); - } - - $databaseId = $this->unique('tdb'); - $tableId = $this->unique('tbl'); - $rowId = $this->unique('row'); - $indexKey = $this->unique('tidx'); - - $this->api('POST', '/tablesdb', ['databaseId' => $databaseId, 'name' => 'Benchmark TablesDB'], $context['apiHeaders'], [201], 'tablesdb.create'); - $this->api('POST', "/tablesdb/{$databaseId}/tables", [ - 'tableId' => $tableId, - 'name' => 'Benchmark Table', - 'permissions' => self::BASE_PERMISSIONS, - 'rowSecurity' => false, - ], $context['apiHeaders'], [201], 'tablesdb.tables.create'); - - $columns = [ - ['string', 'title', ['size' => 128]], - ['integer', 'count', ['min' => 0, 'max' => 100000]], - ['email', 'email', []], - ['boolean', 'active', []], - ]; - - foreach ($columns as [$type, $key, $extra]) { - $started = $this->nowMs(); - $this->api('POST', "/tablesdb/{$databaseId}/tables/{$tableId}/columns/{$type}", [ - 'key' => $key, - 'required' => false, - 'array' => false, - ...$extra, - ], $context['apiHeaders'], [202], "tablesdb.columns.{$type}.create"); - $this->waitForStatus("/tablesdb/{$databaseId}/tables/{$tableId}/columns/{$key}", $context['apiHeaders'], 'available', $this->workerTimeoutMs); - $this->metrics->addTrend('appwrite_worker_tables_duration', $this->nowMs() - $started); - } - - $indexStarted = $this->nowMs(); - $this->api('POST', "/tablesdb/{$databaseId}/tables/{$tableId}/indexes", [ - 'key' => $indexKey, - 'type' => 'key', - 'columns' => ['title'], - 'orders' => ['asc'], - ], $context['apiHeaders'], [202], 'tablesdb.indexes.create'); - $this->waitForStatus("/tablesdb/{$databaseId}/tables/{$tableId}/indexes/{$indexKey}", $context['apiHeaders'], 'available', $this->workerTimeoutMs); - $this->metrics->addTrend('appwrite_worker_tables_duration', $this->nowMs() - $indexStarted); - - $this->api('POST', "/tablesdb/{$databaseId}/tables/{$tableId}/rows", [ - 'rowId' => $rowId, - 'data' => $this->tablePayload(), - 'permissions' => self::ITEM_PERMISSIONS, - ], $context['sessionHeaders'], [201], 'tablesdb.rows.create'); - $this->api('GET', "/tablesdb/{$databaseId}/tables/{$tableId}/rows", null, $context['sessionHeaders'], [200], 'tablesdb.rows.list'); - $this->api('GET', "/tablesdb/{$databaseId}/tables/{$tableId}/rows/{$rowId}", null, $context['sessionHeaders'], [200], 'tablesdb.rows.get'); - $this->api('PATCH', "/tablesdb/{$databaseId}/tables/{$tableId}/rows/{$rowId}", ['data' => ['title' => 'Benchmark Row Updated']], $context['sessionHeaders'], [200], 'tablesdb.rows.update'); - $this->api('PATCH', "/tablesdb/{$databaseId}/tables/{$tableId}/rows/{$rowId}/count/increment", ['value' => 1], $context['sessionHeaders'], [200], 'tablesdb.rows.increment'); - $this->api('PATCH', "/tablesdb/{$databaseId}/tables/{$tableId}/rows/{$rowId}/count/decrement", ['value' => 1], $context['sessionHeaders'], [200], 'tablesdb.rows.decrement'); - $this->api('DELETE', "/tablesdb/{$databaseId}/tables/{$tableId}/rows/{$rowId}", null, $context['sessionHeaders'], [204], 'tablesdb.rows.delete'); - $this->api('DELETE', "/tablesdb/{$databaseId}", null, $context['apiHeaders'], [204], 'tablesdb.delete'); - } - - private function storageFlow(array $context): void - { - if (!isset($context['sessionHeaders']) || !is_array($context['sessionHeaders'])) { - throw new RuntimeException('accountFlow must run before storageFlow'); - } - - $bucketId = $this->unique('bucket'); - $fileId = $this->unique('file'); - - $this->api('POST', '/storage/buckets', [ - 'bucketId' => $bucketId, - 'name' => 'Benchmark Bucket', - 'permissions' => self::BASE_PERMISSIONS, - 'fileSecurity' => false, - 'enabled' => true, - 'maximumFileSize' => 30000000, - 'allowedFileExtensions' => [], - 'compression' => 'none', - 'encryption' => false, - 'antivirus' => false, - ], $context['apiHeaders'], [201], 'storage.buckets.create'); - - $tmpFile = tempnam(sys_get_temp_dir(), 'appwrite-benchmark-'); - if ($tmpFile === false) { - throw new RuntimeException('Unable to create temporary PNG fixture'); - } - - file_put_contents($tmpFile, base64_decode(self::PNG_1X1, true)); - - try { - $fields = [ - 'fileId' => $fileId, - 'file' => new CURLFile($tmpFile, 'image/png', 'benchmark.png'), - ...$this->flattenMultipartArray('permissions', self::ITEM_PERMISSIONS), - ]; - $multipartHeaders = $context['sessionHeaders']; - unset($multipartHeaders['Content-Type']); - - $upload = $this->rawMultipartRequest('POST', "/storage/buckets/{$bucketId}/files", $fields, $multipartHeaders, 'storage.files.create'); - $this->metrics->addTrend('appwrite_api_duration', $upload->duration); - $this->assertStatus($upload, [201], 'storage file created'); - } finally { - @unlink($tmpFile); - } - - $this->api('GET', "/storage/buckets/{$bucketId}/files", null, $context['sessionHeaders'], [200], 'storage.files.list'); - $this->api('GET', "/storage/buckets/{$bucketId}/files/{$fileId}", null, $context['sessionHeaders'], [200], 'storage.files.get'); - $this->api('GET', "/storage/buckets/{$bucketId}/files/{$fileId}/view", null, $context['sessionHeaders'], [200], 'storage.files.view'); - $this->api('GET', "/storage/buckets/{$bucketId}/files/{$fileId}/download", null, $context['sessionHeaders'], [200], 'storage.files.download'); - $this->api('GET', "/storage/buckets/{$bucketId}/files/{$fileId}/preview", null, $context['sessionHeaders'], [200], 'storage.files.preview'); - $this->api('PUT', "/storage/buckets/{$bucketId}/files/{$fileId}", [ - 'name' => 'benchmark-renamed.png', - 'permissions' => self::ITEM_PERMISSIONS, - ], $context['sessionHeaders'], [200], 'storage.files.update'); - - $token = $this->api('POST', "/tokens/buckets/{$bucketId}/files/{$fileId}", (object) [], $context['apiHeaders'], [201], 'tokens.files.create'); - $tokenId = (string) $token->json('$id'); - $this->api('GET', "/tokens/buckets/{$bucketId}/files/{$fileId}", null, $context['apiHeaders'], [200], 'tokens.files.list'); - $this->api('GET', "/tokens/{$tokenId}", null, $context['apiHeaders'], [200], 'tokens.get'); - $this->api('PATCH', "/tokens/{$tokenId}", ['expire' => null], $context['apiHeaders'], [200], 'tokens.update'); - $this->api('DELETE', "/tokens/{$tokenId}", null, $context['apiHeaders'], [204], 'tokens.delete'); - - $this->api('DELETE', "/storage/buckets/{$bucketId}/files/{$fileId}", null, $context['sessionHeaders'], [204], 'storage.files.delete'); - $this->api('DELETE', "/storage/buckets/{$bucketId}", null, $context['apiHeaders'], [204], 'storage.buckets.delete'); - } - - private function messagingFlow(array $context): void - { - if ( - !isset($context['sessionHeaders']) || !is_array($context['sessionHeaders']) - || !isset($context['userId'], $context['userEmail']) - ) { - throw new RuntimeException('accountFlow must run before messagingFlow'); - } - - $providerId = $this->unique('smtp'); - $targetId = $this->unique('target'); - $existingTarget = false; - $topicId = $this->unique('topic'); - $subscriberId = $this->unique('sub'); - $messageId = $this->unique('msg'); - - $this->api('POST', '/messaging/providers/smtp', [ - 'providerId' => $providerId, - 'name' => 'Benchmark SMTP', - 'host' => $this->env('APPWRITE_SMTP_HOST', 'maildev'), - 'port' => (int) $this->env('APPWRITE_SMTP_PORT', '1025'), - 'username' => $this->env('APPWRITE_SMTP_USERNAME', 'user'), - 'password' => $this->env('APPWRITE_SMTP_PASSWORD', 'password'), - 'encryption' => $this->env('APPWRITE_SMTP_ENCRYPTION', 'none'), - 'autoTLS' => false, - 'fromName' => 'Benchmark', - 'fromEmail' => 'benchmark@appwrite.io', - 'replyToName' => 'Benchmark', - 'replyToEmail' => 'benchmark@appwrite.io', - 'enabled' => true, - ], $context['apiHeaders'], [201], 'messaging.providers.smtp.create'); - - $targets = $this->api('GET', "/users/{$context['userId']}/targets", null, $context['apiHeaders'], [200], 'users.targets.list'); - foreach ($targets->json('targets') ?? [] as $target) { - if (($target['providerType'] ?? '') === 'email' && ($target['identifier'] ?? '') === $context['userEmail']) { - $targetId = (string) $target['$id']; - $existingTarget = true; - break; - } - } - - if ($existingTarget) { - $this->api('PATCH', "/users/{$context['userId']}/targets/{$targetId}", [ - 'providerId' => $providerId, - 'name' => 'Benchmark email target', - ], $context['apiHeaders'], [200], 'users.targets.update'); - } else { - $this->api('POST', "/users/{$context['userId']}/targets", [ - 'targetId' => $targetId, - 'providerType' => 'email', - 'identifier' => $context['userEmail'], - 'providerId' => $providerId, - 'name' => 'Benchmark email target', - ], $context['apiHeaders'], [201], 'users.targets.create'); - } - - $this->api('POST', '/messaging/topics', [ - 'topicId' => $topicId, - 'name' => 'Benchmark Topic', - 'subscribe' => ['users'], - ], $context['apiHeaders'], [201], 'messaging.topics.create'); - - $this->api('POST', "/messaging/topics/{$topicId}/subscribers", [ - 'subscriberId' => $subscriberId, - 'targetId' => $targetId, - ], $context['sessionHeaders'], [201], 'messaging.subscribers.create'); - - $started = $this->nowMs(); - $this->api('POST', '/messaging/messages/email', [ - 'messageId' => $messageId, - 'subject' => "Benchmark message {$context['runId']}", - 'content' => "Benchmark messaging worker probe {$context['runId']}", - 'targets' => [$targetId], - 'draft' => false, - 'html' => false, - ], $context['apiHeaders'], [201], 'messaging.messages.email.create'); - - $this->waitForMessage($messageId, $context['apiHeaders'], $this->workerTimeoutMs); - $this->waitForEmail($context['userEmail'], fn (array $message): bool => $this->includes($message['subject'] ?? '', "Benchmark message {$context['runId']}"), $this->mailTimeoutMs, true); - $this->metrics->addTrend('appwrite_worker_messaging_duration', $this->nowMs() - $started); - - $this->api('GET', '/messaging/messages', null, $context['apiHeaders'], [200], 'messaging.messages.list'); - $this->api('GET', "/messaging/messages/{$messageId}/logs", null, $context['apiHeaders'], [200], 'messaging.messages.logs.list'); - $this->api('GET', "/messaging/messages/{$messageId}/targets", null, $context['apiHeaders'], [200], 'messaging.messages.targets.list'); - $this->api('GET', "/messaging/providers/{$providerId}/logs", null, $context['apiHeaders'], [200], 'messaging.providers.logs.list'); - $this->api('GET', "/messaging/topics/{$topicId}/logs", null, $context['apiHeaders'], [200], 'messaging.topics.logs.list'); - $this->api('GET', "/messaging/subscribers/{$subscriberId}/logs", null, $context['apiHeaders'], [200], 'messaging.subscribers.logs.list'); - $this->api('DELETE', "/messaging/topics/{$topicId}/subscribers/{$subscriberId}", null, $context['sessionHeaders'], [204], 'messaging.subscribers.delete'); - $this->api('DELETE', "/messaging/topics/{$topicId}", null, $context['apiHeaders'], [204], 'messaging.topics.delete'); - $this->api('DELETE', "/messaging/messages/{$messageId}", null, $context['apiHeaders'], [204], 'messaging.messages.delete'); - $this->api('DELETE', "/messaging/providers/{$providerId}", null, $context['apiHeaders'], [204], 'messaging.providers.delete'); - } - - private function computeFlow(array $context): void - { - if (!isset($context['sessionHeaders']) || !is_array($context['sessionHeaders'])) { - throw new RuntimeException('accountFlow must run before computeFlow'); - } - - $functionId = $this->unique('fn'); - $siteId = $this->unique('site'); - $runtime = $this->env('APPWRITE_BENCHMARK_RUNTIME', 'node-22'); - - $this->api('POST', '/functions', [ - 'functionId' => $functionId, - 'name' => 'Benchmark Function', - 'runtime' => $runtime, - 'execute' => ['any'], - 'events' => [], - 'schedule' => '', - 'timeout' => 15, - 'enabled' => true, - 'logging' => true, - 'entrypoint' => 'index.js', - 'commands' => 'npm install', - 'scopes' => ['users.read'], - ], $context['apiHeaders'], [201], 'functions.create'); - $this->api('GET', '/functions/runtimes', null, $context['sessionHeaders'], [200], 'functions.runtimes.list'); - $this->api('GET', '/functions/specifications', null, $context['apiHeaders'], [200], 'functions.specifications.list'); - - $functionVariable = $this->api('POST', "/functions/{$functionId}/variables", [ - 'key' => 'BENCHMARK', - 'value' => 'true', - 'secret' => false, - ], $context['apiHeaders'], [201], 'functions.variables.create'); - $functionVariableId = (string) $functionVariable->json('$id'); - $this->api('PUT', "/functions/{$functionId}/variables/{$functionVariableId}", ['key' => 'BENCHMARK', 'value' => 'updated', 'secret' => false], $context['apiHeaders'], [200], 'functions.variables.update'); - $this->api('GET', "/functions/{$functionId}/variables/{$functionVariableId}", null, $context['apiHeaders'], [200], 'functions.variables.get'); - $this->api('DELETE', "/functions/{$functionId}/variables/{$functionVariableId}", null, $context['apiHeaders'], [204], 'functions.variables.delete'); - $this->api('DELETE', "/functions/{$functionId}", null, $context['apiHeaders'], [204], 'functions.delete'); - - $this->api('POST', '/sites', [ - 'siteId' => $siteId, - 'name' => 'Benchmark Site', - 'framework' => 'other', - 'adapter' => 'static', - 'buildRuntime' => $runtime, - 'buildCommand' => '', - 'outputDirectory' => '.', - 'installCommand' => '', - 'fallbackFile' => 'index.html', - 'providerRootDirectory' => '.', - 'specification' => '', - ], $context['apiHeaders'], [201], 'sites.create'); - $this->api('GET', '/sites/frameworks', null, $context['sessionHeaders'], [200], 'sites.frameworks.list'); - $this->api('GET', '/sites/specifications', null, $context['apiHeaders'], [200], 'sites.specifications.list'); - - $siteVariable = $this->api('POST', "/sites/{$siteId}/variables", ['key' => 'BENCHMARK', 'value' => 'true', 'secret' => false], $context['apiHeaders'], [201], 'sites.variables.create'); - $siteVariableId = (string) $siteVariable->json('$id'); - $this->api('PUT', "/sites/{$siteId}/variables/{$siteVariableId}", ['key' => 'BENCHMARK', 'value' => 'updated', 'secret' => false], $context['apiHeaders'], [200], 'sites.variables.update'); - $this->api('GET', "/sites/{$siteId}/variables/{$siteVariableId}", null, $context['apiHeaders'], [200], 'sites.variables.get'); - $this->api('DELETE', "/sites/{$siteId}/variables/{$siteVariableId}", null, $context['apiHeaders'], [204], 'sites.variables.delete'); - $this->api('DELETE', "/sites/{$siteId}", null, $context['apiHeaders'], [204], 'sites.delete'); - } - - private function healthFlow(array $context): void - { - $probes = [ - '/health', - '/health/db', - '/health/cache', - '/health/pubsub', - '/health/storage', - '/health/storage/local', - '/health/time', - '/health/queue/databases', - '/health/queue/mails', - '/health/queue/messaging', - '/health/queue/functions', - '/health/queue/builds', - '/health/queue/deletes', - '/health/queue/webhooks', - '/health/queue/stats-resources', - '/health/queue/stats-usage', - '/health/queue/failed/v1-mails', - ]; - - foreach ($probes as $path) { - $this->api('GET', $path, null, $context['apiHeaders'], [200], 'health' . str_replace('/', '.', $path)); - } - } - - private function api(string $method, string $path, mixed $body, array $headers, array $expected, string $name): BenchmarkResponse - { - $response = $this->rawRequest($method, $path, $body, $headers, $name); - $this->metrics->addTrend('appwrite_api_duration', $response->duration); - $this->assertStatus($response, $expected, $name); - return $response; - } - - private function rawRequest(string $method, string $path, mixed $body, array $headers, string $name, bool $recordHttpDuration = true): BenchmarkResponse - { - return $this->send($method, str_starts_with($path, 'http') ? $path : $this->endpoint . $path, $body, $headers, $name, false, $recordHttpDuration); - } - - private function rawMultipartRequest(string $method, string $path, array $fields, array $headers, string $name): BenchmarkResponse - { - return $this->send($method, $this->endpoint . $path, $fields, $headers, $name, true, true); - } - - private function send(string $method, string $url, mixed $body, array $headers, string $name, bool $multipart, bool $recordHttpDuration): BenchmarkResponse - { - $handle = curl_init($url); - if ($handle === false) { - throw new RuntimeException("Unable to initialize curl for {$url}"); - } - - $headerLines = []; - foreach ($headers as $key => $value) { - $headerLines[] = "{$key}: {$value}"; - } - - curl_setopt_array($handle, [ - CURLOPT_CUSTOMREQUEST => $method, - CURLOPT_RETURNTRANSFER => true, - CURLOPT_HEADER => true, - CURLOPT_HTTPHEADER => $headerLines, - CURLOPT_TIMEOUT => 120, - ]); - - if ($body !== null) { - curl_setopt($handle, CURLOPT_POSTFIELDS, $multipart ? $body : json_encode($body, JSON_UNESCAPED_SLASHES)); - } elseif (in_array($method, ['POST', 'PUT', 'PATCH'], true)) { - curl_setopt($handle, CURLOPT_POSTFIELDS, ''); - } - - $started = hrtime(true); - $raw = curl_exec($handle); - $duration = (hrtime(true) - $started) / 1_000_000; - if ($recordHttpDuration) { - $this->metrics->addTrend('http_req_duration', $duration); - } - - if ($raw === false) { - $error = curl_error($handle); - throw new RuntimeException("{$name} curl error: {$error}"); - } - - $status = (int) curl_getinfo($handle, CURLINFO_RESPONSE_CODE); - $headerSize = (int) curl_getinfo($handle, CURLINFO_HEADER_SIZE); - - return new BenchmarkResponse( - $status, - substr($raw, $headerSize), - $this->parseHeaders(substr($raw, 0, $headerSize)), - $duration, - ); - } - - private function waitForStatus(string $path, array $headers, string $wantedStatus, int $timeoutMs): BenchmarkResponse - { - $started = $this->nowMs(); - - while ($this->nowMs() - $started < $timeoutMs) { - $response = $this->rawRequest('GET', $path, null, $headers, "wait{$path}"); - if ($response->status === 200) { - $status = $response->json('status'); - if ($status === $wantedStatus) { - return $response; - } - - if ($status === 'failed') { - throw new RuntimeException("Resource {$path} failed while waiting for {$wantedStatus}"); - } - } - - usleep(500_000); - } - - throw new RuntimeException("Timed out waiting for {$path} to become {$wantedStatus}"); - } - - private function waitForMessage(string $messageId, array $headers, int $timeoutMs): BenchmarkResponse - { - $started = $this->nowMs(); - - while ($this->nowMs() - $started < $timeoutMs) { - $response = $this->rawRequest('GET', "/messaging/messages/{$messageId}", null, $headers, 'messaging.messages.poll'); - $status = $response->status === 200 ? $response->json('status') : null; - - if (in_array($status, ['sent', 'failed'], true)) { - if ($status === 'failed') { - throw new RuntimeException("Messaging worker marked message {$messageId} as failed"); - } - - return $response; - } - - usleep(500_000); - } - - throw new RuntimeException("Timed out waiting for messaging worker to send message {$messageId}"); - } - - private function waitForEmail(string $address, callable $predicate, int $timeoutMs, bool $allowMissingRecipient = false): array - { - $started = $this->nowMs(); - - while ($this->nowMs() - $started < $timeoutMs) { - $response = $this->rawRequest('GET', $this->maildevEndpoint, null, [], 'maildev.email.list', false); - - if ($response->status === 200) { - $emails = $response->json(); - if (is_array($emails)) { - for ($i = count($emails) - 1; $i >= 0; $i--) { - $message = $emails[$i]; - if (!is_array($message)) { - continue; - } - - if (($this->emailMatches($message, $address) || ($allowMissingRecipient && $this->emailRecipientMissing($message))) && $predicate($message)) { - return $message; - } - } - } - } - - usleep(500_000); - } - - throw new RuntimeException("Timed out waiting for email to {$address}"); - } - - private function assertStatus(BenchmarkResponse $response, array $expected, string $name): void - { - $passed = in_array($response->status, $expected, true); - $this->metrics->addCheck($passed); - - if (!$passed) { - $this->failResponse($response, "{$name} returned an unexpected status"); - } - } - - private function failResponse(BenchmarkResponse $response, string $message): never - { - throw new RuntimeException("{$message}. Status: {$response->status}. Body: {$response->body}"); - } - - private function parseHeaders(string $rawHeaders): array - { - $blocks = preg_split("/\r\n\r\n|\n\n/", trim($rawHeaders)) ?: []; - $headerBlock = end($blocks) ?: ''; - $headers = []; - - foreach (preg_split("/\r\n|\n|\r/", $headerBlock) ?: [] as $line) { - if (!str_contains($line, ':')) { - continue; - } - - [$name, $value] = explode(':', $line, 2); - $headers[strtolower(trim($name))][] = trim($value); - } - - return $headers; - } - - private function emailMatches(array $message, string $address): bool - { - foreach ($message['to'] ?? [] as $recipient) { - if (($recipient['address'] ?? null) === $address) { - return true; - } - } - - return false; - } - - private function emailRecipientMissing(array $message): bool - { - $recipients = $message['to'] ?? []; - if ($recipients === []) { - return true; - } - - foreach ($recipients as $recipient) { - if ($recipient['address'] ?? null) { - return false; - } - } - - return true; - } - - private function extractQueryParams(array $message): array - { - $content = ($message['html'] ?? '') . "\n" . ($message['text'] ?? ''); - preg_match_all('/href="([^"]+)"/', $content, $matches); - $links = $matches[1] ?: [$content]; - - foreach ($links as $link) { - $query = parse_url(html_entity_decode($link), PHP_URL_QUERY); - if (!is_string($query)) { - continue; - } - - parse_str($query, $params); - if (($params['userId'] ?? null) && ($params['secret'] ?? null)) { - return $params; - } - } - - return []; - } - - private function projectHeaders(string $projectId): array - { - return [ - 'Content-Type' => 'application/json', - 'X-Appwrite-Project' => $projectId, - ]; - } - - private function documentPayload(): array - { - return [ - 'title' => 'Benchmark Document', - 'count' => 1, - 'email' => 'document@example.com', - 'active' => true, - 'publishedAt' => gmdate('c'), - 'score' => 10.5, - 'url' => 'https://appwrite.io', - 'ip' => '127.0.0.1', - ]; - } - - private function tablePayload(): array - { - return [ - 'title' => 'Benchmark Row', - 'count' => 1, - 'email' => 'row@example.com', - 'active' => true, - ]; - } - - private function flattenMultipartArray(string $key, array $values): array - { - $output = []; - - foreach (array_values($values) as $index => $value) { - $output["{$key}[{$index}]"] = $value; - } - - return $output; - } - - private function messageIncludes(array $message, array $needles): bool - { - $content = implode("\n", [ - (string) ($message['subject'] ?? ''), - (string) ($message['html'] ?? ''), - (string) ($message['text'] ?? ''), - ]); - - foreach ($needles as $needle) { - if ($this->includes($content, $needle)) { - return true; - } - } - - return false; - } - - private function includes(string $value, string $needle): bool - { - return str_contains(strtolower($value), strtolower($needle)); - } - - private function hostnameFromUrl(string $value): string - { - $host = parse_url($value, PHP_URL_HOST); - if (is_string($host) && $host !== '') { - return $host; - } - - return explode(':', explode('/', preg_replace('/^https?:\/\//', '', $value) ?? '')[0])[0]; - } - - private function unique(string $prefix): string - { - $id = strtolower($prefix . '-' . base_convert((string) ((int) (microtime(true) * 1000)), 10, 36) . '-' . bin2hex(random_bytes(4))); - return substr(preg_replace('/[^a-z0-9-]/', '-', $id) ?? $id, 0, 36); - } - - private function nowMs(): float - { - return hrtime(true) / 1_000_000; - } - - private function env(string $name, string $default): string - { - $value = getenv($name); - return $value === false || $value === '' ? $default : $value; - } - - private function loadPreviousSummary(string $path): ?array - { - if (!is_file($path)) { - return null; - } - - $summary = json_decode((string) file_get_contents($path), true); - return is_array($summary) ? $summary : null; - } - - private function writeSummary(array $summary): void - { - $directory = dirname($this->summaryPath); - if ($directory !== '.' && !is_dir($directory) && !mkdir($directory, 0777, true) && !is_dir($directory)) { - throw new RuntimeException("Unable to create benchmark summary directory: {$directory}"); - } - - $json = json_encode($summary, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); - if ($json === false) { - throw new RuntimeException('Unable to encode benchmark summary: ' . json_last_error_msg()); - } - - if (file_put_contents($this->summaryPath, $json) === false) { - throw new RuntimeException("Unable to write benchmark summary: {$this->summaryPath}"); - } - } - - private function renderSummary(array $summary): string - { - $lines = [ - 'Appwrite curated benchmark review', - '', - 'Before/after comparison', - '', - $this->comparisonTable($this->previousSummary, $summary), - '', - 'Current run details', - '', - $this->metricLine($summary, 'http_req_duration', 'HTTP total'), - $this->metricLine($summary, 'appwrite_api_duration', 'API endpoints'), - $this->metricLine($summary, 'appwrite_worker_database_duration', 'Database worker schema jobs'), - $this->metricLine($summary, 'appwrite_worker_tables_duration', 'TablesDB worker schema jobs'), - $this->metricLine($summary, 'appwrite_worker_mails_duration', 'Mail worker delivery'), - $this->metricLine($summary, 'appwrite_worker_messaging_duration', 'Messaging worker delivery'), - '', - ]; - - return implode(PHP_EOL, $lines) . PHP_EOL; - } - - private function comparisonTable(?array $before, array $after): string - { - $rows = [ - ['HTTP total p95', $this->trendMetric($before, 'http_req_duration', 'p(95)'), $this->trendMetric($after, 'http_req_duration', 'p(95)'), 'ms'], - ['API endpoints p95', $this->trendMetric($before, 'appwrite_api_duration', 'p(95)'), $this->trendMetric($after, 'appwrite_api_duration', 'p(95)'), 'ms'], - ['Database worker p95', $this->trendMetric($before, 'appwrite_worker_database_duration', 'p(95)'), $this->trendMetric($after, 'appwrite_worker_database_duration', 'p(95)'), 'ms'], - ['TablesDB worker p95', $this->trendMetric($before, 'appwrite_worker_tables_duration', 'p(95)'), $this->trendMetric($after, 'appwrite_worker_tables_duration', 'p(95)'), 'ms'], - ['Mail worker p95', $this->trendMetric($before, 'appwrite_worker_mails_duration', 'p(95)'), $this->trendMetric($after, 'appwrite_worker_mails_duration', 'p(95)'), 'ms'], - ['Messaging worker p95', $this->trendMetric($before, 'appwrite_worker_messaging_duration', 'p(95)'), $this->trendMetric($after, 'appwrite_worker_messaging_duration', 'p(95)'), 'ms'], - ]; - - $table = [ - '| Metric | Before | After | Delta |', - '| --- | ---: | ---: | ---: |', - ]; - - foreach ($rows as [$label, $beforeValue, $afterValue, $unit]) { - $table[] = "| {$label} | {$this->formatValue($beforeValue, $unit)} | {$this->formatValue($afterValue, $unit)} | {$this->formatDelta($beforeValue, $afterValue, $unit)} |"; - } - - return implode(PHP_EOL, $table); - } - - private function trendMetric(?array $data, string $metric, string $stat): ?float - { - return $data['metrics'][$metric]['values'][$stat] ?? null; - } - - private function metricLine(array $data, string $metric, string $label): string - { - $values = $data['metrics'][$metric]['values'] ?? null; - if (!is_array($values) || ($values['count'] ?? 0) === 0) { - return "{$label}: no samples"; - } - - return "{$label}: avg={$this->round($values['avg'])}ms p90={$this->round($values['p(90)'])}ms p95={$this->round($values['p(95)'])}ms max={$this->round($values['max'])}ms"; - } - - private function formatValue(?float $value, string $unit): string - { - return $value === null || is_nan($value) ? 'n/a' : $this->round($value) . $unit; - } - - private function formatDelta(?float $before, ?float $after, string $unit): string - { - if ($before === null || $after === null || is_nan($before) || is_nan($after)) { - return 'n/a'; - } - - $delta = $this->round($after - $before); - return ($delta > 0 ? '+' : '') . $delta . $unit; - } - - private function round(float|int|null $value): string - { - $rounded = round((float) ($value ?? 0), 2); - return rtrim(rtrim(number_format($rounded, 2, '.', ''), '0'), '.'); - } -} - -exit((new HttpBenchmark())->run()); From 3cc7b833dbe91b9bff0d8e9fa6188e558d7f2c48 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 09:06:54 +0530 Subject: [PATCH 19/39] Fix k6 benchmark diagnostics --- .github/workflows/ci.yml | 67 +++++++++++++++++++++++++++++++++++++++- tests/benchmarks/http.js | 2 +- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 327f32daa8..22c1f78ddc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -712,8 +712,10 @@ jobs: if: steps.benchmark_before_start.outcome == 'success' continue-on-error: true run: | - rm -f benchmark-before-summary.json benchmark-after-summary.json benchmark-before.txt benchmark.txt + set -o pipefail + rm -f benchmark-before-summary.json benchmark-after-summary.json benchmark-before-samples.json benchmark-after-samples.json benchmark-before.txt benchmark.txt docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts ${{ env.K6_IMAGE }} run --quiet \ + --out json=benchmark-before-samples.json \ -e APPWRITE_ENDPOINT=http://localhost/v1 \ -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ -e APPWRITE_BENCHMARK_ITERATIONS=1 \ @@ -736,8 +738,12 @@ jobs: docker compose up -d --wait --no-build - name: Benchmark after + id: benchmark_after + continue-on-error: true run: | + set -o pipefail docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts ${{ env.K6_IMAGE }} run --quiet \ + --out json=benchmark-after-samples.json \ -e APPWRITE_ENDPOINT=http://localhost/v1 \ -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ -e APPWRITE_BENCHMARK_ITERATIONS=1 \ @@ -817,10 +823,58 @@ jobs: return `| ${label} | ${detailValue(values.avg ?? null, suffix)} | ${detailValue(values['p(90)'] ?? null, suffix)} | ${detailValue(values['p(95)'] ?? null, suffix)} | ${detailValue(values.max ?? null, suffix)} |`; } + function readSamples(path) { + if (!fs.existsSync(path)) { + return []; + } + + const contents = fs.readFileSync(path, 'utf8').trim(); + if (contents === '') { + return []; + } + + return contents + .split('\n') + .filter(Boolean) + .flatMap((line) => { + try { + return [JSON.parse(line)]; + } catch { + return []; + } + }); + } + + function slowestSample(samples, metric) { + return samples.reduce((slowest, sample) => { + if (sample.metric !== metric || typeof sample.data?.value !== 'number') { + return slowest; + } + + const current = { + name: sample.data.tags?.name || 'unknown', + value: sample.data.value, + }; + + return slowest === null || current.value > slowest.value ? current : slowest; + }, null); + } + + function formatSlowest(sample) { + if (sample === null) { + return 'n/a'; + } + + return `${markdownText(sample.name).replace(/\|/g, '\\|')} (${detailValue(sample.value, 'ms')})`; + } + const before = readSummary('benchmark-before-summary.json', false); const after = readSummary('benchmark-after-summary.json'); + const afterSamples = readSamples('benchmark-after-samples.json'); const baseRef = markdownText(process.env.BENCHMARK_BASE_REF || 'base'); const headRef = markdownText(process.env.BENCHMARK_HEAD_REF || 'head'); + const slowestHttp = slowestSample(afterSamples, 'appwrite_http_duration'); + const slowestApi = slowestSample(afterSamples, 'appwrite_api_duration'); const rows = [ row('HTTP total p95', metricValue(before, 'appwrite_http_duration', 'p(95)'), metricValue(after, 'appwrite_http_duration', 'p(95)'), 'ms'), @@ -858,6 +912,11 @@ jobs: console.log(detailRow(after, 'Mail worker delivery', 'appwrite_worker_mails_duration')); console.log(detailRow(after, 'Messaging worker delivery', 'appwrite_worker_messaging_duration')); console.log(); + console.log('| Detail | Value |'); + console.log('| --- | --- |'); + console.log(`| Slowest Appwrite request | ${formatSlowest(slowestHttp)} |`); + console.log(`| Slowest API endpoint | ${formatSlowest(slowestApi)} |`); + console.log(); console.log(''); NODE @@ -871,6 +930,8 @@ jobs: benchmark.txt benchmark-before-summary.json benchmark-after-summary.json + benchmark-before-samples.json + benchmark-after-samples.json retention-days: 7 - name: Find Comment @@ -899,3 +960,7 @@ jobs: issue-number: ${{ github.event.pull_request.number }} body-path: benchmark-comment.txt edit-mode: replace + + - name: Fail benchmark + if: steps.benchmark_after.outcome == 'failure' + run: exit 1 diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index e852794e3b..71a96c69e2 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -885,7 +885,7 @@ function tablePayload() { } function onePixelPng() { - return encoding.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=', 'std', 'b'); + return encoding.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII=', 'std', 'b'); } function flattenMultipartArray(key, values) { From cb7f2ec693e8c34473a87602f60572b8161f27b2 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 09:19:26 +0530 Subject: [PATCH 20/39] Show top benchmark request waits --- .github/workflows/ci.yml | 46 +++++++++++++++++++++++----------------- tests/benchmarks/http.js | 2 ++ 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22c1f78ddc..54f43a0a9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -845,27 +845,35 @@ jobs: }); } - function slowestSample(samples, metric) { - return samples.reduce((slowest, sample) => { + function topSamples(samples, metric, limit) { + const byName = samples.reduce((result, sample) => { if (sample.metric !== metric || typeof sample.data?.value !== 'number') { - return slowest; + return result; } - const current = { - name: sample.data.tags?.name || 'unknown', - value: sample.data.value, - }; + const name = sample.data.tags?.name || 'unknown'; + const current = result.get(name); + if (!current || sample.data.value > current.value) { + result.set(name, { name, value: sample.data.value }); + } - return slowest === null || current.value > slowest.value ? current : slowest; - }, null); + return result; + }, new Map()); + + return [...byName.values()] + .sort((left, right) => right.value - left.value) + .slice(0, limit); } - function formatSlowest(sample) { - if (sample === null) { - return 'n/a'; + function topSampleRows(samples) { + if (samples.length === 0) { + return ['| n/a | n/a |']; } - return `${markdownText(sample.name).replace(/\|/g, '\\|')} (${detailValue(sample.value, 'ms')})`; + return samples.map((sample) => { + const name = markdownText(sample.name).replace(/\|/g, '\\|'); + return `| ${name} | ${detailValue(sample.value, 'ms')} |`; + }); } const before = readSummary('benchmark-before-summary.json', false); @@ -873,8 +881,7 @@ jobs: const afterSamples = readSamples('benchmark-after-samples.json'); const baseRef = markdownText(process.env.BENCHMARK_BASE_REF || 'base'); const headRef = markdownText(process.env.BENCHMARK_HEAD_REF || 'head'); - const slowestHttp = slowestSample(afterSamples, 'appwrite_http_duration'); - const slowestApi = slowestSample(afterSamples, 'appwrite_api_duration'); + const topWaits = topSamples(afterSamples, 'appwrite_http_waiting', 3); const rows = [ row('HTTP total p95', metricValue(before, 'appwrite_http_duration', 'p(95)'), metricValue(after, 'appwrite_http_duration', 'p(95)'), 'ms'), @@ -912,10 +919,11 @@ jobs: console.log(detailRow(after, 'Mail worker delivery', 'appwrite_worker_mails_duration')); console.log(detailRow(after, 'Messaging worker delivery', 'appwrite_worker_messaging_duration')); console.log(); - console.log('| Detail | Value |'); - console.log('| --- | --- |'); - console.log(`| Slowest Appwrite request | ${formatSlowest(slowestHttp)} |`); - console.log(`| Slowest API endpoint | ${formatSlowest(slowestApi)} |`); + console.log('**Top 3 request waits**'); + console.log(); + console.log('| Request | Max wait |'); + console.log('| --- | ---: |'); + console.log(topSampleRows(topWaits).join('\n')); console.log(); console.log(''); NODE diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 71a96c69e2..7762b5d8d6 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -18,6 +18,7 @@ const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || const PREVIOUS_SUMMARY = loadPreviousSummary(); export const httpDuration = new Trend('appwrite_http_duration', true); +export const httpWaiting = new Trend('appwrite_http_waiting', true); export const apiDuration = new Trend('appwrite_api_duration', true); export const databaseWorkerDuration = new Trend('appwrite_worker_database_duration', true); export const tablesWorkerDuration = new Trend('appwrite_worker_tables_duration', true); @@ -720,6 +721,7 @@ function rawRequest(method, path, body, headers, name) { const payload = body === null || body === undefined ? null : JSON.stringify(body); const response = http.request(method, `${ENDPOINT}${path}`, payload, params); httpDuration.add(response.timings.duration, { name }); + httpWaiting.add(response.timings.waiting, { name }); return response; } From 7c486ddcef428c1cf87d4f3b3e3d4d90c9a64b59 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 09:26:36 +0530 Subject: [PATCH 21/39] Keep benchmark comment on missing summary --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54f43a0a9d..7e88feda47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -815,7 +815,7 @@ jobs: } function detailRow(after, label, metric, suffix = 'ms') { - const values = after.metrics?.[metric]?.values; + const values = after?.metrics?.[metric]?.values; if (!values) { return `| ${label} | n/a | n/a | n/a | n/a |`; } @@ -877,7 +877,7 @@ jobs: } const before = readSummary('benchmark-before-summary.json', false); - const after = readSummary('benchmark-after-summary.json'); + const after = readSummary('benchmark-after-summary.json', false); const afterSamples = readSamples('benchmark-after-samples.json'); const baseRef = markdownText(process.env.BENCHMARK_BASE_REF || 'base'); const headRef = markdownText(process.env.BENCHMARK_HEAD_REF || 'head'); @@ -901,6 +901,10 @@ jobs: console.log('> Before benchmark did not complete; showing current branch metrics only.'); console.log(); } + if (after === null) { + console.log('> Current branch benchmark did not complete; showing available metrics only.'); + console.log(); + } console.log('| Metric | Before | After | Delta |'); console.log('| --- | ---: | ---: | ---: |'); console.log(rows.join('\n')); From 3b9c604eb8cbceb81c2879b73c2d678f4ec2e20c Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 09:45:51 +0530 Subject: [PATCH 22/39] Harden benchmark comparison run --- .github/workflows/ci.yml | 3 +++ tests/benchmarks/http.js | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e88feda47..a52f051fa7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -714,12 +714,14 @@ jobs: run: | set -o pipefail rm -f benchmark-before-summary.json benchmark-after-summary.json benchmark-before-samples.json benchmark-after-samples.json benchmark-before.txt benchmark.txt + # Use the current benchmark script for both images so before/after differ only by the Appwrite image. docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts ${{ env.K6_IMAGE }} run --quiet \ --out json=benchmark-before-samples.json \ -e APPWRITE_ENDPOINT=http://localhost/v1 \ -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ -e APPWRITE_BENCHMARK_ITERATIONS=1 \ -e APPWRITE_BENCHMARK_VUS=1 \ + -e APPWRITE_WORKER_TIMEOUT_MS=120000 \ -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-before-summary.json \ tests/benchmarks/http.js | tee benchmark-before.txt @@ -748,6 +750,7 @@ jobs: -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ -e APPWRITE_BENCHMARK_ITERATIONS=1 \ -e APPWRITE_BENCHMARK_VUS=1 \ + -e APPWRITE_WORKER_TIMEOUT_MS=120000 \ -e APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH=benchmark-before-summary.json \ -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-after-summary.json \ tests/benchmarks/http.js | tee benchmark.txt diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 7762b5d8d6..e6b7db58ef 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -956,11 +956,20 @@ function detailRow(data, label, metric, unit = 'ms') { } function loadPreviousSummary() { - try { - return JSON.parse(open(PREVIOUS_SUMMARY_PATH)); - } catch (error) { - return null; + const paths = [PREVIOUS_SUMMARY_PATH]; + if (!PREVIOUS_SUMMARY_PATH.startsWith('/')) { + paths.push(`../../${PREVIOUS_SUMMARY_PATH}`); } + + for (const path of paths) { + try { + return JSON.parse(open(path)); + } catch (error) { + // Try the next path. k6 resolves open() relative to the script file. + } + } + + return null; } function comparisonTable(before, after) { From a0ef5968fb9ea2084f7032cd9dc4fa3fe13b05a8 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 11:58:30 +0530 Subject: [PATCH 23/39] Document local HTTP benchmark command --- tests/benchmarks/http.js | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index e6b7db58ef..0ec27f01db 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -1,3 +1,28 @@ +/* + * Run locally: + * + * docker compose up -d --force-recreate --build --wait + * + * docker run --rm -i \ + * --network appwrite \ + * -p 127.0.0.1:5665:5665 \ + * -v "$PWD:/scripts:ro" \ + * -v /tmp:/host-tmp \ + * -w /scripts \ + * -e K6_WEB_DASHBOARD=true \ + * -e K6_WEB_DASHBOARD_HOST=0.0.0.0 \ + * -e K6_WEB_DASHBOARD_PORT=5665 \ + * -e K6_WEB_DASHBOARD_EXPORT=/host-tmp/appwrite-k6-report.html \ + * -e APPWRITE_ENDPOINT=http://appwrite/v1 \ + * -e APPWRITE_MAILDEV_ENDPOINT=http://maildev:1080/email \ + * -e APPWRITE_WORKER_TIMEOUT_MS=120000 \ + * -e APPWRITE_BENCHMARK_SUMMARY_PATH=/host-tmp/appwrite-k6-summary.json \ + * grafana/k6:0.53.0 run \ + * --out json=/host-tmp/appwrite-k6-samples.json \ + * tests/benchmarks/http.js + * + * Open http://127.0.0.1:5665 while the benchmark is running. + */ import http from 'k6/http'; import { check, group, sleep } from 'k6'; import encoding from 'k6/encoding'; From a98b9f23195548a7f72e91329ae3c88f04dedbb6 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 12:06:32 +0530 Subject: [PATCH 24/39] Handle malformed optional benchmark summaries --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a52f051fa7..de5fbfc433 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -778,7 +778,11 @@ jobs: try { return JSON.parse(fs.readFileSync(path, 'utf8')); } catch (error) { - throw new Error(`Invalid benchmark summary ${path}: ${error.message}`); + if (required) { + throw new Error(`Invalid benchmark summary ${path}: ${error.message}`); + } + console.error(`Invalid benchmark summary ${path}: ${error.message}`); + return null; } } From 32508e7251c770192135774e3349a9823e06e8d6 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 12:53:13 +0530 Subject: [PATCH 25/39] Avoid reserved TablesDB benchmark column name --- tests/benchmarks/http.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 0ec27f01db..90e6003332 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -445,7 +445,7 @@ function tablesDbFlow(ctx) { const columns = [ ['string', 'title', { size: 128 }], - ['integer', 'count', { min: 0, max: 100000 }], + ['integer', 'quantity', { min: 0, max: 100000 }], ['email', 'email', {}], ['boolean', 'active', {}], ]; @@ -482,10 +482,10 @@ function tablesDbFlow(ctx) { api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, { data: { title: 'Benchmark Row Updated' }, }, ctx.sessionHeaders, [200], 'tablesdb.rows.update'); - api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}/count/increment`, { + api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}/quantity/increment`, { value: 1, }, ctx.sessionHeaders, [200], 'tablesdb.rows.increment'); - api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}/count/decrement`, { + api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}/quantity/decrement`, { value: 1, }, ctx.sessionHeaders, [200], 'tablesdb.rows.decrement'); api('DELETE', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, null, ctx.sessionHeaders, [204], 'tablesdb.rows.delete'); @@ -905,7 +905,7 @@ function documentPayload() { function tablePayload() { return { title: 'Benchmark Row', - count: 1, + quantity: 1, email: 'row@example.com', active: true, }; From b3f305f9a805ce065d7114ced5cc3e9561569527 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 13:02:16 +0530 Subject: [PATCH 26/39] Record storage upload wait metric --- tests/benchmarks/http.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 90e6003332..2a3527c95b 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -524,6 +524,7 @@ function storageFlow(ctx) { }); httpDuration.add(upload.timings.duration, { name: 'storage.files.create' }); + httpWaiting.add(upload.timings.waiting, { name: 'storage.files.create' }); apiDuration.add(upload.timings.duration, { name: 'storage.files.create' }); assertStatus(upload, [201], 'storage file created'); From 73a77b8dccb041934ee6a5b817e992d69ea68a0a Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 13:45:50 +0530 Subject: [PATCH 27/39] Show benchmark throughput --- .github/workflows/ci.yml | 1 + tests/benchmarks/http.js | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de5fbfc433..2bbff0e8c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -891,6 +891,7 @@ jobs: const topWaits = topSamples(afterSamples, 'appwrite_http_waiting', 3); const rows = [ + row('HTTP throughput', metricValue(before, 'http_reqs', 'rate'), metricValue(after, 'http_reqs', 'rate'), ' req/s'), row('HTTP total p95', metricValue(before, 'appwrite_http_duration', 'p(95)'), metricValue(after, 'appwrite_http_duration', 'p(95)'), 'ms'), row('API endpoints p95', metricValue(before, 'appwrite_api_duration', 'p(95)'), metricValue(after, 'appwrite_api_duration', 'p(95)'), 'ms'), row('Database worker p95', metricValue(before, 'appwrite_worker_database_duration', 'p(95)'), metricValue(after, 'appwrite_worker_database_duration', 'p(95)'), 'ms'), diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 2a3527c95b..ef873f6688 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -1000,6 +1000,7 @@ function loadPreviousSummary() { function comparisonTable(before, after) { const rows = [ + ['HTTP throughput', trendMetric(before, 'http_reqs', 'rate'), trendMetric(after, 'http_reqs', 'rate'), ' req/s'], ['HTTP total p95', trendMetric(before, 'appwrite_http_duration', 'p(95)'), trendMetric(after, 'appwrite_http_duration', 'p(95)'), 'ms'], ['API endpoints p95', trendMetric(before, 'appwrite_api_duration', 'p(95)'), trendMetric(after, 'appwrite_api_duration', 'p(95)'), 'ms'], ['Database worker p95', trendMetric(before, 'appwrite_worker_database_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_database_duration', 'p(95)'), 'ms'], From 7d7fcea8c0f6c1b24ad3556b9b817971d0d82bf6 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 14:16:12 +0530 Subject: [PATCH 28/39] Ensure benchmark failures fail CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bbff0e8c8..c215e209e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -982,5 +982,5 @@ jobs: edit-mode: replace - name: Fail benchmark - if: steps.benchmark_after.outcome == 'failure' + if: always() && steps.benchmark_after.outcome == 'failure' run: exit 1 From dfd39d394604a9054918093fd160362d0d5eff37 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 14:25:59 +0530 Subject: [PATCH 29/39] Tolerate benchmark cleanup failures --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c215e209e8..802648b0ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -757,7 +757,7 @@ jobs: - name: Stop after Appwrite if: always() - run: docker compose down -v + run: docker compose down -v || true - name: Prepare comment env: From f75a7269c9a1e7f70a56cba87bb48f4353266f1e Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 15:23:35 +0530 Subject: [PATCH 30/39] Address benchmark review simplifications --- .github/workflows/benchmark-comment.js | 355 +++++++++++++++++++++++++ .github/workflows/ci.yml | 288 ++++---------------- tests/benchmarks/http.js | 327 ++++++----------------- 3 files changed, 486 insertions(+), 484 deletions(-) create mode 100644 .github/workflows/benchmark-comment.js diff --git a/.github/workflows/benchmark-comment.js b/.github/workflows/benchmark-comment.js new file mode 100644 index 0000000000..2294611318 --- /dev/null +++ b/.github/workflows/benchmark-comment.js @@ -0,0 +1,355 @@ +const fs = require('fs'); + +const marker = ''; +const serviceLabels = ['Account', 'TablesDB', 'Storage', 'Functions', 'Sites', 'Health']; + +module.exports = async ({ github, context, core }) => { + const body = buildComment(core); + fs.writeFileSync('benchmark-comment.txt', body); + + const pullRequest = context.payload.pull_request; + if (!pullRequest || pullRequest.head.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) { + return; + } + + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequest.number, + per_page: 100, + }); + + const existing = comments.find((comment) => { + return comment.user?.type === 'Bot' && comment.body?.includes(marker); + }) || comments.find((comment) => { + return comment.user?.type === 'Bot' && comment.body?.includes('Benchmark results'); + }); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + return; + } + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequest.number, + body, + }); +}; + +function buildComment(core) { + const before = readSummary('benchmark-before-summary.json', core); + const after = readSummary('benchmark-after-summary.json', core); + const beforeSamples = readSamples('benchmark-before-samples.json', core); + const afterSamples = readSamples('benchmark-after-samples.json', core); + const baseRef = markdownText(process.env.BENCHMARK_BASE_REF || 'base'); + const headRef = markdownText(process.env.BENCHMARK_HEAD_REF || 'head'); + const rows = benchmarkRows(before, after, beforeSamples, afterSamples); + const topWaits = topSamples(afterSamples, 'appwrite_http_waiting', 3); + const lines = [ + marker, + '## :sparkles: Benchmark results', + '', + `Comparing ${baseRef} (before) to ${headRef} (after).`, + '', + ]; + + if (before === null) { + lines.push('> Before benchmark did not complete; showing current branch metrics only.', ''); + } + if (after === null) { + lines.push('> Current branch benchmark did not complete; showing available metrics only.', ''); + } + + lines.push( + '| Scenario | Before P50 (ms) | Before P95 (ms) | After P50 (ms) | After P95 (ms) | Delta P95 (ms) | After iterations | After RPS |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', + ...rows.map(comparisonRow), + '', + '
', + 'Current run details', + '', + '
', + '', + '| Scenario | P50 (ms) | P95 (ms) | Iterations | RPS |', + '| --- | ---: | ---: | ---: | ---: |', + ...rows.map(detailRow), + '', + '**Top 3 request waits**', + '', + '| Request | Max wait (ms) |', + '| --- | ---: |', + ...topWaitRows(topWaits), + '', + '
', + ); + + return `${lines.join('\n')}\n`; +} + +function readSummary(path, core) { + if (!fs.existsSync(path)) { + return null; + } + + try { + return JSON.parse(fs.readFileSync(path, 'utf8')); + } catch (error) { + core?.warning(`Invalid benchmark summary ${path}: ${error.message}`); + return null; + } +} + +function readSamples(path, core) { + if (!fs.existsSync(path)) { + return []; + } + + const contents = fs.readFileSync(path, 'utf8').trim(); + if (contents === '') { + return []; + } + + return contents + .split('\n') + .filter(Boolean) + .flatMap((line) => { + try { + return [JSON.parse(line)]; + } catch (error) { + core?.warning(`Invalid benchmark sample in ${path}: ${error.message}`); + return []; + } + }); +} + +function benchmarkRows(before, after, beforeSamples, afterSamples) { + const beforeServices = serviceStats(beforeSamples); + const afterServices = serviceStats(afterSamples); + return [ + { + label: 'Load test', + before: summaryStats(before, 'appwrite_http_duration', 'iterations', 'http_reqs'), + after: summaryStats(after, 'appwrite_http_duration', 'iterations', 'http_reqs'), + }, + { + label: 'API total', + before: apiSampleStats(beforeSamples) || summaryStats(before, 'appwrite_api_duration'), + after: apiSampleStats(afterSamples) || summaryStats(after, 'appwrite_api_duration'), + }, + ...serviceLabels.map((label) => ({ + label, + before: beforeServices.get(label) || null, + after: afterServices.get(label) || null, + })), + { + label: 'TablesDB schema', + before: summaryStats(before, 'appwrite_worker_tables_duration'), + after: summaryStats(after, 'appwrite_worker_tables_duration'), + }, + { + label: 'Mail delivery', + before: summaryStats(before, 'appwrite_worker_mails_duration'), + after: summaryStats(after, 'appwrite_worker_mails_duration'), + }, + ]; +} + +function summaryStats(summary, durationMetric, iterationsMetric = null, rpsMetric = null) { + const values = metricValues(summary, durationMetric); + if (!values) { + return null; + } + + return { + p50: values.med ?? null, + p95: values['p(95)'] ?? null, + iterations: iterationsMetric ? metricValue(summary, iterationsMetric, 'count') : values.count ?? null, + rps: rpsMetric ? metricValue(summary, rpsMetric, 'rate') : null, + }; +} + +function serviceStats(samples) { + const apiSamples = samples.filter((sample) => { + return sample.metric === 'appwrite_api_duration' && typeof sample.data?.value === 'number'; + }); + const durationSeconds = sampleWindowSeconds(apiSamples); + const groups = new Map(); + + for (const sample of apiSamples) { + const service = serviceFromName(sample.data.tags?.name || ''); + if (!service) { + continue; + } + + const values = groups.get(service) || []; + values.push(sample.data.value); + groups.set(service, values); + } + + return new Map([...groups.entries()].map(([service, values]) => { + return [service, { + p50: percentile(values, 50), + p95: percentile(values, 95), + iterations: values.length, + rps: durationSeconds ? values.length / durationSeconds : null, + }]; + })); +} + +function apiSampleStats(samples) { + const values = samples + .filter((sample) => sample.metric === 'appwrite_api_duration' && typeof sample.data?.value === 'number') + .map((sample) => sample.data.value); + if (values.length === 0) { + return null; + } + + const durationSeconds = sampleWindowSeconds(samples); + return { + p50: percentile(values, 50), + p95: percentile(values, 95), + iterations: values.length, + rps: durationSeconds ? values.length / durationSeconds : null, + }; +} + +function serviceFromName(name) { + if (name.startsWith('account.')) { + return 'Account'; + } + if (name.startsWith('tablesdb.')) { + return 'TablesDB'; + } + if (name.startsWith('storage.') || name.startsWith('tokens.')) { + return 'Storage'; + } + if (name.startsWith('functions.')) { + return 'Functions'; + } + if (name.startsWith('sites.')) { + return 'Sites'; + } + if (name.startsWith('health.')) { + return 'Health'; + } + return null; +} + +function sampleWindowSeconds(samples) { + const times = samples + .map((sample) => Date.parse(sample.data?.time)) + .filter((value) => !Number.isNaN(value)); + if (times.length < 2) { + return null; + } + + return Math.max((Math.max(...times) - Math.min(...times)) / 1000, 1); +} + +function percentile(values, percentileValue) { + if (values.length === 0) { + return null; + } + + const sorted = [...values].sort((left, right) => left - right); + const index = Math.ceil((percentileValue / 100) * sorted.length) - 1; + return sorted[Math.max(0, Math.min(index, sorted.length - 1))]; +} + +function metricValues(data, metric) { + return data?.metrics?.[metric]?.values ?? null; +} + +function metricValue(data, metric, stat) { + return metricValues(data, metric)?.[stat] ?? null; +} + +function comparisonRow(row) { + return `| ${row.label} | ${formatMs(row.before?.p50)} | ${formatMs(row.before?.p95)} | ${formatMs(row.after?.p50)} | ${formatMs(row.after?.p95)} | ${formatDelta(row.before?.p95, row.after?.p95)} | ${formatCount(row.after?.iterations)} | ${formatRate(row.after?.rps)} |`; +} + +function detailRow(row) { + return `| ${row.label} | ${formatMs(row.after?.p50)} | ${formatMs(row.after?.p95)} | ${formatCount(row.after?.iterations)} | ${formatRate(row.after?.rps)} |`; +} + +function topSamples(samples, metric, limit) { + const byName = samples.reduce((result, sample) => { + if (sample.metric !== metric || typeof sample.data?.value !== 'number') { + return result; + } + + const name = sample.data.tags?.name || 'unknown'; + const current = result.get(name); + if (!current || sample.data.value > current.value) { + result.set(name, { name, value: sample.data.value }); + } + + return result; + }, new Map()); + + return [...byName.values()] + .sort((left, right) => right.value - left.value) + .slice(0, limit); +} + +function topWaitRows(samples) { + if (samples.length === 0) { + return ['| n/a | n/a |']; + } + + return samples.map((sample) => { + return `| ${markdownText(sample.name).replace(/\|/g, '\\|')} | ${formatMs(sample.value)} |`; + }); +} + +function markdownText(value) { + return String(value || '').replace(/[\r\n]/g, ' ').replace(/[&<>"']/g, (char) => { + return ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[char]; + }); +} + +function formatMs(value) { + return formatNumber(value, 2); +} + +function formatRate(value) { + return formatNumber(value, 2); +} + +function formatCount(value) { + if (value === null || value === undefined || Number.isNaN(value)) { + return 'n/a'; + } + + return `${Math.round(value)}`; +} + +function formatDelta(before, after) { + if (before === null || before === undefined || after === null || after === undefined || Number.isNaN(before) || Number.isNaN(after)) { + return 'n/a'; + } + + const difference = Number((after - before).toFixed(2)); + return `${difference > 0 ? '+' : ''}${trimNumber(difference)}`; +} + +function formatNumber(value, decimals) { + if (value === null || value === undefined || Number.isNaN(value)) { + return 'n/a'; + } + + return trimNumber(Number(value).toFixed(decimals)); +} + +function trimNumber(value) { + const text = String(value); + const trimmed = text.includes('.') ? text.replace(/\.?0+$/, '') : text; + return trimmed === '' ? '0' : trimmed; +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 802648b0ea..b5f5fc79b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ concurrency: env: COMPOSE_FILE: docker-compose.yml IMAGE: appwrite-dev - K6_IMAGE: grafana/k6:0.53.0 + K6_VERSION: '0.53.0' on: pull_request: @@ -683,6 +683,11 @@ jobs: docker load --input /tmp/${{ env.IMAGE }}.tar docker tag ${{ env.IMAGE }} ${{ env.IMAGE }}:after + - name: Setup k6 + uses: grafana/setup-k6-action@v1 + with: + k6-version: ${{ env.K6_VERSION }} + - name: Prepare benchmark before id: benchmark_before_prepare continue-on-error: true @@ -703,27 +708,34 @@ jobs: if: steps.benchmark_before_prepare.outcome == 'success' continue-on-error: true working-directory: /tmp/appwrite-benchmark-before + env: + _APP_DOMAIN: localhost + _APP_CONSOLE_DOMAIN: localhost + _APP_DOMAIN_FUNCTIONS: functions.localhost + _APP_DOMAIN_SITES: sites.localhost run: | docker tag ${{ env.IMAGE }}:before ${{ env.IMAGE }} - sed -i 's/traefik/localhost/g' .env docker compose up -d --wait --no-build + - name: Prepare benchmark files + run: rm -f benchmark-before-summary.json benchmark-after-summary.json benchmark-before-samples.json benchmark-after-samples.json + - name: Benchmark before if: steps.benchmark_before_start.outcome == 'success' continue-on-error: true - run: | - set -o pipefail - rm -f benchmark-before-summary.json benchmark-after-summary.json benchmark-before-samples.json benchmark-after-samples.json benchmark-before.txt benchmark.txt - # Use the current benchmark script for both images so before/after differ only by the Appwrite image. - docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts ${{ env.K6_IMAGE }} run --quiet \ - --out json=benchmark-before-samples.json \ - -e APPWRITE_ENDPOINT=http://localhost/v1 \ - -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ - -e APPWRITE_BENCHMARK_ITERATIONS=1 \ - -e APPWRITE_BENCHMARK_VUS=1 \ - -e APPWRITE_WORKER_TIMEOUT_MS=120000 \ - -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-before-summary.json \ - tests/benchmarks/http.js | tee benchmark-before.txt + uses: grafana/run-k6-action@v1 + env: + APPWRITE_ENDPOINT: 'http://localhost/v1' + APPWRITE_MAILDEV_ENDPOINT: 'http://localhost:9503/email' + APPWRITE_BENCHMARK_ITERATIONS: '1' + APPWRITE_BENCHMARK_VUS: '1' + APPWRITE_WORKER_TIMEOUT_MS: '120000' + APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-before-summary.json' + with: + path: tests/benchmarks/http.js + flags: --quiet --out json=benchmark-before-samples.json + cloud-comment-on-pr: false + debug: true - name: Stop before Appwrite if: always() @@ -734,211 +746,47 @@ jobs: fi - name: Start after Appwrite + env: + _APP_DOMAIN: localhost + _APP_CONSOLE_DOMAIN: localhost + _APP_DOMAIN_FUNCTIONS: functions.localhost + _APP_DOMAIN_SITES: sites.localhost run: | docker tag ${{ env.IMAGE }}:after ${{ env.IMAGE }} - sed -i 's/traefik/localhost/g' .env docker compose up -d --wait --no-build - name: Benchmark after id: benchmark_after continue-on-error: true - run: | - set -o pipefail - docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts ${{ env.K6_IMAGE }} run --quiet \ - --out json=benchmark-after-samples.json \ - -e APPWRITE_ENDPOINT=http://localhost/v1 \ - -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ - -e APPWRITE_BENCHMARK_ITERATIONS=1 \ - -e APPWRITE_BENCHMARK_VUS=1 \ - -e APPWRITE_WORKER_TIMEOUT_MS=120000 \ - -e APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH=benchmark-before-summary.json \ - -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-after-summary.json \ - tests/benchmarks/http.js | tee benchmark.txt + uses: grafana/run-k6-action@v1 + env: + APPWRITE_ENDPOINT: 'http://localhost/v1' + APPWRITE_MAILDEV_ENDPOINT: 'http://localhost:9503/email' + APPWRITE_BENCHMARK_ITERATIONS: '1' + APPWRITE_BENCHMARK_VUS: '1' + APPWRITE_WORKER_TIMEOUT_MS: '120000' + APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH: 'benchmark-before-summary.json' + APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-after-summary.json' + with: + path: tests/benchmarks/http.js + flags: --quiet --out json=benchmark-after-samples.json + cloud-comment-on-pr: false + debug: true - name: Stop after Appwrite if: always() run: docker compose down -v || true - - name: Prepare comment + - name: Comment on PR + if: always() + uses: actions/github-script@v8 env: BENCHMARK_BASE_REF: ${{ github.event.pull_request.base.ref }} BENCHMARK_HEAD_REF: ${{ github.event.pull_request.head.ref }} - run: | - node <<'NODE' > benchmark-comment.txt - const fs = require('fs'); - - function readSummary(path, required = true) { - if (!fs.existsSync(path)) { - if (required) { - throw new Error(`Missing benchmark summary: ${path}`); - } - return null; - } - - try { - return JSON.parse(fs.readFileSync(path, 'utf8')); - } catch (error) { - if (required) { - throw new Error(`Invalid benchmark summary ${path}: ${error.message}`); - } - console.error(`Invalid benchmark summary ${path}: ${error.message}`); - return null; - } - } - - function markdownText(value) { - return String(value || '').replace(/[\r\n]/g, ' ').replace(/[&<>"']/g, (char) => { - return ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[char]; - }); - } - - function metricValue(data, metric, stat) { - return data?.metrics?.[metric]?.values?.[stat] ?? null; - } - - function formatNumber(value) { - return Number(value).toFixed(2).replace(/\.?0+$/, ''); - } - - function formatValue(value, suffix = '') { - return value === null ? 'n/a' : `${formatNumber(value)}${suffix}`; - } - - function delta(beforeValue, afterValue, suffix = '') { - if (beforeValue === null || afterValue === null) { - return 'n/a'; - } - - const difference = Number((afterValue - beforeValue).toFixed(2)); - return `${difference > 0 ? '+' : ''}${formatNumber(difference)}${suffix}`; - } - - function row(label, beforeValue, afterValue, suffix = '') { - return `| ${label} | ${formatValue(beforeValue, suffix)} | ${formatValue(afterValue, suffix)} | ${delta(beforeValue, afterValue, suffix)} |`; - } - - function detailValue(value, suffix = '') { - return value === null ? 'n/a' : `${Number(value).toFixed(2)}${suffix}`; - } - - function detailRow(after, label, metric, suffix = 'ms') { - const values = after?.metrics?.[metric]?.values; - if (!values) { - return `| ${label} | n/a | n/a | n/a | n/a |`; - } - - return `| ${label} | ${detailValue(values.avg ?? null, suffix)} | ${detailValue(values['p(90)'] ?? null, suffix)} | ${detailValue(values['p(95)'] ?? null, suffix)} | ${detailValue(values.max ?? null, suffix)} |`; - } - - function readSamples(path) { - if (!fs.existsSync(path)) { - return []; - } - - const contents = fs.readFileSync(path, 'utf8').trim(); - if (contents === '') { - return []; - } - - return contents - .split('\n') - .filter(Boolean) - .flatMap((line) => { - try { - return [JSON.parse(line)]; - } catch { - return []; - } - }); - } - - function topSamples(samples, metric, limit) { - const byName = samples.reduce((result, sample) => { - if (sample.metric !== metric || typeof sample.data?.value !== 'number') { - return result; - } - - const name = sample.data.tags?.name || 'unknown'; - const current = result.get(name); - if (!current || sample.data.value > current.value) { - result.set(name, { name, value: sample.data.value }); - } - - return result; - }, new Map()); - - return [...byName.values()] - .sort((left, right) => right.value - left.value) - .slice(0, limit); - } - - function topSampleRows(samples) { - if (samples.length === 0) { - return ['| n/a | n/a |']; - } - - return samples.map((sample) => { - const name = markdownText(sample.name).replace(/\|/g, '\\|'); - return `| ${name} | ${detailValue(sample.value, 'ms')} |`; - }); - } - - const before = readSummary('benchmark-before-summary.json', false); - const after = readSummary('benchmark-after-summary.json', false); - const afterSamples = readSamples('benchmark-after-samples.json'); - const baseRef = markdownText(process.env.BENCHMARK_BASE_REF || 'base'); - const headRef = markdownText(process.env.BENCHMARK_HEAD_REF || 'head'); - const topWaits = topSamples(afterSamples, 'appwrite_http_waiting', 3); - - const rows = [ - row('HTTP throughput', metricValue(before, 'http_reqs', 'rate'), metricValue(after, 'http_reqs', 'rate'), ' req/s'), - row('HTTP total p95', metricValue(before, 'appwrite_http_duration', 'p(95)'), metricValue(after, 'appwrite_http_duration', 'p(95)'), 'ms'), - row('API endpoints p95', metricValue(before, 'appwrite_api_duration', 'p(95)'), metricValue(after, 'appwrite_api_duration', 'p(95)'), 'ms'), - row('Database worker p95', metricValue(before, 'appwrite_worker_database_duration', 'p(95)'), metricValue(after, 'appwrite_worker_database_duration', 'p(95)'), 'ms'), - row('TablesDB worker p95', metricValue(before, 'appwrite_worker_tables_duration', 'p(95)'), metricValue(after, 'appwrite_worker_tables_duration', 'p(95)'), 'ms'), - row('Mail worker p95', metricValue(before, 'appwrite_worker_mails_duration', 'p(95)'), metricValue(after, 'appwrite_worker_mails_duration', 'p(95)'), 'ms'), - row('Messaging worker p95', metricValue(before, 'appwrite_worker_messaging_duration', 'p(95)'), metricValue(after, 'appwrite_worker_messaging_duration', 'p(95)'), 'ms'), - ]; - - console.log(''); - console.log('## :sparkles: Benchmark results'); - console.log(); - console.log(`Comparing ${baseRef} (before) to ${headRef} (after).`); - console.log(); - if (before === null) { - console.log('> Before benchmark did not complete; showing current branch metrics only.'); - console.log(); - } - if (after === null) { - console.log('> Current branch benchmark did not complete; showing available metrics only.'); - console.log(); - } - console.log('| Metric | Before | After | Delta |'); - console.log('| --- | ---: | ---: | ---: |'); - console.log(rows.join('\n')); - console.log(); - console.log('
'); - console.log('Current run details'); - console.log(); - console.log('
'); - console.log(); - console.log('| Scenario | Avg | P90 | P95 | Max |'); - console.log('| --- | ---: | ---: | ---: | ---: |'); - console.log(detailRow(after, 'HTTP total', 'appwrite_http_duration')); - console.log(detailRow(after, 'API endpoints', 'appwrite_api_duration')); - console.log(detailRow(after, 'Database worker schema jobs', 'appwrite_worker_database_duration')); - console.log(detailRow(after, 'TablesDB worker schema jobs', 'appwrite_worker_tables_duration')); - console.log(detailRow(after, 'Mail worker delivery', 'appwrite_worker_mails_duration')); - console.log(detailRow(after, 'Messaging worker delivery', 'appwrite_worker_messaging_duration')); - console.log(); - console.log('**Top 3 request waits**'); - console.log(); - console.log('| Request | Max wait |'); - console.log('| --- | ---: |'); - console.log(topSampleRows(topWaits).join('\n')); - console.log(); - console.log('
'); - NODE + with: + script: | + const comment = require('./.github/workflows/benchmark-comment.js'); + await comment({ github, context, core }); - name: Save results uses: actions/upload-artifact@v7 @@ -946,41 +794,13 @@ jobs: with: name: benchmark-results path: | - benchmark-before.txt - benchmark.txt + benchmark-comment.txt benchmark-before-summary.json benchmark-after-summary.json benchmark-before-samples.json benchmark-after-samples.json retention-days: 7 - - name: Find Comment - if: github.event.pull_request.head.repo.full_name == github.repository - uses: peter-evans/find-comment@v3 - id: fc - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: appwrite-benchmark-results - - - name: Find Legacy Comment - if: github.event.pull_request.head.repo.full_name == github.repository && steps.fc.outputs.comment-id == '' - uses: peter-evans/find-comment@v3 - id: legacy_fc - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: Benchmark results - - - name: Comment on PR - if: github.event.pull_request.head.repo.full_name == github.repository - uses: peter-evans/create-or-update-comment@v4 - with: - comment-id: ${{ steps.fc.outputs.comment-id || steps.legacy_fc.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - body-path: benchmark-comment.txt - edit-mode: replace - - name: Fail benchmark if: always() && steps.benchmark_after.outcome == 'failure' run: exit 1 diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index ef873f6688..c6248dd3b0 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -3,22 +3,16 @@ * * docker compose up -d --force-recreate --build --wait * - * docker run --rm -i \ - * --network appwrite \ - * -p 127.0.0.1:5665:5665 \ - * -v "$PWD:/scripts:ro" \ - * -v /tmp:/host-tmp \ - * -w /scripts \ - * -e K6_WEB_DASHBOARD=true \ - * -e K6_WEB_DASHBOARD_HOST=0.0.0.0 \ - * -e K6_WEB_DASHBOARD_PORT=5665 \ - * -e K6_WEB_DASHBOARD_EXPORT=/host-tmp/appwrite-k6-report.html \ - * -e APPWRITE_ENDPOINT=http://appwrite/v1 \ - * -e APPWRITE_MAILDEV_ENDPOINT=http://maildev:1080/email \ - * -e APPWRITE_WORKER_TIMEOUT_MS=120000 \ - * -e APPWRITE_BENCHMARK_SUMMARY_PATH=/host-tmp/appwrite-k6-summary.json \ - * grafana/k6:0.53.0 run \ - * --out json=/host-tmp/appwrite-k6-samples.json \ + * K6_WEB_DASHBOARD=true \ + * K6_WEB_DASHBOARD_HOST=127.0.0.1 \ + * K6_WEB_DASHBOARD_PORT=5665 \ + * K6_WEB_DASHBOARD_EXPORT=/tmp/appwrite-k6-report.html \ + * APPWRITE_ENDPOINT=http://localhost/v1 \ + * APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ + * APPWRITE_WORKER_TIMEOUT_MS=120000 \ + * APPWRITE_BENCHMARK_SUMMARY_PATH=/tmp/appwrite-k6-summary.json \ + * k6 run \ + * --out json=/tmp/appwrite-k6-samples.json \ * tests/benchmarks/http.js * * Open http://127.0.0.1:5665 while the benchmark is running. @@ -45,10 +39,8 @@ const PREVIOUS_SUMMARY = loadPreviousSummary(); export const httpDuration = new Trend('appwrite_http_duration', true); export const httpWaiting = new Trend('appwrite_http_waiting', true); export const apiDuration = new Trend('appwrite_api_duration', true); -export const databaseWorkerDuration = new Trend('appwrite_worker_database_duration', true); export const tablesWorkerDuration = new Trend('appwrite_worker_tables_duration', true); export const mailsWorkerDuration = new Trend('appwrite_worker_mails_duration', true); -export const messagingWorkerDuration = new Trend('appwrite_worker_messaging_duration', true); export const flowFailures = new Counter('appwrite_benchmark_flow_failures'); export const options = { @@ -105,16 +97,6 @@ const API_SCOPES = [ 'locale.read', 'avatars.read', 'health.read', - 'providers.read', - 'providers.write', - 'messages.read', - 'messages.write', - 'topics.read', - 'topics.write', - 'subscribers.read', - 'subscribers.write', - 'targets.read', - 'targets.write', 'rules.read', 'rules.write', 'migrations.read', @@ -174,13 +156,13 @@ export function setup() { Cookie: cookieHeader(session), }; - const team = api('POST', '/teams', { + const team = setupApi('POST', '/teams', { teamId: unique('team'), name: `Benchmark Team ${runId}`, }, consoleSessionHeaders, [201], 'setup.teams.create'); const teamId = team.json('$id'); - const project = api('POST', '/projects', { + const project = setupApi('POST', '/projects', { projectId: unique('project'), name: `Benchmark Project ${runId}`, teamId, @@ -188,7 +170,7 @@ export function setup() { }, consoleSessionHeaders, [201], 'setup.projects.create'); const projectId = project.json('$id'); - const key = api('POST', `/projects/${projectId}/keys`, { + const key = setupApi('POST', `/projects/${projectId}/keys`, { keyId: unique('key'), name: 'Benchmark API key', scopes: API_SCOPES, @@ -200,7 +182,7 @@ export function setup() { 'X-Appwrite-Key': key.json('secret'), }; - const platform = api('POST', '/project/platforms/web', { + const platform = setupApi('POST', '/project/platforms/web', { platformId: unique('web'), name: 'Benchmark web', hostname: hostnameFromUrl(REDIRECT_URL), @@ -236,11 +218,9 @@ export function curatedFlows(data) { const ctx = { ...data }; try { - group('account and mail worker', () => accountFlow(ctx)); - group('databases documents flow', () => databasesFlow(ctx)); + group('account and mail flow', () => accountFlow(ctx)); group('tablesdb rows flow', () => tablesDbFlow(ctx)); group('storage files and tokens flow', () => storageFlow(ctx)); - group('messaging worker flow', () => messagingFlow(ctx)); group('functions and sites control-plane flow', () => computeFlow(ctx)); group('health and queue probes', () => healthFlow(ctx)); } catch (error) { @@ -360,73 +340,6 @@ function accountFlow(ctx) { } } -function databasesFlow(ctx) { - const databaseId = unique('db'); - const collectionId = unique('col'); - const documentId = unique('doc'); - const indexKey = unique('idx'); - - api('POST', '/databases', { databaseId, name: 'Benchmark DB' }, ctx.apiHeaders, [201], 'databases.create'); - api('POST', `/databases/${databaseId}/collections`, { - collectionId, - name: 'Benchmark Collection', - permissions: BASE_PERMISSIONS, - documentSecurity: false, - }, ctx.apiHeaders, [201], 'databases.collections.create'); - - const attributes = [ - ['string', 'title', { size: 128 }], - ['integer', 'count', { min: 0, max: 100000 }], - ['email', 'email', {}], - ['boolean', 'active', {}], - ['datetime', 'publishedAt', {}], - ['float', 'score', { min: 0, max: 1000 }], - ['url', 'url', {}], - ['ip', 'ip', {}], - ]; - - for (const [type, key, extra] of attributes) { - const started = Date.now(); - api('POST', `/databases/${databaseId}/collections/${collectionId}/attributes/${type}`, { - key, - required: false, - array: false, - ...extra, - }, ctx.apiHeaders, [202], `databases.attributes.${type}.create`); - waitForStatus(`/databases/${databaseId}/collections/${collectionId}/attributes/${key}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); - databaseWorkerDuration.add(Date.now() - started, { job: `attribute_${type}` }); - } - - const indexStarted = Date.now(); - api('POST', `/databases/${databaseId}/collections/${collectionId}/indexes`, { - key: indexKey, - type: 'key', - attributes: ['title'], - orders: ['asc'], - }, ctx.apiHeaders, [202], 'databases.indexes.create'); - waitForStatus(`/databases/${databaseId}/collections/${collectionId}/indexes/${indexKey}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); - databaseWorkerDuration.add(Date.now() - indexStarted, { job: 'index' }); - - api('POST', `/databases/${databaseId}/collections/${collectionId}/documents`, { - documentId, - data: documentPayload(), - permissions: ITEM_PERMISSIONS, - }, ctx.apiHeaders, [201], 'databases.documents.create'); - api('GET', `/databases/${databaseId}/collections/${collectionId}/documents`, null, ctx.apiHeaders, [200], 'databases.documents.list'); - api('GET', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}`, null, ctx.apiHeaders, [200], 'databases.documents.get'); - api('PATCH', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}`, { - data: { title: 'Benchmark Document Updated' }, - }, ctx.apiHeaders, [200], 'databases.documents.update'); - api('PATCH', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}/count/increment`, { - value: 1, - }, ctx.apiHeaders, [200], 'databases.documents.increment'); - api('PATCH', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}/count/decrement`, { - value: 1, - }, ctx.apiHeaders, [200], 'databases.documents.decrement'); - api('DELETE', `/databases/${databaseId}/collections/${collectionId}/documents/${documentId}`, null, ctx.apiHeaders, [204], 'databases.documents.delete'); - api('DELETE', `/databases/${databaseId}`, null, ctx.apiHeaders, [204], 'databases.delete'); -} - function tablesDbFlow(ctx) { requireSession(ctx, 'tablesDbFlow'); @@ -548,92 +461,6 @@ function storageFlow(ctx) { api('DELETE', `/storage/buckets/${bucketId}`, null, ctx.apiHeaders, [204], 'storage.buckets.delete'); } -function messagingFlow(ctx) { - requireSession(ctx, 'messagingFlow'); - if (!ctx.userId || !ctx.userEmail) { - throw new Error('accountFlow must run before messagingFlow'); - } - - const providerId = unique('smtp'); - let targetId = unique('target'); - const topicId = unique('topic'); - const subscriberId = unique('sub'); - const messageId = unique('msg'); - - api('POST', '/messaging/providers/smtp', { - providerId, - name: 'Benchmark SMTP', - host: __ENV.APPWRITE_SMTP_HOST || 'maildev', - port: Number(__ENV.APPWRITE_SMTP_PORT || 1025), - username: __ENV.APPWRITE_SMTP_USERNAME || 'user', - password: __ENV.APPWRITE_SMTP_PASSWORD || 'password', - encryption: __ENV.APPWRITE_SMTP_ENCRYPTION || 'none', - autoTLS: false, - fromName: 'Benchmark', - fromEmail: 'benchmark@appwrite.io', - replyToName: 'Benchmark', - replyToEmail: 'benchmark@appwrite.io', - enabled: true, - }, ctx.apiHeaders, [201], 'messaging.providers.smtp.create'); - - const targets = api('GET', `/users/${ctx.userId}/targets`, null, ctx.apiHeaders, [200], 'users.targets.list'); - const existingTarget = (targets.json('targets') || []).find((target) => { - return target.providerType === 'email' && target.identifier === ctx.userEmail; - }); - - if (existingTarget) { - targetId = existingTarget.$id; - api('PATCH', `/users/${ctx.userId}/targets/${targetId}`, { - providerId, - name: 'Benchmark email target', - }, ctx.apiHeaders, [200], 'users.targets.update'); - } else { - api('POST', `/users/${ctx.userId}/targets`, { - targetId, - providerType: 'email', - identifier: ctx.userEmail, - providerId, - name: 'Benchmark email target', - }, ctx.apiHeaders, [201], 'users.targets.create'); - } - - api('POST', '/messaging/topics', { - topicId, - name: 'Benchmark Topic', - subscribe: ['users'], - }, ctx.apiHeaders, [201], 'messaging.topics.create'); - - api('POST', `/messaging/topics/${topicId}/subscribers`, { - subscriberId, - targetId, - }, ctx.sessionHeaders, [201], 'messaging.subscribers.create'); - - const started = Date.now(); - api('POST', '/messaging/messages/email', { - messageId, - subject: `Benchmark message ${ctx.runId}`, - content: `Benchmark messaging worker probe ${ctx.runId}`, - targets: [targetId], - draft: false, - html: false, - }, ctx.apiHeaders, [201], 'messaging.messages.email.create'); - - waitForMessage(messageId, ctx.apiHeaders, WORKER_TIMEOUT_MS); - waitForEmail(ctx.userEmail, (message) => includes(message.subject, `Benchmark message ${ctx.runId}`), MAIL_TIMEOUT_MS, true); - messagingWorkerDuration.add(Date.now() - started, { job: 'email_message' }); - - api('GET', '/messaging/messages', null, ctx.apiHeaders, [200], 'messaging.messages.list'); - api('GET', `/messaging/messages/${messageId}/logs`, null, ctx.apiHeaders, [200], 'messaging.messages.logs.list'); - api('GET', `/messaging/messages/${messageId}/targets`, null, ctx.apiHeaders, [200], 'messaging.messages.targets.list'); - api('GET', `/messaging/providers/${providerId}/logs`, null, ctx.apiHeaders, [200], 'messaging.providers.logs.list'); - api('GET', `/messaging/topics/${topicId}/logs`, null, ctx.apiHeaders, [200], 'messaging.topics.logs.list'); - api('GET', `/messaging/subscribers/${subscriberId}/logs`, null, ctx.apiHeaders, [200], 'messaging.subscribers.logs.list'); - api('DELETE', `/messaging/topics/${topicId}/subscribers/${subscriberId}`, null, ctx.sessionHeaders, [204], 'messaging.subscribers.delete'); - api('DELETE', `/messaging/topics/${topicId}`, null, ctx.apiHeaders, [204], 'messaging.topics.delete'); - api('DELETE', `/messaging/messages/${messageId}`, null, ctx.apiHeaders, [204], 'messaging.messages.delete'); - api('DELETE', `/messaging/providers/${providerId}`, null, ctx.apiHeaders, [204], 'messaging.providers.delete'); -} - function computeFlow(ctx) { requireSession(ctx, 'computeFlow'); @@ -715,9 +542,7 @@ function healthFlow(ctx) { '/health/storage', '/health/storage/local', '/health/time', - '/health/queue/databases', '/health/queue/mails', - '/health/queue/messaging', '/health/queue/functions', '/health/queue/builds', '/health/queue/deletes', @@ -739,6 +564,12 @@ function api(method, path, body, headers, expected, name) { return response; } +function setupApi(method, path, body, headers, expected, name) { + const response = rawRequest(method, path, body, headers, name); + assertStatus(response, expected, name); + return response; +} + function rawRequest(method, path, body, headers, name) { const params = { headers, @@ -772,26 +603,6 @@ function waitForStatus(path, headers, wantedStatus, timeoutMs) { throw new Error(`Timed out waiting for ${path} to become ${wantedStatus}`); } -function waitForMessage(messageId, headers, timeoutMs) { - const started = Date.now(); - - while (Date.now() - started < timeoutMs) { - const response = rawRequest('GET', `/messaging/messages/${messageId}`, null, headers, 'messaging.messages.poll'); - const status = response.status === 200 ? response.json('status') : null; - - if (['sent', 'failed'].includes(status)) { - if (status === 'failed') { - throw new Error(`Messaging worker marked message ${messageId} as failed`); - } - return response; - } - - sleep(0.5); - } - - throw new Error(`Timed out waiting for messaging worker to send message ${messageId}`); -} - function waitForEmail(address, predicate, timeoutMs, allowMissingRecipient = false) { const started = Date.now(); @@ -890,19 +701,6 @@ function requireSession(ctx, flow) { } } -function documentPayload() { - return { - title: 'Benchmark Document', - count: 1, - email: 'document@example.com', - active: true, - publishedAt: new Date().toISOString(), - score: 10.5, - url: 'https://appwrite.io', - ip: '127.0.0.1', - }; -} - function tablePayload() { return { title: 'Benchmark Row', @@ -961,24 +759,27 @@ export function handleSummary(data) { function detailsTable(data) { return [ - '| Scenario | Avg | P90 | P95 | Max |', + '| Scenario | P50 (ms) | P95 (ms) | Iterations | RPS |', '| --- | ---: | ---: | ---: | ---: |', - detailRow(data, 'HTTP total', 'appwrite_http_duration'), - detailRow(data, 'API endpoints', 'appwrite_api_duration'), - detailRow(data, 'Database worker schema jobs', 'appwrite_worker_database_duration'), - detailRow(data, 'TablesDB worker schema jobs', 'appwrite_worker_tables_duration'), - detailRow(data, 'Mail worker delivery', 'appwrite_worker_mails_duration'), - detailRow(data, 'Messaging worker delivery', 'appwrite_worker_messaging_duration'), + detailRow(data, 'Load test', 'appwrite_http_duration', 'iterations', 'http_reqs'), + detailRow(data, 'API total', 'appwrite_api_duration'), + detailRow(data, 'TablesDB schema', 'appwrite_worker_tables_duration'), + detailRow(data, 'Mail delivery', 'appwrite_worker_mails_duration'), ].join('\n'); } -function detailRow(data, label, metric, unit = 'ms') { +function detailRow(data, label, metric, iterationsMetric = null, rpsMetric = null) { const values = data.metrics[metric] && data.metrics[metric].values; if (!values || values.count === 0) { return `| ${label} | n/a | n/a | n/a | n/a |`; } - return `| ${label} | ${formatDetailValue(values.avg, unit)} | ${formatDetailValue(values['p(90)'], unit)} | ${formatDetailValue(values['p(95)'], unit)} | ${formatDetailValue(values.max, unit)} |`; + const iterations = iterationsMetric + ? trendMetric(data, iterationsMetric, 'count') + : values.count; + const rps = rpsMetric ? trendMetric(data, rpsMetric, 'rate') : null; + + return `| ${label} | ${formatDetailValue(values.med)} | ${formatDetailValue(values['p(95)'])} | ${formatCount(iterations)} | ${formatRate(rps)} |`; } function loadPreviousSummary() { @@ -988,10 +789,19 @@ function loadPreviousSummary() { } for (const path of paths) { + let contents; try { - return JSON.parse(open(path)); + contents = open(path); } catch (error) { // Try the next path. k6 resolves open() relative to the script file. + continue; + } + + try { + return JSON.parse(contents); + } catch (error) { + console.warn(`Invalid benchmark summary at ${path}: ${error.message}`); + return null; } } @@ -1000,20 +810,21 @@ function loadPreviousSummary() { function comparisonTable(before, after) { const rows = [ - ['HTTP throughput', trendMetric(before, 'http_reqs', 'rate'), trendMetric(after, 'http_reqs', 'rate'), ' req/s'], - ['HTTP total p95', trendMetric(before, 'appwrite_http_duration', 'p(95)'), trendMetric(after, 'appwrite_http_duration', 'p(95)'), 'ms'], - ['API endpoints p95', trendMetric(before, 'appwrite_api_duration', 'p(95)'), trendMetric(after, 'appwrite_api_duration', 'p(95)'), 'ms'], - ['Database worker p95', trendMetric(before, 'appwrite_worker_database_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_database_duration', 'p(95)'), 'ms'], - ['TablesDB worker p95', trendMetric(before, 'appwrite_worker_tables_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_tables_duration', 'p(95)'), 'ms'], - ['Mail worker p95', trendMetric(before, 'appwrite_worker_mails_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_mails_duration', 'p(95)'), 'ms'], - ['Messaging worker p95', trendMetric(before, 'appwrite_worker_messaging_duration', 'p(95)'), trendMetric(after, 'appwrite_worker_messaging_duration', 'p(95)'), 'ms'], + ['Load test', 'appwrite_http_duration'], + ['API total', 'appwrite_api_duration'], + ['TablesDB schema', 'appwrite_worker_tables_duration'], + ['Mail delivery', 'appwrite_worker_mails_duration'], ]; return [ - '| Metric | Before | After | Delta |', - '| --- | ---: | ---: | ---: |', - ...rows.map(([label, beforeValue, afterValue, unit]) => { - return `| ${label} | ${formatValue(beforeValue, unit)} | ${formatValue(afterValue, unit)} | ${formatDelta(beforeValue, afterValue, unit)} |`; + '| Scenario | Before P50 (ms) | Before P95 (ms) | After P50 (ms) | After P95 (ms) | Delta P95 (ms) |', + '| --- | ---: | ---: | ---: | ---: | ---: |', + ...rows.map(([label, metric]) => { + const beforeP50 = trendMetric(before, metric, 'med'); + const beforeP95 = trendMetric(before, metric, 'p(95)'); + const afterP50 = trendMetric(after, metric, 'med'); + const afterP95 = trendMetric(after, metric, 'p(95)'); + return `| ${label} | ${formatValue(beforeP50)} | ${formatValue(beforeP95)} | ${formatValue(afterP50)} | ${formatValue(afterP95)} | ${formatDelta(beforeP95, afterP95)} |`; }), ].join('\n'); } @@ -1024,30 +835,46 @@ function trendMetric(data, metric, stat) { : null; } -function formatValue(value, unit) { +function formatValue(value) { if (value === null || value === undefined || Number.isNaN(value)) { return 'n/a'; } - return `${round(value)}${unit}`; + return `${round(value)}`; } -function formatDetailValue(value, unit) { +function formatDetailValue(value) { if (value === null || value === undefined || Number.isNaN(value)) { return 'n/a'; } - return `${Number(value).toFixed(2)}${unit}`; + return `${Number(value).toFixed(2)}`; } -function formatDelta(before, after, unit) { +function formatDelta(before, after) { if (before === null || before === undefined || after === null || after === undefined || Number.isNaN(before) || Number.isNaN(after)) { return 'n/a'; } const delta = round(after - before); const sign = delta > 0 ? '+' : ''; - return `${sign}${delta}${unit}`; + return `${sign}${delta}`; +} + +function formatCount(value) { + if (value === null || value === undefined || Number.isNaN(value)) { + return 'n/a'; + } + + return `${Math.round(value)}`; +} + +function formatRate(value) { + if (value === null || value === undefined || Number.isNaN(value)) { + return 'n/a'; + } + + return `${Number(value).toFixed(2)}`; } function round(value) { From 240cdf43e5cacc2f4f46de516280bbe9b45ca73f Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 15:24:47 +0530 Subject: [PATCH 31/39] Simplify local benchmark command --- tests/benchmarks/http.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index c6248dd3b0..7442a26220 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -1,7 +1,6 @@ /* * Run locally: - * - * docker compose up -d --force-recreate --build --wait + * Requires k6 and a running Appwrite instance. * * K6_WEB_DASHBOARD=true \ * K6_WEB_DASHBOARD_HOST=127.0.0.1 \ From 205c2839355b4ff441bf7881e52637e1c755169a Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 15:33:52 +0530 Subject: [PATCH 32/39] Remove unused JWT benchmark setup --- tests/benchmarks/http.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 7442a26220..970ceff4a5 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -264,12 +264,6 @@ function accountFlow(ctx) { ctx.userEmail = email; ctx.sessionHeaders = sessionHeaders; - const jwt = api('POST', '/account/jwts', null, sessionHeaders, [201], 'account.jwts.create'); - ctx.jwtHeaders = { - ...headers, - 'X-Appwrite-JWT': jwt.json('jwt'), - }; - api('GET', '/account', null, sessionHeaders, [200], 'account.get'); api('GET', '/account/logs', null, sessionHeaders, [200], 'account.logs.list'); api('PATCH', '/account/prefs', { prefs: { benchmark: true, runId: ctx.runId } }, sessionHeaders, [200], 'account.prefs.update'); @@ -331,11 +325,6 @@ function accountFlow(ctx) { Cookie: cookieHeader(recoveredSession), }; - const recoveredJwt = api('POST', '/account/jwts', null, ctx.sessionHeaders, [201], 'account.jwts.recovered'); - ctx.jwtHeaders = { - ...headers, - 'X-Appwrite-JWT': recoveredJwt.json('jwt'), - }; } } From 0f64f542219978d4a586d82ccf92c07e45c28cb7 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 15:49:27 +0530 Subject: [PATCH 33/39] Harden benchmark rerun metrics --- .github/workflows/benchmark-comment.js | 11 ++++++----- .github/workflows/ci.yml | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark-comment.js b/.github/workflows/benchmark-comment.js index 2294611318..1ccf1f69e8 100644 --- a/.github/workflows/benchmark-comment.js +++ b/.github/workflows/benchmark-comment.js @@ -179,7 +179,6 @@ function serviceStats(samples) { const apiSamples = samples.filter((sample) => { return sample.metric === 'appwrite_api_duration' && typeof sample.data?.value === 'number'; }); - const durationSeconds = sampleWindowSeconds(apiSamples); const groups = new Map(); for (const sample of apiSamples) { @@ -188,12 +187,14 @@ function serviceStats(samples) { continue; } - const values = groups.get(service) || []; - values.push(sample.data.value); - groups.set(service, values); + const serviceSamples = groups.get(service) || []; + serviceSamples.push(sample); + groups.set(service, serviceSamples); } - return new Map([...groups.entries()].map(([service, values]) => { + return new Map([...groups.entries()].map(([service, serviceSamples]) => { + const values = serviceSamples.map((sample) => sample.data.value); + const durationSeconds = sampleWindowSeconds(serviceSamples); return [service, { p50: percentile(values, 50), p95: percentile(values, 95), diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5f5fc79b2..a8e5cb38fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -745,6 +745,24 @@ jobs: docker compose down -v || true fi + - name: Wait for benchmark ports + if: always() + run: | + for port in 80 443 8080 9503; do + for attempt in $(seq 1 30); do + if ! ss -ltn | awk '{print $4}' | grep -Eq "[:.]${port}$"; then + break + fi + sleep 1 + done + + if ss -ltn | awk '{print $4}' | grep -Eq "[:.]${port}$"; then + echo "Port ${port} is still in use after stopping the before stack" + ss -ltn + exit 1 + fi + done + - name: Start after Appwrite env: _APP_DOMAIN: localhost From d1962dbc624e43a223d4c470318c03abb0d46376 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 16:45:16 +0530 Subject: [PATCH 34/39] Shorten local benchmark command --- .github/workflows/benchmark-comment.js | 8 +++--- tests/benchmarks/http-local.sh | 17 +++++++++++ tests/benchmarks/http.js | 40 ++++++++++++++------------ 3 files changed, 42 insertions(+), 23 deletions(-) create mode 100755 tests/benchmarks/http-local.sh diff --git a/.github/workflows/benchmark-comment.js b/.github/workflows/benchmark-comment.js index 1ccf1f69e8..4a9c920365 100644 --- a/.github/workflows/benchmark-comment.js +++ b/.github/workflows/benchmark-comment.js @@ -150,13 +150,13 @@ function benchmarkRows(before, after, beforeSamples, afterSamples) { })), { label: 'TablesDB schema', - before: summaryStats(before, 'appwrite_worker_tables_duration'), - after: summaryStats(after, 'appwrite_worker_tables_duration'), + before: summaryStats(before, 'appwrite_worker_tables_duration', 'appwrite_worker_tables_samples', 'appwrite_worker_tables_samples'), + after: summaryStats(after, 'appwrite_worker_tables_duration', 'appwrite_worker_tables_samples', 'appwrite_worker_tables_samples'), }, { label: 'Mail delivery', - before: summaryStats(before, 'appwrite_worker_mails_duration'), - after: summaryStats(after, 'appwrite_worker_mails_duration'), + before: summaryStats(before, 'appwrite_worker_mails_duration', 'appwrite_worker_mails_samples', 'appwrite_worker_mails_samples'), + after: summaryStats(after, 'appwrite_worker_mails_duration', 'appwrite_worker_mails_samples', 'appwrite_worker_mails_samples'), }, ]; } diff --git a/tests/benchmarks/http-local.sh b/tests/benchmarks/http-local.sh new file mode 100755 index 0000000000..acb8a07058 --- /dev/null +++ b/tests/benchmarks/http-local.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +export K6_WEB_DASHBOARD="${K6_WEB_DASHBOARD:-true}" +export K6_WEB_DASHBOARD_HOST="${K6_WEB_DASHBOARD_HOST:-127.0.0.1}" +export K6_WEB_DASHBOARD_PORT="${K6_WEB_DASHBOARD_PORT:-5665}" +export K6_WEB_DASHBOARD_EXPORT="${K6_WEB_DASHBOARD_EXPORT:-/tmp/appwrite-k6-report.html}" +export APPWRITE_ENDPOINT="${APPWRITE_ENDPOINT:-http://localhost/v1}" +export APPWRITE_MAILDEV_ENDPOINT="${APPWRITE_MAILDEV_ENDPOINT:-http://localhost:9503/email}" +export APPWRITE_WORKER_TIMEOUT_MS="${APPWRITE_WORKER_TIMEOUT_MS:-120000}" +export APPWRITE_BENCHMARK_SUMMARY_PATH="${APPWRITE_BENCHMARK_SUMMARY_PATH:-/tmp/appwrite-k6-summary.json}" + +samples_path="${APPWRITE_BENCHMARK_SAMPLES_PATH:-/tmp/appwrite-k6-samples.json}" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "${script_dir}/../.." && pwd)" + +exec k6 run --out "json=${samples_path}" "$@" "${repo_root}/tests/benchmarks/http.js" diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 970ceff4a5..3d541d074c 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -2,17 +2,7 @@ * Run locally: * Requires k6 and a running Appwrite instance. * - * K6_WEB_DASHBOARD=true \ - * K6_WEB_DASHBOARD_HOST=127.0.0.1 \ - * K6_WEB_DASHBOARD_PORT=5665 \ - * K6_WEB_DASHBOARD_EXPORT=/tmp/appwrite-k6-report.html \ - * APPWRITE_ENDPOINT=http://localhost/v1 \ - * APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \ - * APPWRITE_WORKER_TIMEOUT_MS=120000 \ - * APPWRITE_BENCHMARK_SUMMARY_PATH=/tmp/appwrite-k6-summary.json \ - * k6 run \ - * --out json=/tmp/appwrite-k6-samples.json \ - * tests/benchmarks/http.js + * tests/benchmarks/http-local.sh * * Open http://127.0.0.1:5665 while the benchmark is running. */ @@ -28,10 +18,10 @@ const REGION = __ENV.APPWRITE_REGION || 'default'; const REDIRECT_URL = __ENV.APPWRITE_BENCHMARK_REDIRECT_URL || 'http://localhost'; const PASSWORD = __ENV.APPWRITE_BENCHMARK_PASSWORD || 'Password123!'; const MAIL_TIMEOUT_MS = Number(__ENV.APPWRITE_MAIL_TIMEOUT_MS || 20000); -const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 60000); +const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 120000); const ITERATIONS = Number(__ENV.APPWRITE_BENCHMARK_ITERATIONS || 1); const VUS = Number(__ENV.APPWRITE_BENCHMARK_VUS || 1); -const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || 'tests/benchmarks/http-summary.json'; +const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || '/tmp/appwrite-k6-summary.json'; const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || SUMMARY_PATH; const PREVIOUS_SUMMARY = loadPreviousSummary(); @@ -40,6 +30,8 @@ export const httpWaiting = new Trend('appwrite_http_waiting', true); export const apiDuration = new Trend('appwrite_api_duration', true); export const tablesWorkerDuration = new Trend('appwrite_worker_tables_duration', true); export const mailsWorkerDuration = new Trend('appwrite_worker_mails_duration', true); +export const tablesWorkerSamples = new Counter('appwrite_worker_tables_samples'); +export const mailsWorkerSamples = new Counter('appwrite_worker_mails_samples'); export const flowFailures = new Counter('appwrite_benchmark_flow_failures'); export const options = { @@ -238,6 +230,16 @@ export function teardown(data) { } } +function recordTablesWorkerDuration(duration, tags) { + tablesWorkerDuration.add(duration, tags); + tablesWorkerSamples.add(1, tags); +} + +function recordMailsWorkerDuration(duration, tags) { + mailsWorkerDuration.add(duration, tags); + mailsWorkerSamples.add(1, tags); +} + function accountFlow(ctx) { const userId = unique('user'); const email = `bench-user-${unique('mail')}@example.com`; @@ -280,7 +282,7 @@ function accountFlow(ctx) { || includes(message.text, 'verify') || includes(message.text, 'verification'); }, MAIL_TIMEOUT_MS); - mailsWorkerDuration.add(Date.now() - verificationStarted, { job: 'email_verification' }); + recordMailsWorkerDuration(Date.now() - verificationStarted, { job: 'email_verification' }); const verification = extractQueryParams(verificationEmail); if (verification.userId && verification.secret) { @@ -303,7 +305,7 @@ function accountFlow(ctx) { || includes(message.text, 'recover') || includes(message.text, 'reset'); }, MAIL_TIMEOUT_MS); - mailsWorkerDuration.add(Date.now() - recoveryStarted, { job: 'password_recovery' }); + recordMailsWorkerDuration(Date.now() - recoveryStarted, { job: 'password_recovery' }); const recovery = extractQueryParams(recoveryEmail); if (recovery.userId && recovery.secret) { @@ -360,7 +362,7 @@ function tablesDbFlow(ctx) { ...extra, }, ctx.apiHeaders, [202], `tablesdb.columns.${type}.create`); waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/columns/${key}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); - tablesWorkerDuration.add(Date.now() - started, { job: `column_${type}` }); + recordTablesWorkerDuration(Date.now() - started, { job: `column_${type}` }); } const indexStarted = Date.now(); @@ -371,7 +373,7 @@ function tablesDbFlow(ctx) { orders: ['asc'], }, ctx.apiHeaders, [202], 'tablesdb.indexes.create'); waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/indexes/${indexKey}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); - tablesWorkerDuration.add(Date.now() - indexStarted, { job: 'index' }); + recordTablesWorkerDuration(Date.now() - indexStarted, { job: 'index' }); api('POST', `/tablesdb/${databaseId}/tables/${tableId}/rows`, { rowId, @@ -751,8 +753,8 @@ function detailsTable(data) { '| --- | ---: | ---: | ---: | ---: |', detailRow(data, 'Load test', 'appwrite_http_duration', 'iterations', 'http_reqs'), detailRow(data, 'API total', 'appwrite_api_duration'), - detailRow(data, 'TablesDB schema', 'appwrite_worker_tables_duration'), - detailRow(data, 'Mail delivery', 'appwrite_worker_mails_duration'), + detailRow(data, 'TablesDB schema', 'appwrite_worker_tables_duration', 'appwrite_worker_tables_samples', 'appwrite_worker_tables_samples'), + detailRow(data, 'Mail delivery', 'appwrite_worker_mails_duration', 'appwrite_worker_mails_samples', 'appwrite_worker_mails_samples'), ].join('\n'); } From c97435d95c29ea846806cbc82ac1ff928a7fd3e4 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 16:53:35 +0530 Subject: [PATCH 35/39] Stabilize benchmark wait metric tags --- tests/benchmarks/http.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 3d541d074c..3592d0248e 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -361,7 +361,7 @@ function tablesDbFlow(ctx) { array: false, ...extra, }, ctx.apiHeaders, [202], `tablesdb.columns.${type}.create`); - waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/columns/${key}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); + waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/columns/${key}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS, `tablesdb.columns.${type}.wait`); recordTablesWorkerDuration(Date.now() - started, { job: `column_${type}` }); } @@ -372,7 +372,7 @@ function tablesDbFlow(ctx) { columns: ['title'], orders: ['asc'], }, ctx.apiHeaders, [202], 'tablesdb.indexes.create'); - waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/indexes/${indexKey}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS); + waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/indexes/${indexKey}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS, 'tablesdb.indexes.wait'); recordTablesWorkerDuration(Date.now() - indexStarted, { job: 'index' }); api('POST', `/tablesdb/${databaseId}/tables/${tableId}/rows`, { @@ -573,11 +573,11 @@ function rawRequest(method, path, body, headers, name) { return response; } -function waitForStatus(path, headers, wantedStatus, timeoutMs) { +function waitForStatus(path, headers, wantedStatus, timeoutMs, name) { const started = Date.now(); while (Date.now() - started < timeoutMs) { - const response = rawRequest('GET', path, null, headers, `wait${path}`); + const response = rawRequest('GET', path, null, headers, name); if (response.status === 200) { const status = response.json('status'); if (status === wantedStatus) { From 7b25d778d4ce6713c42c308cbda102edf88fa72f Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 19:21:51 +0530 Subject: [PATCH 36/39] Trim benchmark scenarios --- .github/workflows/benchmark-comment.js | 77 +++--- .github/workflows/ci.yml | 4 - tests/benchmarks/http-local.sh | 1 - tests/benchmarks/http.js | 360 +++++-------------------- 4 files changed, 96 insertions(+), 346 deletions(-) diff --git a/.github/workflows/benchmark-comment.js b/.github/workflows/benchmark-comment.js index 4a9c920365..fa0fea87f4 100644 --- a/.github/workflows/benchmark-comment.js +++ b/.github/workflows/benchmark-comment.js @@ -1,7 +1,7 @@ const fs = require('fs'); const marker = ''; -const serviceLabels = ['Account', 'TablesDB', 'Storage', 'Functions', 'Sites', 'Health']; +const serviceLabels = ['Account', 'TablesDB', 'Storage', 'Functions']; module.exports = async ({ github, context, core }) => { const body = buildComment(core); @@ -51,7 +51,7 @@ function buildComment(core) { const baseRef = markdownText(process.env.BENCHMARK_BASE_REF || 'base'); const headRef = markdownText(process.env.BENCHMARK_HEAD_REF || 'head'); const rows = benchmarkRows(before, after, beforeSamples, afterSamples); - const topWaits = topSamples(afterSamples, 'appwrite_http_waiting', 3); + const topWaits = topSamples(afterSamples, 'appwrite_api_waiting', 3); const lines = [ marker, '## :sparkles: Benchmark results', @@ -68,22 +68,26 @@ function buildComment(core) { } lines.push( - '| Scenario | Before P50 (ms) | Before P95 (ms) | After P50 (ms) | After P95 (ms) | Delta P95 (ms) | After iterations | After RPS |', - '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', - ...rows.map(comparisonRow), + '**Before**', + '', + metricTable(rows, 'before'), + '', + '**After**', + '', + metricTable(rows, 'after'), + '', + '**Delta**', + '', + '| Scenario | P95 delta (ms) |', + '| --- | ---: |', + ...rows.map(deltaRow), '', '
', - 'Current run details', + 'Top API waits', '', '
', '', - '| Scenario | P50 (ms) | P95 (ms) | Iterations | RPS |', - '| --- | ---: | ---: | ---: | ---: |', - ...rows.map(detailRow), - '', - '**Top 3 request waits**', - '', - '| Request | Max wait (ms) |', + '| API request | Max wait (ms) |', '| --- | ---: |', ...topWaitRows(topWaits), '', @@ -133,11 +137,6 @@ function benchmarkRows(before, after, beforeSamples, afterSamples) { const beforeServices = serviceStats(beforeSamples); const afterServices = serviceStats(afterSamples); return [ - { - label: 'Load test', - before: summaryStats(before, 'appwrite_http_duration', 'iterations', 'http_reqs'), - after: summaryStats(after, 'appwrite_http_duration', 'iterations', 'http_reqs'), - }, { label: 'API total', before: apiSampleStats(beforeSamples) || summaryStats(before, 'appwrite_api_duration'), @@ -148,16 +147,6 @@ function benchmarkRows(before, after, beforeSamples, afterSamples) { before: beforeServices.get(label) || null, after: afterServices.get(label) || null, })), - { - label: 'TablesDB schema', - before: summaryStats(before, 'appwrite_worker_tables_duration', 'appwrite_worker_tables_samples', 'appwrite_worker_tables_samples'), - after: summaryStats(after, 'appwrite_worker_tables_duration', 'appwrite_worker_tables_samples', 'appwrite_worker_tables_samples'), - }, - { - label: 'Mail delivery', - before: summaryStats(before, 'appwrite_worker_mails_duration', 'appwrite_worker_mails_samples', 'appwrite_worker_mails_samples'), - after: summaryStats(after, 'appwrite_worker_mails_duration', 'appwrite_worker_mails_samples', 'appwrite_worker_mails_samples'), - }, ]; } @@ -205,14 +194,15 @@ function serviceStats(samples) { } function apiSampleStats(samples) { - const values = samples - .filter((sample) => sample.metric === 'appwrite_api_duration' && typeof sample.data?.value === 'number') - .map((sample) => sample.data.value); + const apiSamples = samples.filter((sample) => { + return sample.metric === 'appwrite_api_duration' && typeof sample.data?.value === 'number'; + }); + const values = apiSamples.map((sample) => sample.data.value); if (values.length === 0) { return null; } - const durationSeconds = sampleWindowSeconds(samples); + const durationSeconds = sampleWindowSeconds(apiSamples); return { p50: percentile(values, 50), p95: percentile(values, 95), @@ -234,12 +224,6 @@ function serviceFromName(name) { if (name.startsWith('functions.')) { return 'Functions'; } - if (name.startsWith('sites.')) { - return 'Sites'; - } - if (name.startsWith('health.')) { - return 'Health'; - } return null; } @@ -272,12 +256,21 @@ function metricValue(data, metric, stat) { return metricValues(data, metric)?.[stat] ?? null; } -function comparisonRow(row) { - return `| ${row.label} | ${formatMs(row.before?.p50)} | ${formatMs(row.before?.p95)} | ${formatMs(row.after?.p50)} | ${formatMs(row.after?.p95)} | ${formatDelta(row.before?.p95, row.after?.p95)} | ${formatCount(row.after?.iterations)} | ${formatRate(row.after?.rps)} |`; +function metricTable(rows, side) { + return [ + '| Scenario | P50 (ms) | P95 (ms) | Iterations | RPS |', + '| --- | ---: | ---: | ---: | ---: |', + ...rows.map((row) => metricRow(row, side)), + ].join('\n'); } -function detailRow(row) { - return `| ${row.label} | ${formatMs(row.after?.p50)} | ${formatMs(row.after?.p95)} | ${formatCount(row.after?.iterations)} | ${formatRate(row.after?.rps)} |`; +function metricRow(row, side) { + const values = row[side]; + return `| ${row.label} | ${formatMs(values?.p50)} | ${formatMs(values?.p95)} | ${formatCount(values?.iterations)} | ${formatRate(values?.rps)} |`; +} + +function deltaRow(row) { + return `| ${row.label} | ${formatDelta(row.before?.p95, row.after?.p95)} |`; } function topSamples(samples, metric, limit) { diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8e5cb38fb..97ca5546d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -712,7 +712,6 @@ jobs: _APP_DOMAIN: localhost _APP_CONSOLE_DOMAIN: localhost _APP_DOMAIN_FUNCTIONS: functions.localhost - _APP_DOMAIN_SITES: sites.localhost run: | docker tag ${{ env.IMAGE }}:before ${{ env.IMAGE }} docker compose up -d --wait --no-build @@ -726,7 +725,6 @@ jobs: uses: grafana/run-k6-action@v1 env: APPWRITE_ENDPOINT: 'http://localhost/v1' - APPWRITE_MAILDEV_ENDPOINT: 'http://localhost:9503/email' APPWRITE_BENCHMARK_ITERATIONS: '1' APPWRITE_BENCHMARK_VUS: '1' APPWRITE_WORKER_TIMEOUT_MS: '120000' @@ -768,7 +766,6 @@ jobs: _APP_DOMAIN: localhost _APP_CONSOLE_DOMAIN: localhost _APP_DOMAIN_FUNCTIONS: functions.localhost - _APP_DOMAIN_SITES: sites.localhost run: | docker tag ${{ env.IMAGE }}:after ${{ env.IMAGE }} docker compose up -d --wait --no-build @@ -779,7 +776,6 @@ jobs: uses: grafana/run-k6-action@v1 env: APPWRITE_ENDPOINT: 'http://localhost/v1' - APPWRITE_MAILDEV_ENDPOINT: 'http://localhost:9503/email' APPWRITE_BENCHMARK_ITERATIONS: '1' APPWRITE_BENCHMARK_VUS: '1' APPWRITE_WORKER_TIMEOUT_MS: '120000' diff --git a/tests/benchmarks/http-local.sh b/tests/benchmarks/http-local.sh index acb8a07058..734c825fda 100755 --- a/tests/benchmarks/http-local.sh +++ b/tests/benchmarks/http-local.sh @@ -6,7 +6,6 @@ export K6_WEB_DASHBOARD_HOST="${K6_WEB_DASHBOARD_HOST:-127.0.0.1}" export K6_WEB_DASHBOARD_PORT="${K6_WEB_DASHBOARD_PORT:-5665}" export K6_WEB_DASHBOARD_EXPORT="${K6_WEB_DASHBOARD_EXPORT:-/tmp/appwrite-k6-report.html}" export APPWRITE_ENDPOINT="${APPWRITE_ENDPOINT:-http://localhost/v1}" -export APPWRITE_MAILDEV_ENDPOINT="${APPWRITE_MAILDEV_ENDPOINT:-http://localhost:9503/email}" export APPWRITE_WORKER_TIMEOUT_MS="${APPWRITE_WORKER_TIMEOUT_MS:-120000}" export APPWRITE_BENCHMARK_SUMMARY_PATH="${APPWRITE_BENCHMARK_SUMMARY_PATH:-/tmp/appwrite-k6-summary.json}" diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 3592d0248e..ccab88c011 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -12,12 +12,10 @@ import encoding from 'k6/encoding'; import { Counter, Trend } from 'k6/metrics'; const ENDPOINT = (__ENV.APPWRITE_ENDPOINT || 'http://localhost/v1').replace(/\/+$/, ''); -const MAILDEV_ENDPOINT = __ENV.APPWRITE_MAILDEV_ENDPOINT || 'http://localhost:9503/email'; const CONSOLE_PROJECT = __ENV.APPWRITE_CONSOLE_PROJECT || 'console'; const REGION = __ENV.APPWRITE_REGION || 'default'; const REDIRECT_URL = __ENV.APPWRITE_BENCHMARK_REDIRECT_URL || 'http://localhost'; const PASSWORD = __ENV.APPWRITE_BENCHMARK_PASSWORD || 'Password123!'; -const MAIL_TIMEOUT_MS = Number(__ENV.APPWRITE_MAIL_TIMEOUT_MS || 20000); const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 120000); const ITERATIONS = Number(__ENV.APPWRITE_BENCHMARK_ITERATIONS || 1); const VUS = Number(__ENV.APPWRITE_BENCHMARK_VUS || 1); @@ -25,13 +23,9 @@ const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || '/tmp/appwrite-k6- const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || SUMMARY_PATH; const PREVIOUS_SUMMARY = loadPreviousSummary(); -export const httpDuration = new Trend('appwrite_http_duration', true); export const httpWaiting = new Trend('appwrite_http_waiting', true); export const apiDuration = new Trend('appwrite_api_duration', true); -export const tablesWorkerDuration = new Trend('appwrite_worker_tables_duration', true); -export const mailsWorkerDuration = new Trend('appwrite_worker_mails_duration', true); -export const tablesWorkerSamples = new Counter('appwrite_worker_tables_samples'); -export const mailsWorkerSamples = new Counter('appwrite_worker_mails_samples'); +export const apiWaiting = new Trend('appwrite_api_waiting', true); export const flowFailures = new Counter('appwrite_benchmark_flow_failures'); export const options = { @@ -79,15 +73,12 @@ const API_SCOPES = [ 'buckets.write', 'functions.read', 'functions.write', - 'sites.read', - 'sites.write', 'log.read', 'log.write', 'execution.read', 'execution.write', 'locale.read', 'avatars.read', - 'health.read', 'rules.read', 'rules.write', 'migrations.read', @@ -179,41 +170,60 @@ export function setup() { hostname: hostnameFromUrl(REDIRECT_URL), }, apiHeaders, [201, 409], 'setup.project.platforms.web.create'); - const smtp = rawRequest('PATCH', `/projects/${projectId}/smtp`, { - enabled: true, - senderName: 'Benchmark', - senderEmail: 'benchmark@appwrite.io', - replyTo: 'benchmark@appwrite.io', - host: __ENV.APPWRITE_SMTP_HOST || 'maildev', - port: Number(__ENV.APPWRITE_SMTP_PORT || 1025), - username: __ENV.APPWRITE_SMTP_USERNAME || 'user', - password: __ENV.APPWRITE_SMTP_PASSWORD || 'password', - ...(String(__ENV.APPWRITE_SMTP_SECURE || '') !== '' ? { secure: __ENV.APPWRITE_SMTP_SECURE } : {}), - }, consoleSessionHeaders, 'setup.projects.smtp.update'); - - if (smtp.status !== 200) { - console.warn(`Custom SMTP was not enabled (${smtp.status}). Mail worker timings may be unavailable.`); - } + const tablesDb = setupTablesDb(apiHeaders); return { runId, teamId, projectId, + databaseId: tablesDb.databaseId, + tableId: tablesDb.tableId, consoleSessionHeaders, apiHeaders, platformStatus: platform.status, }; } +function setupTablesDb(apiHeaders) { + const databaseId = unique('tdb'); + const tableId = unique('tbl'); + + setupApi('POST', '/tablesdb', { databaseId, name: 'Benchmark TablesDB' }, apiHeaders, [201], 'setup.tablesdb.create'); + setupApi('POST', `/tablesdb/${databaseId}/tables`, { + tableId, + name: 'Benchmark Table', + permissions: BASE_PERMISSIONS, + rowSecurity: false, + }, apiHeaders, [201], 'setup.tablesdb.tables.create'); + + const columns = [ + ['string', 'title', { size: 128 }], + ['integer', 'quantity', { min: 0, max: 100000 }], + ['email', 'email', {}], + ['boolean', 'active', {}], + ]; + + for (const [type, key, extra] of columns) { + setupApi('POST', `/tablesdb/${databaseId}/tables/${tableId}/columns/${type}`, { + key, + required: false, + array: false, + ...extra, + }, apiHeaders, [202], `setup.tablesdb.columns.${type}.create`); + waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/columns/${key}`, apiHeaders, 'available', WORKER_TIMEOUT_MS, `setup.tablesdb.columns.${type}.wait`); + } + + return { databaseId, tableId }; +} + export function curatedFlows(data) { const ctx = { ...data }; try { - group('account and mail flow', () => accountFlow(ctx)); + group('account flow', () => accountFlow(ctx)); group('tablesdb rows flow', () => tablesDbFlow(ctx)); group('storage files and tokens flow', () => storageFlow(ctx)); - group('functions and sites control-plane flow', () => computeFlow(ctx)); - group('health and queue probes', () => healthFlow(ctx)); + group('functions control-plane flow', () => computeFlow(ctx)); } catch (error) { flowFailures.add(1); throw error; @@ -230,16 +240,6 @@ export function teardown(data) { } } -function recordTablesWorkerDuration(duration, tags) { - tablesWorkerDuration.add(duration, tags); - tablesWorkerSamples.add(1, tags); -} - -function recordMailsWorkerDuration(duration, tags) { - mailsWorkerDuration.add(duration, tags); - mailsWorkerSamples.add(1, tags); -} - function accountFlow(ctx) { const userId = unique('user'); const email = `bench-user-${unique('mail')}@example.com`; @@ -271,109 +271,14 @@ function accountFlow(ctx) { api('PATCH', '/account/prefs', { prefs: { benchmark: true, runId: ctx.runId } }, sessionHeaders, [200], 'account.prefs.update'); api('PATCH', '/account/name', { name: 'Benchmark User Updated' }, sessionHeaders, [200], 'account.name.update'); api('PATCH', '/account/password', { password: `${PASSWORD}2`, oldPassword: PASSWORD }, sessionHeaders, [200], 'account.password.update'); - - const verificationStarted = Date.now(); - api('POST', '/account/verifications/email', { url: REDIRECT_URL }, sessionHeaders, [201], 'account.emailVerification.create'); - const verificationEmail = waitForEmail(email, (message) => { - return includes(message.subject, 'verify') - || includes(message.subject, 'verification') - || includes(message.html, 'verify') - || includes(message.html, 'verification') - || includes(message.text, 'verify') - || includes(message.text, 'verification'); - }, MAIL_TIMEOUT_MS); - recordMailsWorkerDuration(Date.now() - verificationStarted, { job: 'email_verification' }); - - const verification = extractQueryParams(verificationEmail); - if (verification.userId && verification.secret) { - api('PUT', '/account/verifications/email', { - userId: verification.userId, - secret: verification.secret, - }, sessionHeaders, [200], 'account.emailVerification.update'); - } - - const recoveryStarted = Date.now(); - api('POST', '/account/recovery', { email, url: REDIRECT_URL }, headers, [201], 'account.recovery.create'); - const recoveryEmail = waitForEmail(email, (message) => { - return includes(message.subject, 'recovery') - || includes(message.subject, 'recover') - || includes(message.subject, 'reset') - || includes(message.html, 'recovery') - || includes(message.html, 'recover') - || includes(message.html, 'reset') - || includes(message.text, 'recovery') - || includes(message.text, 'recover') - || includes(message.text, 'reset'); - }, MAIL_TIMEOUT_MS); - recordMailsWorkerDuration(Date.now() - recoveryStarted, { job: 'password_recovery' }); - - const recovery = extractQueryParams(recoveryEmail); - if (recovery.userId && recovery.secret) { - api('DELETE', '/account/sessions/current', null, sessionHeaders, [204], 'account.sessions.current.delete'); - - api('PUT', '/account/recovery', { - userId: recovery.userId, - secret: recovery.secret, - password: `${PASSWORD}3`, - }, headers, [200], 'account.recovery.update'); - - const recoveredSession = api('POST', '/account/sessions/email', { - email, - password: `${PASSWORD}3`, - }, headers, [201], 'account.sessions.email.recovered'); - - ctx.sessionHeaders = { - ...headers, - Cookie: cookieHeader(recoveredSession), - }; - - } } function tablesDbFlow(ctx) { requireSession(ctx, 'tablesDbFlow'); - const databaseId = unique('tdb'); - const tableId = unique('tbl'); + const databaseId = ctx.databaseId; + const tableId = ctx.tableId; const rowId = unique('row'); - const indexKey = unique('tidx'); - - api('POST', '/tablesdb', { databaseId, name: 'Benchmark TablesDB' }, ctx.apiHeaders, [201], 'tablesdb.create'); - api('POST', `/tablesdb/${databaseId}/tables`, { - tableId, - name: 'Benchmark Table', - permissions: BASE_PERMISSIONS, - rowSecurity: false, - }, ctx.apiHeaders, [201], 'tablesdb.tables.create'); - - const columns = [ - ['string', 'title', { size: 128 }], - ['integer', 'quantity', { min: 0, max: 100000 }], - ['email', 'email', {}], - ['boolean', 'active', {}], - ]; - - for (const [type, key, extra] of columns) { - const started = Date.now(); - api('POST', `/tablesdb/${databaseId}/tables/${tableId}/columns/${type}`, { - key, - required: false, - array: false, - ...extra, - }, ctx.apiHeaders, [202], `tablesdb.columns.${type}.create`); - waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/columns/${key}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS, `tablesdb.columns.${type}.wait`); - recordTablesWorkerDuration(Date.now() - started, { job: `column_${type}` }); - } - - const indexStarted = Date.now(); - api('POST', `/tablesdb/${databaseId}/tables/${tableId}/indexes`, { - key: indexKey, - type: 'key', - columns: ['title'], - orders: ['asc'], - }, ctx.apiHeaders, [202], 'tablesdb.indexes.create'); - waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/indexes/${indexKey}`, ctx.apiHeaders, 'available', WORKER_TIMEOUT_MS, 'tablesdb.indexes.wait'); - recordTablesWorkerDuration(Date.now() - indexStarted, { job: 'index' }); api('POST', `/tablesdb/${databaseId}/tables/${tableId}/rows`, { rowId, @@ -392,7 +297,6 @@ function tablesDbFlow(ctx) { value: 1, }, ctx.sessionHeaders, [200], 'tablesdb.rows.decrement'); api('DELETE', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, null, ctx.sessionHeaders, [204], 'tablesdb.rows.delete'); - api('DELETE', `/tablesdb/${databaseId}`, null, ctx.apiHeaders, [204], 'tablesdb.delete'); } function storageFlow(ctx) { @@ -426,9 +330,9 @@ function storageFlow(ctx) { tags: { name: 'storage.files.create' }, }); - httpDuration.add(upload.timings.duration, { name: 'storage.files.create' }); httpWaiting.add(upload.timings.waiting, { name: 'storage.files.create' }); apiDuration.add(upload.timings.duration, { name: 'storage.files.create' }); + apiWaiting.add(upload.timings.waiting, { name: 'storage.files.create' }); assertStatus(upload, [201], 'storage file created'); api('GET', `/storage/buckets/${bucketId}/files`, null, ctx.sessionHeaders, [200], 'storage.files.list'); @@ -456,8 +360,6 @@ function computeFlow(ctx) { const functionId = unique('fn'); let functionVariableId; - const siteId = unique('site'); - let siteVariableId; api('POST', '/functions', { functionId, @@ -490,66 +392,12 @@ function computeFlow(ctx) { api('GET', `/functions/${functionId}/variables/${functionVariableId}`, null, ctx.apiHeaders, [200], 'functions.variables.get'); api('DELETE', `/functions/${functionId}/variables/${functionVariableId}`, null, ctx.apiHeaders, [204], 'functions.variables.delete'); api('DELETE', `/functions/${functionId}`, null, ctx.apiHeaders, [204], 'functions.delete'); - - api('POST', '/sites', { - siteId, - name: 'Benchmark Site', - framework: 'other', - adapter: 'static', - buildRuntime: __ENV.APPWRITE_BENCHMARK_RUNTIME || 'node-22', - buildCommand: '', - outputDirectory: '.', - installCommand: '', - fallbackFile: 'index.html', - providerRootDirectory: '.', - specification: '', - }, ctx.apiHeaders, [201], 'sites.create'); - api('GET', '/sites/frameworks', null, ctx.sessionHeaders, [200], 'sites.frameworks.list'); - api('GET', '/sites/specifications', null, ctx.apiHeaders, [200], 'sites.specifications.list'); - const siteVariable = api('POST', `/sites/${siteId}/variables`, { - key: 'BENCHMARK', - value: 'true', - secret: false, - }, ctx.apiHeaders, [201], 'sites.variables.create'); - siteVariableId = siteVariable.json('$id'); - - api('PUT', `/sites/${siteId}/variables/${siteVariableId}`, { - key: 'BENCHMARK', - value: 'updated', - secret: false, - }, ctx.apiHeaders, [200], 'sites.variables.update'); - api('GET', `/sites/${siteId}/variables/${siteVariableId}`, null, ctx.apiHeaders, [200], 'sites.variables.get'); - api('DELETE', `/sites/${siteId}/variables/${siteVariableId}`, null, ctx.apiHeaders, [204], 'sites.variables.delete'); - api('DELETE', `/sites/${siteId}`, null, ctx.apiHeaders, [204], 'sites.delete'); -} - -function healthFlow(ctx) { - const probes = [ - '/health', - '/health/db', - '/health/cache', - '/health/pubsub', - '/health/storage', - '/health/storage/local', - '/health/time', - '/health/queue/mails', - '/health/queue/functions', - '/health/queue/builds', - '/health/queue/deletes', - '/health/queue/webhooks', - '/health/queue/stats-resources', - '/health/queue/stats-usage', - '/health/queue/failed/v1-mails', - ]; - - for (const path of probes) { - api('GET', path, null, ctx.apiHeaders, [200], `health${path.replace(/\//g, '.')}`); - } } function api(method, path, body, headers, expected, name) { const response = rawRequest(method, path, body, headers, name); apiDuration.add(response.timings.duration, { name }); + apiWaiting.add(response.timings.waiting, { name }); assertStatus(response, expected, name); return response; } @@ -567,7 +415,6 @@ function rawRequest(method, path, body, headers, name) { }; const payload = body === null || body === undefined ? null : JSON.stringify(body); const response = http.request(method, `${ENDPOINT}${path}`, payload, params); - httpDuration.add(response.timings.duration, { name }); httpWaiting.add(response.timings.waiting, { name }); return response; @@ -593,73 +440,6 @@ function waitForStatus(path, headers, wantedStatus, timeoutMs, name) { throw new Error(`Timed out waiting for ${path} to become ${wantedStatus}`); } -function waitForEmail(address, predicate, timeoutMs, allowMissingRecipient = false) { - const started = Date.now(); - - while (Date.now() - started < timeoutMs) { - const response = http.get(MAILDEV_ENDPOINT, { tags: { name: 'maildev.email.list' } }); - if (response.status === 200) { - const emails = response.json(); - for (let i = emails.length - 1; i >= 0; i--) { - const message = emails[i]; - if ((emailMatches(message, address) || (allowMissingRecipient && emailRecipientMissing(message))) && predicate(message)) { - return message; - } - } - } - sleep(0.5); - } - - throw new Error(`Timed out waiting for email to ${address}`); -} - -function emailMatches(message, address) { - const recipients = message.to || []; - return recipients.some((recipient) => recipient.address === address); -} - -function emailRecipientMissing(message) { - const recipients = message.to || []; - return recipients.length === 0 || recipients.every((recipient) => !recipient.address); -} - -function extractQueryParams(message) { - const content = `${message.html || ''}\n${message.text || ''}`; - const links = []; - const hrefPattern = /href="([^"]+)"/g; - let hrefMatch = hrefPattern.exec(content); - - while (hrefMatch !== null) { - links.push(hrefMatch[1]); - hrefMatch = hrefPattern.exec(content); - } - - if (links.length === 0) { - links.push(content); - } - - for (const link of links) { - const queryStart = link.indexOf('?'); - if (queryStart === -1) { - continue; - } - - const query = link.slice(queryStart + 1).split('#')[0].replace(/&/g, '&'); - const params = {}; - - for (const pair of query.split('&')) { - const [key, value] = pair.split('='); - params[decodeURIComponent(key)] = decodeURIComponent(value || ''); - } - - if (params.userId && params.secret) { - return params; - } - } - - return {}; -} - function assertStatus(response, expected, name) { const ok = check(response, { [`${name} status ${expected.join('|')}`]: (r) => expected.includes(r.status), @@ -719,10 +499,6 @@ function unique(prefix) { .slice(0, 36); } -function includes(value, needle) { - return String(value || '').toLowerCase().includes(String(needle).toLowerCase()); -} - function hostnameFromUrl(value) { return value.replace(/^https?:\/\//, '').split('/')[0].split(':')[0]; } @@ -731,13 +507,17 @@ export function handleSummary(data) { const lines = [ 'Appwrite curated benchmark review', '', - 'Before/after comparison', + 'Before', '', - comparisonTable(PREVIOUS_SUMMARY, data), + summaryTable(PREVIOUS_SUMMARY), '', - 'Current run details', + 'After', '', - detailsTable(data), + summaryTable(data), + '', + 'Delta', + '', + deltaTable(PREVIOUS_SUMMARY, data), '', ]; @@ -747,19 +527,16 @@ export function handleSummary(data) { }; } -function detailsTable(data) { +function summaryTable(data) { return [ '| Scenario | P50 (ms) | P95 (ms) | Iterations | RPS |', '| --- | ---: | ---: | ---: | ---: |', - detailRow(data, 'Load test', 'appwrite_http_duration', 'iterations', 'http_reqs'), - detailRow(data, 'API total', 'appwrite_api_duration'), - detailRow(data, 'TablesDB schema', 'appwrite_worker_tables_duration', 'appwrite_worker_tables_samples', 'appwrite_worker_tables_samples'), - detailRow(data, 'Mail delivery', 'appwrite_worker_mails_duration', 'appwrite_worker_mails_samples', 'appwrite_worker_mails_samples'), + summaryRow(data, 'API total', 'appwrite_api_duration'), ].join('\n'); } -function detailRow(data, label, metric, iterationsMetric = null, rpsMetric = null) { - const values = data.metrics[metric] && data.metrics[metric].values; +function summaryRow(data, label, metric, iterationsMetric = null, rpsMetric = null) { + const values = data && data.metrics[metric] && data.metrics[metric].values; if (!values || values.count === 0) { return `| ${label} | n/a | n/a | n/a | n/a |`; } @@ -798,23 +575,16 @@ function loadPreviousSummary() { return null; } -function comparisonTable(before, after) { - const rows = [ - ['Load test', 'appwrite_http_duration'], - ['API total', 'appwrite_api_duration'], - ['TablesDB schema', 'appwrite_worker_tables_duration'], - ['Mail delivery', 'appwrite_worker_mails_duration'], - ]; - +function deltaTable(before, after) { return [ - '| Scenario | Before P50 (ms) | Before P95 (ms) | After P50 (ms) | After P95 (ms) | Delta P95 (ms) |', - '| --- | ---: | ---: | ---: | ---: | ---: |', - ...rows.map(([label, metric]) => { - const beforeP50 = trendMetric(before, metric, 'med'); + '| Scenario | P95 delta (ms) |', + '| --- | ---: |', + ...[ + ['API total', 'appwrite_api_duration'], + ].map(([label, metric]) => { const beforeP95 = trendMetric(before, metric, 'p(95)'); - const afterP50 = trendMetric(after, metric, 'med'); const afterP95 = trendMetric(after, metric, 'p(95)'); - return `| ${label} | ${formatValue(beforeP50)} | ${formatValue(beforeP95)} | ${formatValue(afterP50)} | ${formatValue(afterP95)} | ${formatDelta(beforeP95, afterP95)} |`; + return `| ${label} | ${formatDelta(beforeP95, afterP95)} |`; }), ].join('\n'); } @@ -825,14 +595,6 @@ function trendMetric(data, metric, stat) { : null; } -function formatValue(value) { - if (value === null || value === undefined || Number.isNaN(value)) { - return 'n/a'; - } - - return `${round(value)}`; -} - function formatDetailValue(value) { if (value === null || value === undefined || Number.isNaN(value)) { return 'n/a'; From c15e8d0126a236c5197eba27134cf28b6b5f2ea6 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 19:30:01 +0530 Subject: [PATCH 37/39] Harden benchmark failure guard --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97ca5546d4..5aac048161 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -712,6 +712,7 @@ jobs: _APP_DOMAIN: localhost _APP_CONSOLE_DOMAIN: localhost _APP_DOMAIN_FUNCTIONS: functions.localhost + _APP_OPTIONS_ABUSE: disabled run: | docker tag ${{ env.IMAGE }}:before ${{ env.IMAGE }} docker compose up -d --wait --no-build @@ -766,6 +767,7 @@ jobs: _APP_DOMAIN: localhost _APP_CONSOLE_DOMAIN: localhost _APP_DOMAIN_FUNCTIONS: functions.localhost + _APP_OPTIONS_ABUSE: disabled run: | docker tag ${{ env.IMAGE }}:after ${{ env.IMAGE }} docker compose up -d --wait --no-build @@ -816,5 +818,5 @@ jobs: retention-days: 7 - name: Fail benchmark - if: always() && steps.benchmark_after.outcome == 'failure' + if: always() && steps.benchmark_after.outcome != 'success' run: exit 1 From 9a6a5977106c769958dc8039b53aa2ee1de24ed8 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 19:38:48 +0530 Subject: [PATCH 38/39] Address benchmark hardening review --- .github/workflows/ci.yml | 8 ++++---- tests/benchmarks/http.js | 37 ++++++++++++++----------------------- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5aac048161..c5486c739a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -684,7 +684,7 @@ jobs: docker tag ${{ env.IMAGE }} ${{ env.IMAGE }}:after - name: Setup k6 - uses: grafana/setup-k6-action@v1 + uses: grafana/setup-k6-action@ffe7d7290dfa715e48c2ccc924d068444c94bde2 with: k6-version: ${{ env.K6_VERSION }} @@ -723,7 +723,7 @@ jobs: - name: Benchmark before if: steps.benchmark_before_start.outcome == 'success' continue-on-error: true - uses: grafana/run-k6-action@v1 + uses: grafana/run-k6-action@a15e2072ede004e8d46141e33d7f7dad8ad08d9d env: APPWRITE_ENDPOINT: 'http://localhost/v1' APPWRITE_BENCHMARK_ITERATIONS: '1' @@ -775,13 +775,13 @@ jobs: - name: Benchmark after id: benchmark_after continue-on-error: true - uses: grafana/run-k6-action@v1 + uses: grafana/run-k6-action@a15e2072ede004e8d46141e33d7f7dad8ad08d9d env: APPWRITE_ENDPOINT: 'http://localhost/v1' APPWRITE_BENCHMARK_ITERATIONS: '1' APPWRITE_BENCHMARK_VUS: '1' APPWRITE_WORKER_TIMEOUT_MS: '120000' - APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH: 'benchmark-before-summary.json' + APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH: '../../benchmark-before-summary.json' APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-after-summary.json' with: path: tests/benchmarks/http.js diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index ccab88c011..2f3a73c41f 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -20,8 +20,8 @@ const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 120000); const ITERATIONS = Number(__ENV.APPWRITE_BENCHMARK_ITERATIONS || 1); const VUS = Number(__ENV.APPWRITE_BENCHMARK_VUS || 1); const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || '/tmp/appwrite-k6-summary.json'; -const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || SUMMARY_PATH; -const PREVIOUS_SUMMARY = loadPreviousSummary(); +const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || ''; +const PREVIOUS_SUMMARY = PREVIOUS_SUMMARY_PATH ? loadPreviousSummary(PREVIOUS_SUMMARY_PATH) : null; export const httpWaiting = new Trend('appwrite_http_waiting', true); export const apiDuration = new Trend('appwrite_api_duration', true); @@ -549,30 +549,21 @@ function summaryRow(data, label, metric, iterationsMetric = null, rpsMetric = nu return `| ${label} | ${formatDetailValue(values.med)} | ${formatDetailValue(values['p(95)'])} | ${formatCount(iterations)} | ${formatRate(rps)} |`; } -function loadPreviousSummary() { - const paths = [PREVIOUS_SUMMARY_PATH]; - if (!PREVIOUS_SUMMARY_PATH.startsWith('/')) { - paths.push(`../../${PREVIOUS_SUMMARY_PATH}`); +function loadPreviousSummary(path) { + let contents; + try { + contents = open(path); + } catch (error) { + console.warn(`Missing benchmark summary at ${path}: ${error.message}`); + return null; } - for (const path of paths) { - let contents; - try { - contents = open(path); - } catch (error) { - // Try the next path. k6 resolves open() relative to the script file. - continue; - } - - try { - return JSON.parse(contents); - } catch (error) { - console.warn(`Invalid benchmark summary at ${path}: ${error.message}`); - return null; - } + try { + return JSON.parse(contents); + } catch (error) { + console.warn(`Invalid benchmark summary at ${path}: ${error.message}`); + return null; } - - return null; } function deltaTable(before, after) { From 3d66078fe97bb1b545548a81b69631a3203edcc4 Mon Sep 17 00:00:00 2001 From: Chirag Aggarwal Date: Wed, 22 Apr 2026 19:46:27 +0530 Subject: [PATCH 39/39] Increase benchmark iterations --- .github/workflows/benchmark-comment.js | 2 +- .github/workflows/ci.yml | 4 ++-- tests/benchmarks/http.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark-comment.js b/.github/workflows/benchmark-comment.js index fa0fea87f4..f25116c4f2 100644 --- a/.github/workflows/benchmark-comment.js +++ b/.github/workflows/benchmark-comment.js @@ -258,7 +258,7 @@ function metricValue(data, metric, stat) { function metricTable(rows, side) { return [ - '| Scenario | P50 (ms) | P95 (ms) | Iterations | RPS |', + '| Scenario | P50 (ms) | P95 (ms) | Requests | RPS |', '| --- | ---: | ---: | ---: | ---: |', ...rows.map((row) => metricRow(row, side)), ].join('\n'); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5486c739a..8a199f6eab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -726,7 +726,7 @@ jobs: uses: grafana/run-k6-action@a15e2072ede004e8d46141e33d7f7dad8ad08d9d env: APPWRITE_ENDPOINT: 'http://localhost/v1' - APPWRITE_BENCHMARK_ITERATIONS: '1' + APPWRITE_BENCHMARK_ITERATIONS: '5' APPWRITE_BENCHMARK_VUS: '1' APPWRITE_WORKER_TIMEOUT_MS: '120000' APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-before-summary.json' @@ -778,7 +778,7 @@ jobs: uses: grafana/run-k6-action@a15e2072ede004e8d46141e33d7f7dad8ad08d9d env: APPWRITE_ENDPOINT: 'http://localhost/v1' - APPWRITE_BENCHMARK_ITERATIONS: '1' + APPWRITE_BENCHMARK_ITERATIONS: '5' APPWRITE_BENCHMARK_VUS: '1' APPWRITE_WORKER_TIMEOUT_MS: '120000' APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH: '../../benchmark-before-summary.json' diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 2f3a73c41f..4009024069 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -529,7 +529,7 @@ export function handleSummary(data) { function summaryTable(data) { return [ - '| Scenario | P50 (ms) | P95 (ms) | Iterations | RPS |', + '| Scenario | P50 (ms) | P95 (ms) | Requests | RPS |', '| --- | ---: | ---: | ---: | ---: |', summaryRow(data, 'API total', 'appwrite_api_duration'), ].join('\n');