diff --git a/.github/workflows/benchmark-comment.js b/.github/workflows/benchmark-comment.js new file mode 100644 index 0000000000..f25116c4f2 --- /dev/null +++ b/.github/workflows/benchmark-comment.js @@ -0,0 +1,349 @@ +const fs = require('fs'); + +const marker = ''; +const serviceLabels = ['Account', 'TablesDB', 'Storage', 'Functions']; + +module.exports = async ({ github, context, core }) => { + const body = buildComment(core); + fs.writeFileSync('benchmark-comment.txt', body); + + const pullRequest = context.payload.pull_request; + if (!pullRequest || pullRequest.head.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) { + return; + } + + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequest.number, + per_page: 100, + }); + + const existing = comments.find((comment) => { + return comment.user?.type === 'Bot' && comment.body?.includes(marker); + }) || comments.find((comment) => { + return comment.user?.type === 'Bot' && comment.body?.includes('Benchmark results'); + }); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + return; + } + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequest.number, + body, + }); +}; + +function buildComment(core) { + const before = readSummary('benchmark-before-summary.json', core); + const after = readSummary('benchmark-after-summary.json', core); + const beforeSamples = readSamples('benchmark-before-samples.json', core); + const afterSamples = readSamples('benchmark-after-samples.json', core); + const baseRef = markdownText(process.env.BENCHMARK_BASE_REF || 'base'); + const headRef = markdownText(process.env.BENCHMARK_HEAD_REF || 'head'); + const rows = benchmarkRows(before, after, beforeSamples, afterSamples); + const topWaits = topSamples(afterSamples, 'appwrite_api_waiting', 3); + const lines = [ + marker, + '## :sparkles: Benchmark results', + '', + `Comparing ${baseRef} (before) to ${headRef} (after).`, + '', + ]; + + if (before === null) { + lines.push('> Before benchmark did not complete; showing current branch metrics only.', ''); + } + if (after === null) { + lines.push('> Current branch benchmark did not complete; showing available metrics only.', ''); + } + + lines.push( + '**Before**', + '', + metricTable(rows, 'before'), + '', + '**After**', + '', + metricTable(rows, 'after'), + '', + '**Delta**', + '', + '| Scenario | P95 delta (ms) |', + '| --- | ---: |', + ...rows.map(deltaRow), + '', + '
', + 'Top API waits', + '', + '
', + '', + '| API request | Max wait (ms) |', + '| --- | ---: |', + ...topWaitRows(topWaits), + '', + '
', + ); + + return `${lines.join('\n')}\n`; +} + +function readSummary(path, core) { + if (!fs.existsSync(path)) { + return null; + } + + try { + return JSON.parse(fs.readFileSync(path, 'utf8')); + } catch (error) { + core?.warning(`Invalid benchmark summary ${path}: ${error.message}`); + return null; + } +} + +function readSamples(path, core) { + if (!fs.existsSync(path)) { + return []; + } + + const contents = fs.readFileSync(path, 'utf8').trim(); + if (contents === '') { + return []; + } + + return contents + .split('\n') + .filter(Boolean) + .flatMap((line) => { + try { + return [JSON.parse(line)]; + } catch (error) { + core?.warning(`Invalid benchmark sample in ${path}: ${error.message}`); + return []; + } + }); +} + +function benchmarkRows(before, after, beforeSamples, afterSamples) { + const beforeServices = serviceStats(beforeSamples); + const afterServices = serviceStats(afterSamples); + return [ + { + label: 'API total', + before: apiSampleStats(beforeSamples) || summaryStats(before, 'appwrite_api_duration'), + after: apiSampleStats(afterSamples) || summaryStats(after, 'appwrite_api_duration'), + }, + ...serviceLabels.map((label) => ({ + label, + before: beforeServices.get(label) || null, + after: afterServices.get(label) || null, + })), + ]; +} + +function summaryStats(summary, durationMetric, iterationsMetric = null, rpsMetric = null) { + const values = metricValues(summary, durationMetric); + if (!values) { + return null; + } + + return { + p50: values.med ?? null, + p95: values['p(95)'] ?? null, + iterations: iterationsMetric ? metricValue(summary, iterationsMetric, 'count') : values.count ?? null, + rps: rpsMetric ? metricValue(summary, rpsMetric, 'rate') : null, + }; +} + +function serviceStats(samples) { + const apiSamples = samples.filter((sample) => { + return sample.metric === 'appwrite_api_duration' && typeof sample.data?.value === 'number'; + }); + const groups = new Map(); + + for (const sample of apiSamples) { + const service = serviceFromName(sample.data.tags?.name || ''); + if (!service) { + continue; + } + + const serviceSamples = groups.get(service) || []; + serviceSamples.push(sample); + groups.set(service, serviceSamples); + } + + return new Map([...groups.entries()].map(([service, serviceSamples]) => { + const values = serviceSamples.map((sample) => sample.data.value); + const durationSeconds = sampleWindowSeconds(serviceSamples); + return [service, { + p50: percentile(values, 50), + p95: percentile(values, 95), + iterations: values.length, + rps: durationSeconds ? values.length / durationSeconds : null, + }]; + })); +} + +function apiSampleStats(samples) { + const apiSamples = samples.filter((sample) => { + return sample.metric === 'appwrite_api_duration' && typeof sample.data?.value === 'number'; + }); + const values = apiSamples.map((sample) => sample.data.value); + if (values.length === 0) { + return null; + } + + const durationSeconds = sampleWindowSeconds(apiSamples); + return { + p50: percentile(values, 50), + p95: percentile(values, 95), + iterations: values.length, + rps: durationSeconds ? values.length / durationSeconds : null, + }; +} + +function serviceFromName(name) { + if (name.startsWith('account.')) { + return 'Account'; + } + if (name.startsWith('tablesdb.')) { + return 'TablesDB'; + } + if (name.startsWith('storage.') || name.startsWith('tokens.')) { + return 'Storage'; + } + if (name.startsWith('functions.')) { + return 'Functions'; + } + return null; +} + +function sampleWindowSeconds(samples) { + const times = samples + .map((sample) => Date.parse(sample.data?.time)) + .filter((value) => !Number.isNaN(value)); + if (times.length < 2) { + return null; + } + + return Math.max((Math.max(...times) - Math.min(...times)) / 1000, 1); +} + +function percentile(values, percentileValue) { + if (values.length === 0) { + return null; + } + + const sorted = [...values].sort((left, right) => left - right); + const index = Math.ceil((percentileValue / 100) * sorted.length) - 1; + return sorted[Math.max(0, Math.min(index, sorted.length - 1))]; +} + +function metricValues(data, metric) { + return data?.metrics?.[metric]?.values ?? null; +} + +function metricValue(data, metric, stat) { + return metricValues(data, metric)?.[stat] ?? null; +} + +function metricTable(rows, side) { + return [ + '| Scenario | P50 (ms) | P95 (ms) | Requests | RPS |', + '| --- | ---: | ---: | ---: | ---: |', + ...rows.map((row) => metricRow(row, side)), + ].join('\n'); +} + +function metricRow(row, side) { + const values = row[side]; + return `| ${row.label} | ${formatMs(values?.p50)} | ${formatMs(values?.p95)} | ${formatCount(values?.iterations)} | ${formatRate(values?.rps)} |`; +} + +function deltaRow(row) { + return `| ${row.label} | ${formatDelta(row.before?.p95, row.after?.p95)} |`; +} + +function topSamples(samples, metric, limit) { + const byName = samples.reduce((result, sample) => { + if (sample.metric !== metric || typeof sample.data?.value !== 'number') { + return result; + } + + const name = sample.data.tags?.name || 'unknown'; + const current = result.get(name); + if (!current || sample.data.value > current.value) { + result.set(name, { name, value: sample.data.value }); + } + + return result; + }, new Map()); + + return [...byName.values()] + .sort((left, right) => right.value - left.value) + .slice(0, limit); +} + +function topWaitRows(samples) { + if (samples.length === 0) { + return ['| n/a | n/a |']; + } + + return samples.map((sample) => { + return `| ${markdownText(sample.name).replace(/\|/g, '\\|')} | ${formatMs(sample.value)} |`; + }); +} + +function markdownText(value) { + return String(value || '').replace(/[\r\n]/g, ' ').replace(/[&<>"']/g, (char) => { + return ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[char]; + }); +} + +function formatMs(value) { + return formatNumber(value, 2); +} + +function formatRate(value) { + return formatNumber(value, 2); +} + +function formatCount(value) { + if (value === null || value === undefined || Number.isNaN(value)) { + return 'n/a'; + } + + return `${Math.round(value)}`; +} + +function formatDelta(before, after) { + if (before === null || before === undefined || after === null || after === undefined || Number.isNaN(before) || Number.isNaN(after)) { + return 'n/a'; + } + + const difference = Number((after - before).toFixed(2)); + return `${difference > 0 ? '+' : ''}${trimNumber(difference)}`; +} + +function formatNumber(value, decimals) { + if (value === null || value === undefined || Number.isNaN(value)) { + return 'n/a'; + } + + return trimNumber(Number(value).toFixed(decimals)); +} + +function trimNumber(value) { + const text = String(value); + const trimmed = text.includes('.') ? text.replace(/\.?0+$/, '') : text; + return trimmed === '' ? '0' : trimmed; +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7307d3c3f..a056ff8510 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,7 @@ concurrency: env: COMPOSE_FILE: docker-compose.yml IMAGE: appwrite-dev + K6_VERSION: '0.53.0' on: pull_request: @@ -547,6 +548,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v6 + with: + fetch-depth: 1 - name: Download Docker Image uses: actions/download-artifact@v7 @@ -660,13 +663,19 @@ jobs: benchmark: name: Benchmark + if: github.event_name == 'pull_request' runs-on: ubuntu-latest needs: build permissions: + actions: read + contents: read + issues: write pull-requests: write steps: - name: Checkout repository uses: actions/checkout@v6 + with: + fetch-depth: 1 - name: Download Docker Image uses: actions/download-artifact@v7 @@ -680,80 +689,145 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Load and Start Appwrite + - name: Load Appwrite image run: | - sed -i 's/traefik/localhost/g' .env docker load --input /tmp/${{ env.IMAGE }}.tar - docker compose up -d - sleep 10 + docker tag ${{ env.IMAGE }} ${{ env.IMAGE }}:after - - name: Install Oha + - name: Setup k6 + uses: grafana/setup-k6-action@ffe7d7290dfa715e48c2ccc924d068444c94bde2 + with: + k6-version: ${{ env.K6_VERSION }} + + - name: Prepare benchmark before + id: benchmark_before_prepare + continue-on-error: true run: | - echo "deb [signed-by=/usr/share/keyrings/azlux-archive-keyring.gpg] http://packages.azlux.fr/debian/ stable main" | sudo tee /etc/apt/sources.list.d/azlux.list - sudo wget -O /usr/share/keyrings/azlux-archive-keyring.gpg https://azlux.fr/repo.gpg - sudo apt update - sudo apt install oha - oha --version + git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} + git worktree add --detach /tmp/appwrite-benchmark-before ${{ github.event.pull_request.base.sha }} + docker build \ + --cache-from ${{ env.IMAGE }}:after \ + --target development \ + --build-arg DEBUG=false \ + --build-arg TESTING=true \ + --build-arg VERSION=dev \ + --tag ${{ env.IMAGE }}:before \ + /tmp/appwrite-benchmark-before - - name: Benchmark PR - run: 'oha -z 180s http://localhost/v1/health/version --output-format json > benchmark.json' - - - name: Cleaning - run: docker compose down -v - - - name: Installing latest version + - name: Start before Appwrite + id: benchmark_before_start + if: steps.benchmark_before_prepare.outcome == 'success' + continue-on-error: true + working-directory: /tmp/appwrite-benchmark-before + env: + _APP_DOMAIN: localhost + _APP_CONSOLE_DOMAIN: localhost + _APP_DOMAIN_FUNCTIONS: functions.localhost + _APP_OPTIONS_ABUSE: disabled run: | - rm .env - LATEST_TAG=$(curl -fsSL -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/appwrite/appwrite/releases/latest | jq -r .tag_name) - echo "Latest release tag: $LATEST_TAG" - curl -fsSL "https://raw.githubusercontent.com/appwrite/appwrite/${LATEST_TAG}/docker-compose.yml" -o docker-compose.yml - curl -fsSL "https://raw.githubusercontent.com/appwrite/appwrite/${LATEST_TAG}/.env" -o .env - sed -i 's/_APP_OPTIONS_ABUSE=enabled/_APP_OPTIONS_ABUSE=disabled/g' .env - docker compose up -d - sleep 10 + docker tag ${{ env.IMAGE }}:before ${{ env.IMAGE }} + docker compose up -d --wait --no-build - - name: Benchmark Latest - run: oha -z 180s http://localhost/v1/health/version --output-format json > benchmark-latest.json + - name: Prepare benchmark files + run: rm -f benchmark-before-summary.json benchmark-after-summary.json benchmark-before-samples.json benchmark-after-samples.json - - name: Prepare comment + - name: Benchmark before + if: steps.benchmark_before_start.outcome == 'success' + continue-on-error: true + uses: grafana/run-k6-action@a15e2072ede004e8d46141e33d7f7dad8ad08d9d + env: + APPWRITE_ENDPOINT: 'http://localhost/v1' + APPWRITE_BENCHMARK_ITERATIONS: '5' + APPWRITE_BENCHMARK_VUS: '1' + APPWRITE_WORKER_TIMEOUT_MS: '120000' + APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-before-summary.json' + with: + path: tests/benchmarks/http.js + flags: --quiet --out json=benchmark-before-samples.json + cloud-comment-on-pr: false + debug: true + + - name: Stop before Appwrite + if: always() run: | - echo '## :sparkles: Benchmark results' > benchmark.txt - echo ' ' >> benchmark.txt - echo "- Requests per second: $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt - echo "- Requests with 200 status code: $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json)" >> benchmark.txt - echo "- P99 latency: $(jq -r '.latencyPercentiles.p99' benchmark.json )" >> benchmark.txt - echo " " >> benchmark.txt - echo " " >> benchmark.txt - echo "## :zap: Benchmark Comparison" >> benchmark.txt - echo " " >> benchmark.txt - echo "| Metric | This PR | Latest version | " >> benchmark.txt - echo "| --- | --- | --- | " >> benchmark.txt - echo "| RPS | $(jq -r '.summary.requestsPerSec|tonumber?|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.summary.requestsPerSec|tonumber|floor|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt - echo "| 200 | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark.json) | $(jq -r '.statusCodeDistribution."200"|tostring|[while(length>0;.[:-3])|.[-3:]]|reverse|join(",")' benchmark-latest.json) | " >> benchmark.txt - echo "| P99 | $(jq -r '.latencyPercentiles.p99' benchmark.json ) | $(jq -r '.latencyPercentiles.p99' benchmark-latest.json ) | " >> benchmark.txt + if [ -d /tmp/appwrite-benchmark-before ]; then + cd /tmp/appwrite-benchmark-before + docker compose down -v || true + fi + + - name: Wait for benchmark ports + if: always() + run: | + for port in 80 443 8080 9503; do + for attempt in $(seq 1 30); do + if ! ss -ltn | awk '{print $4}' | grep -Eq "[:.]${port}$"; then + break + fi + sleep 1 + done + + if ss -ltn | awk '{print $4}' | grep -Eq "[:.]${port}$"; then + echo "Port ${port} is still in use after stopping the before stack" + ss -ltn + exit 1 + fi + done + + - name: Start after Appwrite + env: + _APP_DOMAIN: localhost + _APP_CONSOLE_DOMAIN: localhost + _APP_DOMAIN_FUNCTIONS: functions.localhost + _APP_OPTIONS_ABUSE: disabled + run: | + docker tag ${{ env.IMAGE }}:after ${{ env.IMAGE }} + docker compose up -d --wait --no-build + + - name: Benchmark after + id: benchmark_after + continue-on-error: true + uses: grafana/run-k6-action@a15e2072ede004e8d46141e33d7f7dad8ad08d9d + env: + APPWRITE_ENDPOINT: 'http://localhost/v1' + APPWRITE_BENCHMARK_ITERATIONS: '5' + APPWRITE_BENCHMARK_VUS: '1' + APPWRITE_WORKER_TIMEOUT_MS: '120000' + APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH: '../../benchmark-before-summary.json' + APPWRITE_BENCHMARK_SUMMARY_PATH: 'benchmark-after-summary.json' + with: + path: tests/benchmarks/http.js + flags: --quiet --out json=benchmark-after-samples.json + cloud-comment-on-pr: false + debug: true + + - name: Stop after Appwrite + if: always() + run: docker compose down -v || true + + - name: Comment on PR + if: always() + uses: actions/github-script@v8 + env: + BENCHMARK_BASE_REF: ${{ github.event.pull_request.base.ref }} + BENCHMARK_HEAD_REF: ${{ github.event.pull_request.head.ref }} + with: + script: | + const comment = require('./.github/workflows/benchmark-comment.js'); + await comment({ github, context, core }); - name: Save results uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: - name: benchmark.json - path: benchmark.json + name: benchmark-results + path: | + benchmark-comment.txt + benchmark-before-summary.json + benchmark-after-summary.json + benchmark-before-samples.json + benchmark-after-samples.json retention-days: 7 - - name: Find Comment - if: github.event.pull_request.head.repo.full_name == github.repository - uses: peter-evans/find-comment@v3 - id: fc - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: Benchmark results - - - name: Comment on PR - if: github.event.pull_request.head.repo.full_name == github.repository - uses: peter-evans/create-or-update-comment@v4 - with: - comment-id: ${{ steps.fc.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - body-path: benchmark.txt - edit-mode: replace + - name: Fail benchmark + if: always() && steps.benchmark_after.outcome != 'success' + run: exit 1 diff --git a/tests/benchmarks/http-local.sh b/tests/benchmarks/http-local.sh new file mode 100755 index 0000000000..734c825fda --- /dev/null +++ b/tests/benchmarks/http-local.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +export K6_WEB_DASHBOARD="${K6_WEB_DASHBOARD:-true}" +export K6_WEB_DASHBOARD_HOST="${K6_WEB_DASHBOARD_HOST:-127.0.0.1}" +export K6_WEB_DASHBOARD_PORT="${K6_WEB_DASHBOARD_PORT:-5665}" +export K6_WEB_DASHBOARD_EXPORT="${K6_WEB_DASHBOARD_EXPORT:-/tmp/appwrite-k6-report.html}" +export APPWRITE_ENDPOINT="${APPWRITE_ENDPOINT:-http://localhost/v1}" +export APPWRITE_WORKER_TIMEOUT_MS="${APPWRITE_WORKER_TIMEOUT_MS:-120000}" +export APPWRITE_BENCHMARK_SUMMARY_PATH="${APPWRITE_BENCHMARK_SUMMARY_PATH:-/tmp/appwrite-k6-summary.json}" + +samples_path="${APPWRITE_BENCHMARK_SAMPLES_PATH:-/tmp/appwrite-k6-samples.json}" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "${script_dir}/../.." && pwd)" + +exec k6 run --out "json=${samples_path}" "$@" "${repo_root}/tests/benchmarks/http.js" diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js index 799c8fb23c..4009024069 100644 --- a/tests/benchmarks/http.js +++ b/tests/benchmarks/http.js @@ -1,34 +1,625 @@ +/* + * Run locally: + * Requires k6 and a running Appwrite instance. + * + * tests/benchmarks/http-local.sh + * + * Open http://127.0.0.1:5665 while the benchmark is running. + */ import http from 'k6/http'; -import { check } from 'k6'; -import { Counter } from 'k6/metrics'; +import { check, group, sleep } from 'k6'; +import encoding from 'k6/encoding'; +import { Counter, Trend } from 'k6/metrics'; -// A simple counter for http requests -export const requests = new Counter('http_reqs'); +const ENDPOINT = (__ENV.APPWRITE_ENDPOINT || 'http://localhost/v1').replace(/\/+$/, ''); +const CONSOLE_PROJECT = __ENV.APPWRITE_CONSOLE_PROJECT || 'console'; +const REGION = __ENV.APPWRITE_REGION || 'default'; +const REDIRECT_URL = __ENV.APPWRITE_BENCHMARK_REDIRECT_URL || 'http://localhost'; +const PASSWORD = __ENV.APPWRITE_BENCHMARK_PASSWORD || 'Password123!'; +const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 120000); +const ITERATIONS = Number(__ENV.APPWRITE_BENCHMARK_ITERATIONS || 1); +const VUS = Number(__ENV.APPWRITE_BENCHMARK_VUS || 1); +const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || '/tmp/appwrite-k6-summary.json'; +const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || ''; +const PREVIOUS_SUMMARY = PREVIOUS_SUMMARY_PATH ? loadPreviousSummary(PREVIOUS_SUMMARY_PATH) : null; -// you can specify stages of your test (ramp up/down patterns) through the options object -// target is the number of VUs you are aiming for +export const httpWaiting = new Trend('appwrite_http_waiting', true); +export const apiDuration = new Trend('appwrite_api_duration', true); +export const apiWaiting = new Trend('appwrite_api_waiting', true); +export const flowFailures = new Counter('appwrite_benchmark_flow_failures'); export const options = { - stages: [ - { target: 50, duration: '1m' }, - // { target: 15, duration: '1m' }, - // { target: 0, duration: '1m' }, - ], + scenarios: { + curated_flows: { + executor: 'shared-iterations', + exec: 'curatedFlows', + vus: VUS, + iterations: ITERATIONS, + maxDuration: __ENV.APPWRITE_BENCHMARK_MAX_DURATION || '30m', + }, + }, thresholds: { - requests: ['count < 100'], + http_req_failed: ['rate<0.05'], + appwrite_api_duration: ['p(95)<2000'], + appwrite_benchmark_flow_failures: ['count<1'], }, }; -export default function () { - const config = { - headers: { - 'X-Appwrite-Key': '24356eb021863f81eb7dd77c7750304d0464e141cad6e9a8befa1f7d2b066fde190df3dab1e8d2639dbb82ee848da30501424923f4cd80d887ee40ad77ded62763ee489448523f6e39667f290f9a54b2ab8fad131a0bc985e6c0f760015f7f3411e40626c75646bb19d2bb2f7bf2f63130918220a206758cbc48845fd725a695', - 'X-Appwrite-Project': '60479fe35d95d' - }} +const API_SCOPES = [ + 'sessions.write', + 'users.read', + 'users.write', + 'teams.read', + 'teams.write', + 'databases.read', + 'databases.write', + 'collections.read', + 'collections.write', + 'tables.read', + 'tables.write', + 'attributes.read', + 'attributes.write', + 'columns.read', + 'columns.write', + 'indexes.read', + 'indexes.write', + 'documents.read', + 'documents.write', + 'rows.read', + 'rows.write', + 'files.read', + 'files.write', + 'buckets.read', + 'buckets.write', + 'functions.read', + 'functions.write', + 'log.read', + 'log.write', + 'execution.read', + 'execution.write', + 'locale.read', + 'avatars.read', + 'rules.read', + 'rules.write', + 'migrations.read', + 'migrations.write', + 'vcs.read', + 'vcs.write', + 'assistant.read', + 'tokens.read', + 'tokens.write', + 'platforms.read', + 'platforms.write', +]; - const resDb = http.get('http://localhost:9501/', config); +const BASE_PERMISSIONS = [ + 'read("any")', + 'create("any")', + 'update("any")', + 'delete("any")', +]; - check(resDb, { - 'status is 200': (r) => r.status === 200, +const ITEM_PERMISSIONS = [ + 'read("any")', + 'update("any")', + 'delete("any")', +]; + +export function setup() { + const runId = unique('run'); + const consoleEmail = __ENV.APPWRITE_ADMIN_EMAIL || `bench-admin-${runId}@example.com`; + const consolePassword = __ENV.APPWRITE_ADMIN_PASSWORD || PASSWORD; + + const consoleHeaders = { + 'Content-Type': 'application/json', + 'X-Appwrite-Project': CONSOLE_PROJECT, + }; + + const account = rawRequest('POST', '/account', { + userId: unique('admin'), + email: consoleEmail, + password: consolePassword, + name: 'Benchmark Admin', + }, consoleHeaders, 'setup.account.create'); + + if (![201, 409].includes(account.status)) { + failResponse(account, 'Unable to create or reuse the benchmark console account'); + } + + const session = rawRequest('POST', '/account/sessions/email', { + email: consoleEmail, + password: consolePassword, + }, consoleHeaders, 'setup.account.session'); + + assertStatus(session, [201], 'console session created'); + + const consoleSessionHeaders = { + ...consoleHeaders, + Cookie: cookieHeader(session), + }; + + const team = setupApi('POST', '/teams', { + teamId: unique('team'), + name: `Benchmark Team ${runId}`, + }, consoleSessionHeaders, [201], 'setup.teams.create'); + + const teamId = team.json('$id'); + const project = setupApi('POST', '/projects', { + projectId: unique('project'), + name: `Benchmark Project ${runId}`, + teamId, + region: REGION, + }, consoleSessionHeaders, [201], 'setup.projects.create'); + + const projectId = project.json('$id'); + const key = setupApi('POST', `/projects/${projectId}/keys`, { + keyId: unique('key'), + name: 'Benchmark API key', + scopes: API_SCOPES, + }, consoleSessionHeaders, [201], 'setup.projects.keys.create'); + + const apiHeaders = { + 'Content-Type': 'application/json', + 'X-Appwrite-Project': projectId, + 'X-Appwrite-Key': key.json('secret'), + }; + + const platform = setupApi('POST', '/project/platforms/web', { + platformId: unique('web'), + name: 'Benchmark web', + hostname: hostnameFromUrl(REDIRECT_URL), + }, apiHeaders, [201, 409], 'setup.project.platforms.web.create'); + + const tablesDb = setupTablesDb(apiHeaders); + + return { + runId, + teamId, + projectId, + databaseId: tablesDb.databaseId, + tableId: tablesDb.tableId, + consoleSessionHeaders, + apiHeaders, + platformStatus: platform.status, + }; +} + +function setupTablesDb(apiHeaders) { + const databaseId = unique('tdb'); + const tableId = unique('tbl'); + + setupApi('POST', '/tablesdb', { databaseId, name: 'Benchmark TablesDB' }, apiHeaders, [201], 'setup.tablesdb.create'); + setupApi('POST', `/tablesdb/${databaseId}/tables`, { + tableId, + name: 'Benchmark Table', + permissions: BASE_PERMISSIONS, + rowSecurity: false, + }, apiHeaders, [201], 'setup.tablesdb.tables.create'); + + const columns = [ + ['string', 'title', { size: 128 }], + ['integer', 'quantity', { min: 0, max: 100000 }], + ['email', 'email', {}], + ['boolean', 'active', {}], + ]; + + for (const [type, key, extra] of columns) { + setupApi('POST', `/tablesdb/${databaseId}/tables/${tableId}/columns/${type}`, { + key, + required: false, + array: false, + ...extra, + }, apiHeaders, [202], `setup.tablesdb.columns.${type}.create`); + waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/columns/${key}`, apiHeaders, 'available', WORKER_TIMEOUT_MS, `setup.tablesdb.columns.${type}.wait`); + } + + return { databaseId, tableId }; +} + +export function curatedFlows(data) { + const ctx = { ...data }; + + try { + group('account flow', () => accountFlow(ctx)); + group('tablesdb rows flow', () => tablesDbFlow(ctx)); + group('storage files and tokens flow', () => storageFlow(ctx)); + group('functions control-plane flow', () => computeFlow(ctx)); + } catch (error) { + flowFailures.add(1); + throw error; + } +} + +export function teardown(data) { + if (data && data.projectId && data.consoleSessionHeaders) { + rawRequest('DELETE', `/projects/${data.projectId}`, null, data.consoleSessionHeaders, 'teardown.projects.delete'); + } + + if (data && data.teamId && data.consoleSessionHeaders) { + rawRequest('DELETE', `/teams/${data.teamId}`, null, data.consoleSessionHeaders, 'teardown.teams.delete'); + } +} + +function accountFlow(ctx) { + const userId = unique('user'); + const email = `bench-user-${unique('mail')}@example.com`; + const headers = projectHeaders(ctx.projectId); + + api('POST', '/account', { + userId, + email, + password: PASSWORD, + name: 'Benchmark User', + }, headers, [201], 'account.create'); + + const session = api('POST', '/account/sessions/email', { + email, + password: PASSWORD, + }, headers, [201], 'account.sessions.email.create'); + + const sessionHeaders = { + ...headers, + Cookie: cookieHeader(session), + }; + + ctx.userId = userId; + ctx.userEmail = email; + ctx.sessionHeaders = sessionHeaders; + + api('GET', '/account', null, sessionHeaders, [200], 'account.get'); + api('GET', '/account/logs', null, sessionHeaders, [200], 'account.logs.list'); + api('PATCH', '/account/prefs', { prefs: { benchmark: true, runId: ctx.runId } }, sessionHeaders, [200], 'account.prefs.update'); + api('PATCH', '/account/name', { name: 'Benchmark User Updated' }, sessionHeaders, [200], 'account.name.update'); + api('PATCH', '/account/password', { password: `${PASSWORD}2`, oldPassword: PASSWORD }, sessionHeaders, [200], 'account.password.update'); +} + +function tablesDbFlow(ctx) { + requireSession(ctx, 'tablesDbFlow'); + + const databaseId = ctx.databaseId; + const tableId = ctx.tableId; + const rowId = unique('row'); + + api('POST', `/tablesdb/${databaseId}/tables/${tableId}/rows`, { + rowId, + data: tablePayload(), + permissions: ITEM_PERMISSIONS, + }, ctx.sessionHeaders, [201], 'tablesdb.rows.create'); + api('GET', `/tablesdb/${databaseId}/tables/${tableId}/rows`, null, ctx.sessionHeaders, [200], 'tablesdb.rows.list'); + api('GET', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, null, ctx.sessionHeaders, [200], 'tablesdb.rows.get'); + api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, { + data: { title: 'Benchmark Row Updated' }, + }, ctx.sessionHeaders, [200], 'tablesdb.rows.update'); + api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}/quantity/increment`, { + value: 1, + }, ctx.sessionHeaders, [200], 'tablesdb.rows.increment'); + api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}/quantity/decrement`, { + value: 1, + }, ctx.sessionHeaders, [200], 'tablesdb.rows.decrement'); + api('DELETE', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, null, ctx.sessionHeaders, [204], 'tablesdb.rows.delete'); +} + +function storageFlow(ctx) { + requireSession(ctx, 'storageFlow'); + + const bucketId = unique('bucket'); + const fileId = unique('file'); + + api('POST', '/storage/buckets', { + bucketId, + name: 'Benchmark Bucket', + permissions: BASE_PERMISSIONS, + fileSecurity: false, + enabled: true, + maximumFileSize: 30000000, + allowedFileExtensions: [], + compression: 'none', + encryption: false, + antivirus: false, + }, ctx.apiHeaders, [201], 'storage.buckets.create'); + + const multipartHeaders = { ...ctx.sessionHeaders }; + delete multipartHeaders['Content-Type']; + + const upload = http.post(`${ENDPOINT}/storage/buckets/${bucketId}/files`, { + fileId, + file: http.file(onePixelPng(), 'benchmark.png', 'image/png'), + ...flattenMultipartArray('permissions', ITEM_PERMISSIONS), + }, { + headers: multipartHeaders, + tags: { name: 'storage.files.create' }, }); -} \ No newline at end of file + + httpWaiting.add(upload.timings.waiting, { name: 'storage.files.create' }); + apiDuration.add(upload.timings.duration, { name: 'storage.files.create' }); + apiWaiting.add(upload.timings.waiting, { name: 'storage.files.create' }); + assertStatus(upload, [201], 'storage file created'); + + api('GET', `/storage/buckets/${bucketId}/files`, null, ctx.sessionHeaders, [200], 'storage.files.list'); + api('GET', `/storage/buckets/${bucketId}/files/${fileId}`, null, ctx.sessionHeaders, [200], 'storage.files.get'); + api('GET', `/storage/buckets/${bucketId}/files/${fileId}/view`, null, ctx.sessionHeaders, [200], 'storage.files.view'); + api('GET', `/storage/buckets/${bucketId}/files/${fileId}/download`, null, ctx.sessionHeaders, [200], 'storage.files.download'); + api('GET', `/storage/buckets/${bucketId}/files/${fileId}/preview`, null, ctx.sessionHeaders, [200], 'storage.files.preview'); + api('PUT', `/storage/buckets/${bucketId}/files/${fileId}`, { + name: 'benchmark-renamed.png', + permissions: ITEM_PERMISSIONS, + }, ctx.sessionHeaders, [200], 'storage.files.update'); + + const token = api('POST', `/tokens/buckets/${bucketId}/files/${fileId}`, {}, ctx.apiHeaders, [201], 'tokens.files.create'); + api('GET', `/tokens/buckets/${bucketId}/files/${fileId}`, null, ctx.apiHeaders, [200], 'tokens.files.list'); + api('GET', `/tokens/${token.json('$id')}`, null, ctx.apiHeaders, [200], 'tokens.get'); + api('PATCH', `/tokens/${token.json('$id')}`, { expire: null }, ctx.apiHeaders, [200], 'tokens.update'); + api('DELETE', `/tokens/${token.json('$id')}`, null, ctx.apiHeaders, [204], 'tokens.delete'); + + api('DELETE', `/storage/buckets/${bucketId}/files/${fileId}`, null, ctx.sessionHeaders, [204], 'storage.files.delete'); + api('DELETE', `/storage/buckets/${bucketId}`, null, ctx.apiHeaders, [204], 'storage.buckets.delete'); +} + +function computeFlow(ctx) { + requireSession(ctx, 'computeFlow'); + + const functionId = unique('fn'); + let functionVariableId; + + api('POST', '/functions', { + functionId, + name: 'Benchmark Function', + runtime: __ENV.APPWRITE_BENCHMARK_RUNTIME || 'node-22', + execute: ['any'], + events: [], + schedule: '', + timeout: 15, + enabled: true, + logging: true, + entrypoint: 'index.js', + commands: 'npm install', + scopes: ['users.read'], + }, ctx.apiHeaders, [201], 'functions.create'); + api('GET', '/functions/runtimes', null, ctx.sessionHeaders, [200], 'functions.runtimes.list'); + api('GET', '/functions/specifications', null, ctx.apiHeaders, [200], 'functions.specifications.list'); + const functionVariable = api('POST', `/functions/${functionId}/variables`, { + key: 'BENCHMARK', + value: 'true', + secret: false, + }, ctx.apiHeaders, [201], 'functions.variables.create'); + functionVariableId = functionVariable.json('$id'); + + api('PUT', `/functions/${functionId}/variables/${functionVariableId}`, { + key: 'BENCHMARK', + value: 'updated', + secret: false, + }, ctx.apiHeaders, [200], 'functions.variables.update'); + api('GET', `/functions/${functionId}/variables/${functionVariableId}`, null, ctx.apiHeaders, [200], 'functions.variables.get'); + api('DELETE', `/functions/${functionId}/variables/${functionVariableId}`, null, ctx.apiHeaders, [204], 'functions.variables.delete'); + api('DELETE', `/functions/${functionId}`, null, ctx.apiHeaders, [204], 'functions.delete'); +} + +function api(method, path, body, headers, expected, name) { + const response = rawRequest(method, path, body, headers, name); + apiDuration.add(response.timings.duration, { name }); + apiWaiting.add(response.timings.waiting, { name }); + assertStatus(response, expected, name); + return response; +} + +function setupApi(method, path, body, headers, expected, name) { + const response = rawRequest(method, path, body, headers, name); + assertStatus(response, expected, name); + return response; +} + +function rawRequest(method, path, body, headers, name) { + const params = { + headers, + tags: { name }, + }; + const payload = body === null || body === undefined ? null : JSON.stringify(body); + const response = http.request(method, `${ENDPOINT}${path}`, payload, params); + httpWaiting.add(response.timings.waiting, { name }); + + return response; +} + +function waitForStatus(path, headers, wantedStatus, timeoutMs, name) { + const started = Date.now(); + + while (Date.now() - started < timeoutMs) { + const response = rawRequest('GET', path, null, headers, name); + if (response.status === 200) { + const status = response.json('status'); + if (status === wantedStatus) { + return response; + } + if (status === 'failed') { + throw new Error(`${path} failed while waiting for ${wantedStatus}`); + } + } + sleep(0.5); + } + + throw new Error(`Timed out waiting for ${path} to become ${wantedStatus}`); +} + +function assertStatus(response, expected, name) { + const ok = check(response, { + [`${name} status ${expected.join('|')}`]: (r) => expected.includes(r.status), + }); + + if (!ok) { + failResponse(response, `${name} returned an unexpected status`); + } +} + +function failResponse(response, message) { + throw new Error(`${message}. Status: ${response.status}. Body: ${response.body}`); +} + +function cookieHeader(response) { + return response.headers['Set-Cookie'] || response.headers['set-cookie'] || ''; +} + +function projectHeaders(projectId) { + return { + 'Content-Type': 'application/json', + 'X-Appwrite-Project': projectId, + }; +} + +function requireSession(ctx, flow) { + if (!ctx.sessionHeaders || typeof ctx.sessionHeaders !== 'object') { + throw new Error(`accountFlow must run before ${flow}`); + } +} + +function tablePayload() { + return { + title: 'Benchmark Row', + quantity: 1, + email: 'row@example.com', + active: true, + }; +} + +function onePixelPng() { + return encoding.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII=', 'std', 'b'); +} + +function flattenMultipartArray(key, values) { + const output = {}; + values.forEach((value, index) => { + output[`${key}[${index}]`] = value; + }); + return output; +} + +function unique(prefix) { + return `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}` + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .slice(0, 36); +} + +function hostnameFromUrl(value) { + return value.replace(/^https?:\/\//, '').split('/')[0].split(':')[0]; +} + +export function handleSummary(data) { + const lines = [ + 'Appwrite curated benchmark review', + '', + 'Before', + '', + summaryTable(PREVIOUS_SUMMARY), + '', + 'After', + '', + summaryTable(data), + '', + 'Delta', + '', + deltaTable(PREVIOUS_SUMMARY, data), + '', + ]; + + return { + stdout: `${lines.join('\n')}\n`, + [SUMMARY_PATH]: JSON.stringify(data, null, 2), + }; +} + +function summaryTable(data) { + return [ + '| Scenario | P50 (ms) | P95 (ms) | Requests | RPS |', + '| --- | ---: | ---: | ---: | ---: |', + summaryRow(data, 'API total', 'appwrite_api_duration'), + ].join('\n'); +} + +function summaryRow(data, label, metric, iterationsMetric = null, rpsMetric = null) { + const values = data && data.metrics[metric] && data.metrics[metric].values; + if (!values || values.count === 0) { + return `| ${label} | n/a | n/a | n/a | n/a |`; + } + + const iterations = iterationsMetric + ? trendMetric(data, iterationsMetric, 'count') + : values.count; + const rps = rpsMetric ? trendMetric(data, rpsMetric, 'rate') : null; + + return `| ${label} | ${formatDetailValue(values.med)} | ${formatDetailValue(values['p(95)'])} | ${formatCount(iterations)} | ${formatRate(rps)} |`; +} + +function loadPreviousSummary(path) { + let contents; + try { + contents = open(path); + } catch (error) { + console.warn(`Missing benchmark summary at ${path}: ${error.message}`); + return null; + } + + try { + return JSON.parse(contents); + } catch (error) { + console.warn(`Invalid benchmark summary at ${path}: ${error.message}`); + return null; + } +} + +function deltaTable(before, after) { + return [ + '| Scenario | P95 delta (ms) |', + '| --- | ---: |', + ...[ + ['API total', 'appwrite_api_duration'], + ].map(([label, metric]) => { + const beforeP95 = trendMetric(before, metric, 'p(95)'); + const afterP95 = trendMetric(after, metric, 'p(95)'); + return `| ${label} | ${formatDelta(beforeP95, afterP95)} |`; + }), + ].join('\n'); +} + +function trendMetric(data, metric, stat) { + return data && data.metrics[metric] && data.metrics[metric].values + ? data.metrics[metric].values[stat] + : null; +} + +function formatDetailValue(value) { + if (value === null || value === undefined || Number.isNaN(value)) { + return 'n/a'; + } + + return `${Number(value).toFixed(2)}`; +} + +function formatDelta(before, after) { + if (before === null || before === undefined || after === null || after === undefined || Number.isNaN(before) || Number.isNaN(after)) { + return 'n/a'; + } + + const delta = round(after - before); + const sign = delta > 0 ? '+' : ''; + return `${sign}${delta}`; +} + +function formatCount(value) { + if (value === null || value === undefined || Number.isNaN(value)) { + return 'n/a'; + } + + return `${Math.round(value)}`; +} + +function formatRate(value) { + if (value === null || value === undefined || Number.isNaN(value)) { + return 'n/a'; + } + + return `${Number(value).toFixed(2)}`; +} + +function round(value) { + return Math.round((value || 0) * 100) / 100; +}