appwrite/tests/benchmarks/http.js

/*
 * Run locally:
 * Requires k6 and a running Appwrite instance.
 *
 * tests/benchmarks/http-local.sh
 *
 * Open http://127.0.0.1:5665 while the benchmark is running.
 */
import http from 'k6/http';
import { check, group, sleep } from 'k6';
import encoding from 'k6/encoding';
import { Counter, Trend } from 'k6/metrics';

const ENDPOINT = (__ENV.APPWRITE_ENDPOINT || 'http://localhost/v1').replace(/\/+$/, '');
const CONSOLE_PROJECT = __ENV.APPWRITE_CONSOLE_PROJECT || 'console';
const REGION = __ENV.APPWRITE_REGION || 'default';
const REDIRECT_URL = __ENV.APPWRITE_BENCHMARK_REDIRECT_URL || 'http://localhost';
const PASSWORD = __ENV.APPWRITE_BENCHMARK_PASSWORD || 'Password123!';
const WORKER_TIMEOUT_MS = Number(__ENV.APPWRITE_WORKER_TIMEOUT_MS || 120000);
const ITERATIONS = Number(__ENV.APPWRITE_BENCHMARK_ITERATIONS || 1);
const VUS = Number(__ENV.APPWRITE_BENCHMARK_VUS || 1);
const SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_SUMMARY_PATH || '/tmp/appwrite-k6-summary.json';
const PREVIOUS_SUMMARY_PATH = __ENV.APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH || '';
const PREVIOUS_SUMMARY = PREVIOUS_SUMMARY_PATH ? loadPreviousSummary(PREVIOUS_SUMMARY_PATH) : null;

export const httpWaiting = new Trend('appwrite_http_waiting', true);
export const apiDuration = new Trend('appwrite_api_duration', true);
export const apiWaiting = new Trend('appwrite_api_waiting', true);
export const flowFailures = new Counter('appwrite_benchmark_flow_failures');

export const options = {
    scenarios: {
        curated_flows: {
            executor: 'shared-iterations',
            exec: 'curatedFlows',
            vus: VUS,
            iterations: ITERATIONS,
            maxDuration: __ENV.APPWRITE_BENCHMARK_MAX_DURATION || '30m',
        },
    },
    thresholds: {
        http_req_failed: ['rate<0.05'],
        appwrite_api_duration: ['p(95)<2000'],
        appwrite_benchmark_flow_failures: ['count<1'],
    },
};

const API_SCOPES = [
    'sessions.write',
    'users.read',
    'users.write',
    'teams.read',
    'teams.write',
    'databases.read',
    'databases.write',
    'collections.read',
    'collections.write',
    'tables.read',
    'tables.write',
    'attributes.read',
    'attributes.write',
    'columns.read',
    'columns.write',
    'indexes.read',
    'indexes.write',
    'documents.read',
    'documents.write',
    'rows.read',
    'rows.write',
    'files.read',
    'files.write',
    'buckets.read',
    'buckets.write',
    'functions.read',
    'functions.write',
    'log.read',
    'log.write',
    'executions.read',
    'executions.write',
    'locale.read',
    'avatars.read',
    'rules.read',
    'rules.write',
    'migrations.read',
    'migrations.write',
    'vcs.read',
    'vcs.write',
    'assistant.read',
    'tokens.read',
    'tokens.write',
    'platforms.read',
    'platforms.write',
    'oauth2.read',
    'oauth2.write',
];

const BASE_PERMISSIONS = [
    'read("any")',
    'create("any")',
    'update("any")',
    'delete("any")',
];

const ITEM_PERMISSIONS = [
    'read("any")',
    'update("any")',
    'delete("any")',
];

export function setup() {
    const runId = unique('run');
    const consoleEmail = __ENV.APPWRITE_ADMIN_EMAIL || `bench-admin-${runId}@example.com`;
    const consolePassword = __ENV.APPWRITE_ADMIN_PASSWORD || PASSWORD;

    const consoleHeaders = {
        'Content-Type': 'application/json',
        'X-Appwrite-Project': CONSOLE_PROJECT,
    };

    const account = rawRequest('POST', '/account', {
        userId: unique('admin'),
        email: consoleEmail,
        password: consolePassword,
        name: 'Benchmark Admin',
    }, consoleHeaders, 'setup.account.create');

    if (![201, 409].includes(account.status)) {
        failResponse(account, 'Unable to create or reuse the benchmark console account');
    }

    const session = rawRequest('POST', '/account/sessions/email', {
        email: consoleEmail,
        password: consolePassword,
    }, consoleHeaders, 'setup.account.session');

    assertStatus(session, [201], 'console session created');

    const consoleSessionHeaders = {
        ...consoleHeaders,
        Cookie: cookieHeader(session),
    };

    const team = setupApi('POST', '/teams', {
        teamId: unique('team'),
        name: `Benchmark Team ${runId}`,
    }, consoleSessionHeaders, [201], 'setup.teams.create');

    const teamId = team.json('$id');
    const project = setupApi('POST', '/projects', {
        projectId: unique('project'),
        name: `Benchmark Project ${runId}`,
        teamId,
        region: REGION,
    }, consoleSessionHeaders, [201], 'setup.projects.create');

    const projectId = project.json('$id');
    const key = setupApi('POST', `/projects/${projectId}/keys`, {
        keyId: unique('key'),
        name: 'Benchmark API key',
        scopes: API_SCOPES,
    }, consoleSessionHeaders, [201], 'setup.projects.keys.create');

    const apiHeaders = {
        'Content-Type': 'application/json',
        'X-Appwrite-Project': projectId,
        'X-Appwrite-Key': key.json('secret'),
    };

    const platform = setupApi('POST', '/project/platforms/web', {
        platformId: unique('web'),
        name: 'Benchmark web',
        hostname: hostnameFromUrl(REDIRECT_URL),
    }, apiHeaders, [201, 409], 'setup.project.platforms.web.create');

    const tablesDb = setupTablesDb(apiHeaders);

    return {
        runId,
        teamId,
        projectId,
        databaseId: tablesDb.databaseId,
        tableId: tablesDb.tableId,
        consoleSessionHeaders,
        apiHeaders,
        platformStatus: platform.status,
    };
}

function setupTablesDb(apiHeaders) {
    const databaseId = unique('tdb');
    const tableId = unique('tbl');

    setupApi('POST', '/tablesdb', { databaseId, name: 'Benchmark TablesDB' }, apiHeaders, [201], 'setup.tablesdb.create');
    setupApi('POST', `/tablesdb/${databaseId}/tables`, {
        tableId,
        name: 'Benchmark Table',
        permissions: BASE_PERMISSIONS,
        rowSecurity: false,
    }, apiHeaders, [201], 'setup.tablesdb.tables.create');

    const columns = [
        ['string', 'title', { size: 128 }],
        ['integer', 'quantity', { min: 0, max: 100000 }],
        ['email', 'email', {}],
        ['boolean', 'active', {}],
    ];

    for (const [type, key, extra] of columns) {
        setupApi('POST', `/tablesdb/${databaseId}/tables/${tableId}/columns/${type}`, {
            key,
            required: false,
            array: false,
            ...extra,
        }, apiHeaders, [202], `setup.tablesdb.columns.${type}.create`);
        waitForStatus(`/tablesdb/${databaseId}/tables/${tableId}/columns/${key}`, apiHeaders, 'available', WORKER_TIMEOUT_MS, `setup.tablesdb.columns.${type}.wait`);
    }

    return { databaseId, tableId };
}

export function curatedFlows(data) {
    const ctx = { ...data };

    try {
        group('account flow', () => accountFlow(ctx));
        group('tablesdb rows flow', () => tablesDbFlow(ctx));
        group('storage files and tokens flow', () => storageFlow(ctx));
        group('functions control-plane flow', () => computeFlow(ctx));
    } catch (error) {
        flowFailures.add(1);
        throw error;
    }
}

export function teardown(data) {
    if (data && data.projectId && data.consoleSessionHeaders) {
        rawRequest('DELETE', `/projects/${data.projectId}`, null, data.consoleSessionHeaders, 'teardown.projects.delete');
    }

    if (data && data.teamId && data.consoleSessionHeaders) {
        rawRequest('DELETE', `/teams/${data.teamId}`, null, data.consoleSessionHeaders, 'teardown.teams.delete');
    }
}

function accountFlow(ctx) {
    const userId = unique('user');
    const email = `bench-user-${unique('mail')}@example.com`;
    const headers = projectHeaders(ctx.projectId);

    api('POST', '/account', {
        userId,
        email,
        password: PASSWORD,
        name: 'Benchmark User',
    }, headers, [201], 'account.create');

    const session = api('POST', '/account/sessions/email', {
        email,
        password: PASSWORD,
    }, headers, [201], 'account.sessions.email.create');

    const sessionHeaders = {
        ...headers,
        Cookie: cookieHeader(session),
    };

    ctx.userId = userId;
    ctx.userEmail = email;
    ctx.sessionHeaders = sessionHeaders;

    api('GET', '/account', null, sessionHeaders, [200], 'account.get');
    api('GET', '/account/logs', null, sessionHeaders, [200], 'account.logs.list');
    api('PATCH', '/account/prefs', { prefs: { benchmark: true, runId: ctx.runId } }, sessionHeaders, [200], 'account.prefs.update');
    api('PATCH', '/account/name', { name: 'Benchmark User Updated' }, sessionHeaders, [200], 'account.name.update');
    api('PATCH', '/account/password', { password: `${PASSWORD}2`, oldPassword: PASSWORD }, sessionHeaders, [200], 'account.password.update');
}

function tablesDbFlow(ctx) {
    requireSession(ctx, 'tablesDbFlow');

    const databaseId = ctx.databaseId;
    const tableId = ctx.tableId;
    const rowId = unique('row');

    api('POST', `/tablesdb/${databaseId}/tables/${tableId}/rows`, {
        rowId,
        data: tablePayload(),
        permissions: ITEM_PERMISSIONS,
    }, ctx.sessionHeaders, [201], 'tablesdb.rows.create');
    api('GET', `/tablesdb/${databaseId}/tables/${tableId}/rows`, null, ctx.sessionHeaders, [200], 'tablesdb.rows.list');
    api('GET', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, null, ctx.sessionHeaders, [200], 'tablesdb.rows.get');
    api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, {
        data: { title: 'Benchmark Row Updated' },
    }, ctx.sessionHeaders, [200], 'tablesdb.rows.update');
    api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}/quantity/increment`, {
        value: 1,
    }, ctx.sessionHeaders, [200], 'tablesdb.rows.increment');
    api('PATCH', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}/quantity/decrement`, {
        value: 1,
    }, ctx.sessionHeaders, [200], 'tablesdb.rows.decrement');
    api('DELETE', `/tablesdb/${databaseId}/tables/${tableId}/rows/${rowId}`, null, ctx.sessionHeaders, [204], 'tablesdb.rows.delete');
}

function storageFlow(ctx) {
    requireSession(ctx, 'storageFlow');

    const bucketId = unique('bucket');
    const fileId = unique('file');

    api('POST', '/storage/buckets', {
        bucketId,
        name: 'Benchmark Bucket',
        permissions: BASE_PERMISSIONS,
        fileSecurity: false,
        enabled: true,
        maximumFileSize: 30000000,
        allowedFileExtensions: [],
        compression: 'none',
        encryption: false,
        antivirus: false,
    }, ctx.apiHeaders, [201], 'storage.buckets.create');

    const multipartHeaders = { ...ctx.sessionHeaders };
    delete multipartHeaders['Content-Type'];

    const upload = http.post(`${ENDPOINT}/storage/buckets/${bucketId}/files`, {
        fileId,
        file: http.file(onePixelPng(), 'benchmark.png', 'image/png'),
        ...flattenMultipartArray('permissions', ITEM_PERMISSIONS),
    }, {
        headers: multipartHeaders,
        tags: { name: 'storage.files.create' },
    });

    httpWaiting.add(upload.timings.waiting, { name: 'storage.files.create' });
    apiDuration.add(upload.timings.duration, { name: 'storage.files.create' });
    apiWaiting.add(upload.timings.waiting, { name: 'storage.files.create' });
    assertStatus(upload, [201], 'storage file created');

    api('GET', `/storage/buckets/${bucketId}/files`, null, ctx.sessionHeaders, [200], 'storage.files.list');
    api('GET', `/storage/buckets/${bucketId}/files/${fileId}`, null, ctx.sessionHeaders, [200], 'storage.files.get');
    api('GET', `/storage/buckets/${bucketId}/files/${fileId}/view`, null, ctx.sessionHeaders, [200], 'storage.files.view');
    api('GET', `/storage/buckets/${bucketId}/files/${fileId}/download`, null, ctx.sessionHeaders, [200], 'storage.files.download');
    api('GET', `/storage/buckets/${bucketId}/files/${fileId}/preview`, null, ctx.sessionHeaders, [200], 'storage.files.preview');
    api('PUT', `/storage/buckets/${bucketId}/files/${fileId}`, {
        name: 'benchmark-renamed.png',
        permissions: ITEM_PERMISSIONS,
    }, ctx.sessionHeaders, [200], 'storage.files.update');

    const token = api('POST', `/tokens/buckets/${bucketId}/files/${fileId}`, {}, ctx.apiHeaders, [201], 'tokens.files.create');
    api('GET', `/tokens/buckets/${bucketId}/files/${fileId}`, null, ctx.apiHeaders, [200], 'tokens.files.list');
    api('GET', `/tokens/${token.json('$id')}`, null, ctx.apiHeaders, [200], 'tokens.get');
    api('PATCH', `/tokens/${token.json('$id')}`, { expire: null }, ctx.apiHeaders, [200], 'tokens.update');
    api('DELETE', `/tokens/${token.json('$id')}`, null, ctx.apiHeaders, [204], 'tokens.delete');

    api('DELETE', `/storage/buckets/${bucketId}/files/${fileId}`, null, ctx.sessionHeaders, [204], 'storage.files.delete');
    api('DELETE', `/storage/buckets/${bucketId}`, null, ctx.apiHeaders, [204], 'storage.buckets.delete');
}

function computeFlow(ctx) {
    requireSession(ctx, 'computeFlow');

    const functionId = unique('fn');
    let functionVariableId;

    api('POST', '/functions', {
        functionId,
        name: 'Benchmark Function',
        runtime: __ENV.APPWRITE_BENCHMARK_RUNTIME || 'node-22',
        execute: ['any'],
        events: [],
        schedule: '',
        timeout: 15,
        enabled: true,
        logging: true,
        entrypoint: 'index.js',
        commands: 'npm install',
        scopes: ['users.read'],
    }, ctx.apiHeaders, [201], 'functions.create');
    api('GET', '/functions/runtimes', null, ctx.sessionHeaders, [200], 'functions.runtimes.list');
    api('GET', '/functions/specifications', null, ctx.apiHeaders, [200], 'functions.specifications.list');
    const functionVariable = api('POST', `/functions/${functionId}/variables`, {
        variableId: 'unique()',
        key: 'BENCHMARK',
        value: 'true',
        secret: false,
    }, ctx.apiHeaders, [201], 'functions.variables.create');
    functionVariableId = functionVariable.json('$id');

    api('PUT', `/functions/${functionId}/variables/${functionVariableId}`, {
        key: 'BENCHMARK',
        value: 'updated',
        secret: false,
    }, ctx.apiHeaders, [200], 'functions.variables.update');
    api('GET', `/functions/${functionId}/variables/${functionVariableId}`, null, ctx.apiHeaders, [200], 'functions.variables.get');
    api('DELETE', `/functions/${functionId}/variables/${functionVariableId}`, null, ctx.apiHeaders, [204], 'functions.variables.delete');
    api('DELETE', `/functions/${functionId}`, null, ctx.apiHeaders, [204], 'functions.delete');
}

function api(method, path, body, headers, expected, name) {
    const response = rawRequest(method, path, body, headers, name);
    apiDuration.add(response.timings.duration, { name });
    apiWaiting.add(response.timings.waiting, { name });
    assertStatus(response, expected, name);
    return response;
}

function setupApi(method, path, body, headers, expected, name) {
    const response = rawRequest(method, path, body, headers, name);
    assertStatus(response, expected, name);
    return response;
}

function rawRequest(method, path, body, headers, name) {
    const params = {
        headers,
        tags: { name },
    };
    const payload = body === null || body === undefined ? null : JSON.stringify(body);
    const response = http.request(method, `${ENDPOINT}${path}`, payload, params);
    httpWaiting.add(response.timings.waiting, { name });

    return response;
}

function waitForStatus(path, headers, wantedStatus, timeoutMs, name) {
    const started = Date.now();

    while (Date.now() - started < timeoutMs) {
        const response = rawRequest('GET', path, null, headers, name);
        if (response.status === 200) {
            const status = response.json('status');
            if (status === wantedStatus) {
                return response;
            }
            if (status === 'failed') {
                throw new Error(`${path} failed while waiting for ${wantedStatus}`);
            }
        }
        sleep(0.5);
    }

    throw new Error(`Timed out waiting for ${path} to become ${wantedStatus}`);
}

function assertStatus(response, expected, name) {
    const ok = check(response, {
        [`${name} status ${expected.join('|')}`]: (r) => expected.includes(r.status),
    });

    if (!ok) {
        failResponse(response, `${name} returned an unexpected status`);
    }
}

function failResponse(response, message) {
    throw new Error(`${message}. Status: ${response.status}. Body: ${response.body}`);
}

function cookieHeader(response) {
    return response.headers['Set-Cookie'] || response.headers['set-cookie'] || '';
}

function projectHeaders(projectId) {
    return {
        'Content-Type': 'application/json',
        'X-Appwrite-Project': projectId,
    };
}

function requireSession(ctx, flow) {
    if (!ctx.sessionHeaders || typeof ctx.sessionHeaders !== 'object') {
        throw new Error(`accountFlow must run before ${flow}`);
    }
}

function tablePayload() {
    return {
        title: 'Benchmark Row',
        quantity: 1,
        email: 'row@example.com',
        active: true,
    };
}

function onePixelPng() {
    return encoding.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII=', 'std', 'b');
}

function flattenMultipartArray(key, values) {
    const output = {};
    values.forEach((value, index) => {
        output[`${key}[${index}]`] = value;
    });
    return output;
}

function unique(prefix) {
    return `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`
        .toLowerCase()
        .replace(/[^a-z0-9-]/g, '-')
        .slice(0, 36);
}

function hostnameFromUrl(value) {
    return value.replace(/^https?:\/\//, '').split('/')[0].split(':')[0];
}

export function handleSummary(data) {
    const lines = [
        'Appwrite curated benchmark review',
        '',
        'Before',
        '',
        summaryTable(PREVIOUS_SUMMARY),
        '',
        'After',
        '',
        summaryTable(data),
        '',
        'Delta',
        '',
        deltaTable(PREVIOUS_SUMMARY, data),
        '',
    ];

    return {
        stdout: `${lines.join('\n')}\n`,
        [SUMMARY_PATH]: JSON.stringify(data, null, 2),
    };
}

function summaryTable(data) {
    return [
        '| Scenario | P50 (ms) | P95 (ms) | Requests | RPS |',
        '| --- | ---: | ---: | ---: | ---: |',
        summaryRow(data, 'API total', 'appwrite_api_duration'),
    ].join('\n');
}

function summaryRow(data, label, metric, iterationsMetric = null, rpsMetric = null) {
    const values = data && data.metrics[metric] && data.metrics[metric].values;
    if (!values || values.count === 0) {
        return `| ${label} | n/a | n/a | n/a | n/a |`;
    }

    const iterations = iterationsMetric
        ? trendMetric(data, iterationsMetric, 'count')
        : values.count;
    const rps = rpsMetric ? trendMetric(data, rpsMetric, 'rate') : null;

    return `| ${label} | ${formatDetailValue(values.med)} | ${formatDetailValue(values['p(95)'])} | ${formatCount(iterations)} | ${formatRate(rps)} |`;
}

function loadPreviousSummary(path) {
    let contents;
    try {
        contents = open(path);
    } catch (error) {
        console.warn(`Missing benchmark summary at ${path}: ${error.message}`);
        return null;
    }

    try {
        return JSON.parse(contents);
    } catch (error) {
        console.warn(`Invalid benchmark summary at ${path}: ${error.message}`);
        return null;
    }
}

function deltaTable(before, after) {
    return [
        '| Scenario | P95 delta (ms) |',
        '| --- | ---: |',
        ...[
            ['API total', 'appwrite_api_duration'],
        ].map(([label, metric]) => {
            const beforeP95 = trendMetric(before, metric, 'p(95)');
            const afterP95 = trendMetric(after, metric, 'p(95)');
            return `| ${label} | ${formatDelta(beforeP95, afterP95)} |`;
        }),
    ].join('\n');
}

function trendMetric(data, metric, stat) {
    return data && data.metrics[metric] && data.metrics[metric].values
        ? data.metrics[metric].values[stat]
        : null;
}

function formatDetailValue(value) {
    if (value === null || value === undefined || Number.isNaN(value)) {
        return 'n/a';
    }

    return `${Number(value).toFixed(2)}`;
}

function formatDelta(before, after) {
    if (before === null || before === undefined || after === null || after === undefined || Number.isNaN(before) || Number.isNaN(after)) {
        return 'n/a';
    }

    const delta = round(after - before);
    const sign = delta > 0 ? '+' : '';
    return `${sign}${delta}`;
}

function formatCount(value) {
    if (value === null || value === undefined || Number.isNaN(value)) {
        return 'n/a';
    }

    return `${Math.round(value)}`;
}

function formatRate(value) {
    if (value === null || value === undefined || Number.isNaN(value)) {
        return 'n/a';
    }

    return `${Number(value).toFixed(2)}`;
}

function round(value) {
    return Math.round((value || 0) * 100) / 100;
}