diff --git a/tests/benchmarks/bulk-operations.js b/tests/benchmarks/bulk-operations.js deleted file mode 100644 index a60129735a..0000000000 --- a/tests/benchmarks/bulk-operations.js +++ /dev/null @@ -1,32 +0,0 @@ -import { check } from "k6"; -import http from "k6/http"; - -const amount = 100_000; -const databaseId = "674918b20017411b94b2"; -const collectionId = "674918b4002b46c47d5d"; - -const documents = Array(amount).fill({ - $id: "unique()", - name: "asd", -}); - -export default function () { - const payload = JSON.stringify({ - documents, - }); - - const res = http.post(`http://localhost/v1/databases/${databaseId}/collections/${collectionId}/documents`, - payload, - { - headers: { - "X-Appwrite-Key": "standard_fa89c4834660f39e95ca2c2996fe7dd4ff498725e37c09323234c009570c1719f1c10610bf3541cf9ead120c107e41397a4eae1c787c83bdf577857bbc5963341641c77f582cc41e11a0d50eb4c2e4b1fda74418a8b9a253d6e63008e33560ba35310b9dc2fed5f09ca599e646f744cc6308b8ccd27ff04f9e498ec5a5f2c3db", - "X-Appwrite-Project": "674818bc0017934d58dd", - "Content-Type": "application/json", - }, - } - ); - - check(res, { - "status is 200": (r) => r.status === 201, - }); -} diff --git a/tests/benchmarks/bulk-operations/documentCreation.js b/tests/benchmarks/bulk-operations/documentCreation.js new file mode 100644 index 0000000000..730b6bb541 --- /dev/null +++ b/tests/benchmarks/bulk-operations/documentCreation.js @@ -0,0 +1,82 @@ +import { check, sleep } from "k6"; +import http from "k6/http"; +import { provisionProject, provisionDatabase, cleanup, unique } from "./utils.js"; + +const amount = 10_000; + +export function setup() { + const resources = provisionProject({ + endpoint: 'http://localhost/v1', + email: 'test@test.com', + password: 'password123', + name: 'Test User', + projectName: 'Bulk Operations Test' + }); + + const { databaseId, collectionId } = provisionDatabase({ + endpoint: 'http://localhost/v1', + apiHeaders: resources.apiHeaders + }); + + sleep(3); // Await Attributes to be provisioned + + console.log(`----- Amount of documents: ${amount} -----`); + + return { + databaseId, + collectionId, + apiHeaders: resources.apiHeaders, + resources + }; +} + +export function teardown(data) { + cleanup(data.resources); +} + +let documents = Array(amount).fill({ + $id: "unique()", + name: "asd", +}); + +documents = documents.map((document) => { + return { + ...document, + age: Math.floor(Math.random() * 100), + email: `${unique()}@test.com`, + height: Math.random() * 100, + }; +}); + +export default function (data) { + const payload = JSON.stringify({ + documents, + }); + + const res = http.post( + `http://localhost/v1/databases/${data.databaseId}/collections/${data.collectionId}/documents`, + payload, + { + headers: data.apiHeaders + } + ); + + check(res, { + "status is 201": (r) => r.status === 201, + }); + + return { + resources: data.resources + }; +} + +export const options = { + scenarios: { + bulk_create: { + executor: 'per-vu-iterations', + vus: 1, + iterations: 20, + exec: 'default' + } + } +}; diff --git a/tests/benchmarks/bulk-operations/noisyNeighbor.js b/tests/benchmarks/bulk-operations/noisyNeighbor.js new file mode 100644 index 0000000000..4bfec4e564 --- /dev/null +++ b/tests/benchmarks/bulk-operations/noisyNeighbor.js @@ -0,0 +1,144 @@ +import { check, sleep } from "k6"; +import http from "k6/http"; +import { Trend } from "k6/metrics"; +import { provisionProject, provisionDatabase, cleanup, unique } from "./utils.js"; + +// Custom Trend metric for light response time tracking +export const lightResponseTime = new Trend("light_response_time", true); + +const BULK_AMOUNT = 100_000; // Heavy operation amount +const LIGHT_AMOUNT = 10; // Light operation amount + +export function setup() { + // Set up two separate projects - one for bulk operations (noisy neighbor) and one for light operations + const heavyResources = provisionProject({ + endpoint: 'http://localhost/v1', + email: 'heavy@test.com', + password: 'password123', + name: 'Heavy User', + projectName: 'Noisy Neighbor - Heavy' + }); + + const lightResources = provisionProject({ + endpoint: 'http://localhost/v1', + email: 'light@test.com', + password: 'password123', + name: 'Light User', + projectName: 'Noisy Neighbor - Light' + }); + + // Set up databases for both projects + const heavy = provisionDatabase({ + endpoint: 'http://localhost/v1', + apiHeaders: heavyResources.apiHeaders + }); + + const light = provisionDatabase({ + endpoint: 'http://localhost/v1', + apiHeaders: lightResources.apiHeaders + }); + + sleep(3); // Await Attributes to be provisioned + + console.log(`----- Heavy operations: ${BULK_AMOUNT} documents | Light operations: ${LIGHT_AMOUNT} document -----`); + + return { + heavy: { + databaseId: heavy.databaseId, + collectionId: heavy.collectionId, + apiHeaders: heavyResources.apiHeaders, + resources: heavyResources + }, + light: { + databaseId: light.databaseId, + collectionId: light.collectionId, + apiHeaders: lightResources.apiHeaders, + resources: lightResources + } + }; +} + +export function teardown(data) { + cleanup(data.heavy.resources); + cleanup(data.light.resources); +} + +// Create document payloads +function createDocuments(amount) { + let documents = Array(amount).fill({ + $id: "unique()", + name: "test", + }); + + return documents.map((document) => ({ + ...document, + age: Math.floor(Math.random() * 100), + email: `${unique()}@test.com`, + height: Math.random() * 100, + })); +} + +// Heavy operation function +export function heavy(data) { + const documents = createDocuments(BULK_AMOUNT); + const payload = JSON.stringify({ documents }); + + const res = http.post( + `http://localhost/v1/databases/${data.heavy.databaseId}/collections/${data.heavy.collectionId}/documents`, + payload, + { + headers: data.heavy.apiHeaders + } + ); + + check(res, { + "heavy operation status is 201": (r) => r.status === 201, + }); +} + +// Light operation function +export function light(data) { + const documents = createDocuments(LIGHT_AMOUNT); + const payload = JSON.stringify({ documents }); + + const startTime = new Date(); + const res = http.post( + `http://localhost/v1/databases/${data.light.databaseId}/collections/${data.light.collectionId}/documents`, + payload, + { + headers: data.light.apiHeaders + } + ); + const duration = new Date() - startTime; + + // Record the light operation response time using the custom Trend metric + lightResponseTime.add(duration); + + check(res, { + "light operation status is 201": (r) => r.status === 201, + }); +} + +export const options = { + scenarios: { + // Heavy bulk operations running continuously + heavy_load: { + executor: 'constant-vus', + vus: 5, + duration: '30s', + exec: 'heavy' + }, + // Light operations to measure impact + light_operations: { + executor: 'constant-arrival-rate', + rate: 5, + timeUnit: '1s', + duration: '30s', + preAllocatedVUs: 10, + exec: 'light' + } + }, + thresholds: { + http_req_duration: ['p(95)<2000'], // 95% of requests should complete within 2s + } +}; \ No newline at end of file diff --git a/tests/benchmarks/bulk-operations/utils.js b/tests/benchmarks/bulk-operations/utils.js new file mode 100644 index 0000000000..dc8dcac569 --- /dev/null +++ b/tests/benchmarks/bulk-operations/utils.js @@ -0,0 +1,336 @@ +import http from 'k6/http'; +import { check } from 'k6'; + +/** + * @typedef {Object} AuthHeaders + * @property {string} 'Content-Type' - Content type header + * @property {string} 'Cookie' - Session cookie + * @property {string} 'X-Appwrite-Project' - Project ID header + */ + +/** + * @typedef {Object} ApiHeaders + * @property {string} 'Content-Type' - Content type header + * @property {string} 'X-Appwrite-Project' - Project ID header + * @property {string} 'X-Appwrite-Key' - API key header + */ + +/** + * @typedef {Object} ProvisionedResources + * @property {string} userId - The ID of the created user + * @property {string} teamId - The ID of the created team + * @property {string} projectId - The ID of the created project + * @property {string} cookies - Session cookies for authentication + * @property {AuthHeaders} headers - Headers for cookie-based authentication + * @property {string} apiKey - The API key secret + * @property {ApiHeaders} apiHeaders - Headers for API key authentication + */ + +function assert(response, checkName, condition) { + const result = check(response, { + [checkName]: condition + }); + if (!result) { + console.error(`Assertion failed: ${checkName}`); + console.error(`Response status: ${response.status}`); + console.error(`Response body: ${response.body}`); + throw new Error(`Assertion failed: ${checkName}`); + } +} + +/** + * Provisions an Appwrite project setup including: + * - Account creation + * - Session creation + * - Team creation + * - Project creation + * - API Key creation + * + * @param {Object} config Configuration object + * @param {string} config.endpoint Base endpoint URL (e.g., 'http://localhost:80/v1') + * @param {string} config.email Email for account creation + * @param {string} config.password Password for account creation + * @param {string} config.name Name for account creation + * @param {string} config.projectName Name for the project + * @returns {ProvisionedResources} Object containing all created resource IDs and session information + */ +export function provisionProject(config) { + const { + endpoint, + email, + password, + name, + projectName, + } = config; + + // Step 1: Create Account + const accountResponse = http.post(`${endpoint}/account`, JSON.stringify({ + userId: 'unique()', + email, + password, + name + }), { + headers: { + 'Content-Type': 'application/json', + } + }); + + assert(accountResponse, 'account created successfully', (r) => r.status === 201 || r.status === 409); + + const userId = accountResponse.json('$id'); + + // Step 2: Create Session + const sessionResponse = http.post(`${endpoint}/account/sessions/email`, JSON.stringify({ + email, + password + }), { + headers: { + 'Content-Type': 'application/json', + } + }); + + assert(sessionResponse, 'session created successfully', (r) => r.status === 201); + + // Keep manual control of the cookies to allow for simultaneous requests + const jar = http.cookieJar(); + jar.clear(`${endpoint}`); + + // Extract cookies for subsequent requests + const cookies = sessionResponse.headers['Set-Cookie']; + + // Common headers for authenticated requests + const authHeaders = { + 'Content-Type': 'application/json', + 'Cookie': cookies + }; + + // Step 3: Create Team + const teamResponse = http.post(`${endpoint}/teams`, JSON.stringify({ + teamId: 'unique()', + name: `${projectName} Team` + }), { + headers: authHeaders + }); + + assert(teamResponse, 'team created successfully', (r) => r.status === 201); + + const teamId = teamResponse.json('$id'); + + // Step 4: Create Project + const projectResponse = http.post(`${endpoint}/projects`, JSON.stringify({ + projectId: 'unique()', + name: projectName, + teamId: teamId + }), { + headers: authHeaders + }); + + assert(projectResponse, 'project created successfully', (r) => r.status === 201); + + const projectId = projectResponse.json('$id'); + + // Step 5: Create API Key + const apiKeyResponse = http.post(`${endpoint}/projects/${projectId}/keys`, JSON.stringify({ + name: 'Test API Key', + scopes: SCOPES, // All permissions + }), { + headers: authHeaders + }); + + assert(apiKeyResponse, 'api key created successfully', (r) => r.status === 201); + + const apiKey = apiKeyResponse.json('secret'); + + // Create a new headers object for API key authentication + const apiHeaders = { + 'Content-Type': 'application/json', + 'X-Appwrite-Project': projectId, + 'X-Appwrite-Key': apiKey + }; + + // Return all created resources and session info + return { + endpoint, + userId, + teamId, + projectId, + cookies, + headers: authHeaders, + apiKey, + apiHeaders + }; +} + +/** + * Example usage: + * + * const config = { + * endpoint: 'http://localhost:80/v1', + * email: 'test@example.com', + * password: 'complex-password', + * name: 'Test User', + * projectName: 'Test Project' + * }; + * + * const resources = provisionProject(config); + */ + +const SCOPES = [ + "sessions.write", + "users.read", + "users.write", + "teams.read", + "teams.write", + "databases.read", + "databases.write", + "collections.read", + "collections.write", + "attributes.read", + "attributes.write", + "indexes.read", + "indexes.write", + "documents.read", + "documents.write", + "files.read", + "files.write", + "buckets.read", + "buckets.write", + "functions.read", + "functions.write", + "execution.read", + "execution.write", + "targets.read", + "targets.write", + "providers.read", + "providers.write", + "messages.read", + "messages.write", + "topics.read", + "topics.write", + "subscribers.read", + "subscribers.write", + "locale.read", + "avatars.read", + "health.read", + "migrations.read", + "migrations.write" +] + +export function provisionDatabase(config) { + const { + endpoint, + apiHeaders + } = config; + + // Create database + const databaseResponse = http.post( + `${endpoint}/databases`, + JSON.stringify({ + databaseId: 'unique()', + name: 'Bulk Test DB' + }), + { headers: apiHeaders } + ); + + assert(databaseResponse, 'database created successfully', (r) => r.status === 201); + + const databaseId = databaseResponse.json('$id'); + + // Create collection + const collectionResponse = http.post( + `${endpoint}/databases/${databaseId}/collections`, + JSON.stringify({ + collectionId: 'unique()', + name: 'Bulk Test Collection', + permissions: ['read("any")', 'write("any")'], + documentSecurity: false + }), + { headers: apiHeaders } + ); + + assert(collectionResponse, 'collection created successfully', (r) => r.status === 201); + + const collectionId = collectionResponse.json('$id'); + + // Create name attribute + const nameAttributeResponse = http.post( + `${endpoint}/databases/${databaseId}/collections/${collectionId}/attributes/string`, + JSON.stringify({ + key: 'name', + size: 100, + required: false, + default: null, + array: false, + encrypt: false + }), + { headers: apiHeaders } + ); + + assert(nameAttributeResponse, 'name attribute created successfully', (r) => r.status === 202); + + // Create age attribute + const ageAttributeResponse = http.post( + `${endpoint}/databases/${databaseId}/collections/${collectionId}/attributes/integer`, + JSON.stringify({ + key: 'age', + required: false, + }), + { headers: apiHeaders } + ); + + assert(ageAttributeResponse, 'age attribute created successfully', (r) => r.status === 202); + + // Create email attribute + const emailAttributeResponse = http.post( + `${endpoint}/databases/${databaseId}/collections/${collectionId}/attributes/email`, + JSON.stringify({ + key: 'email', + required: false, + }), + { headers: apiHeaders } + ); + + assert(emailAttributeResponse, 'email attribute created successfully', (r) => r.status === 202); + + // Create height attribute + const heightAttributeResponse = http.post( + `${endpoint}/databases/${databaseId}/collections/${collectionId}/attributes/float`, + JSON.stringify({ + key: 'height', + required: false, + }), + { headers: apiHeaders } + ); + + assert(heightAttributeResponse, 'height attribute created successfully', (r) => r.status === 202); + + return { + databaseId, + collectionId + }; +} + +export function cleanup(config) { + const { + endpoint, + teamId, + headers + } = config; + + // Delete Organization + const organizationResponse = http.del( + `${endpoint}/teams/${teamId}`, + null, + { + headers + } + ); + + assert(organizationResponse, 'organization deleted successfully', (r) => r.status === 204); +} + +export function unique() { + const timestamp = Date.now().toString(36); + const randomPart = Math.random().toString(36).substring(2, 15); + return `${timestamp}-${randomPart}`; +} \ No newline at end of file