From a8a5b4d00c46b4528bd4859c4874004449189511 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 11 Feb 2025 09:24:18 +0900 Subject: [PATCH 1/5] Update docs/references/databases/update-documents.md Co-authored-by: Jake Barnby --- docs/references/databases/update-documents.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/references/databases/update-documents.md b/docs/references/databases/update-documents.md index f04ec420f8..5f560c6435 100644 --- a/docs/references/databases/update-documents.md +++ b/docs/references/databases/update-documents.md @@ -1 +1 @@ -Update all documents that match your queries, If none are submitted then all documents are updated. Using the patch method you can pass only specific fields that will get updated. \ No newline at end of file +Update all documents that match your queries, if no queries are submitted then all documents are updated. You can pass only specific fields to be updated. \ No newline at end of file From 709f616f7f7ad0f8b509e02741317a288242b814 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 11 Feb 2025 09:24:26 +0900 Subject: [PATCH 2/5] Update app/controllers/api/databases.php Co-authored-by: Jake Barnby --- app/controllers/api/databases.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 9ab779b355..2d5f17f97b 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3178,7 +3178,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } if (!empty($documents) && !empty($documentId)) { - throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Param "documentId" is disallowed when creating multiple documents, use $id inside the documents'); + throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Param "documentId" is disallowed when creating multiple documents, set $id in each document'); } if (!empty($data)) { From d49087961579cc97a6dd487368934298a042782f Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 11 Feb 2025 09:24:35 +0900 Subject: [PATCH 3/5] Update app/controllers/api/databases.php Co-authored-by: Jake Barnby --- app/controllers/api/databases.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 2d5f17f97b..86d7cc0eeb 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3339,7 +3339,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $document['$id'] = $documentId == 'unique()' ? ID::unique() : $documentId; } else { if (empty($document['$id'])) { - throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, '$id is required inside documents when creating bulk documents'); + throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, '$id must be set in each document when creating bulk documents'); } $document['$id'] = $document['$id'] == 'unique()' ? ID::unique() : $document['$id']; From 0ccaf9adffa4e8e9068acc5a21018e1bdb67e0fb Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 11 Feb 2025 15:57:27 +0900 Subject: [PATCH 4/5] Add more benchmarks --- tests/benchmarks/bulk-operations.js | 32 -- .../bulk-operations/documentCreation.js | 82 +++++ .../bulk-operations/noisyNeighbor.js | 144 ++++++++ tests/benchmarks/bulk-operations/utils.js | 336 ++++++++++++++++++ 4 files changed, 562 insertions(+), 32 deletions(-) delete mode 100644 tests/benchmarks/bulk-operations.js create mode 100644 tests/benchmarks/bulk-operations/documentCreation.js create mode 100644 tests/benchmarks/bulk-operations/noisyNeighbor.js create mode 100644 tests/benchmarks/bulk-operations/utils.js diff --git a/tests/benchmarks/bulk-operations.js b/tests/benchmarks/bulk-operations.js deleted file mode 100644 index a60129735a..0000000000 --- a/tests/benchmarks/bulk-operations.js +++ /dev/null @@ -1,32 +0,0 @@ -import { check } from "k6"; -import http from "k6/http"; - -const amount = 100_000; -const databaseId = "674918b20017411b94b2"; -const collectionId = "674918b4002b46c47d5d"; - -const documents = Array(amount).fill({ - $id: "unique()", - name: "asd", -}); - -export default function () { - const payload = JSON.stringify({ - documents, - }); - - const res = http.post(`http://localhost/v1/databases/${databaseId}/collections/${collectionId}/documents`, - payload, - { - headers: { - "X-Appwrite-Key": "standard_fa89c4834660f39e95ca2c2996fe7dd4ff498725e37c09323234c009570c1719f1c10610bf3541cf9ead120c107e41397a4eae1c787c83bdf577857bbc5963341641c77f582cc41e11a0d50eb4c2e4b1fda74418a8b9a253d6e63008e33560ba35310b9dc2fed5f09ca599e646f744cc6308b8ccd27ff04f9e498ec5a5f2c3db", - "X-Appwrite-Project": "674818bc0017934d58dd", - "Content-Type": "application/json", - }, - } - ); - - check(res, { - "status is 200": (r) => r.status === 201, - }); -} diff --git a/tests/benchmarks/bulk-operations/documentCreation.js b/tests/benchmarks/bulk-operations/documentCreation.js new file mode 100644 index 0000000000..730b6bb541 --- /dev/null +++ b/tests/benchmarks/bulk-operations/documentCreation.js @@ -0,0 +1,82 @@ +import { check, sleep } from "k6"; +import http from "k6/http"; +import { provisionProject, provisionDatabase, cleanup, unique } from "./utils.js"; + +const amount = 10_000; + +export function setup() { + const resources = provisionProject({ + endpoint: 'http://localhost/v1', + email: 'test@test.com', + password: 'password123', + name: 'Test User', + projectName: 'Bulk Operations Test' + }); + + const { databaseId, collectionId } = provisionDatabase({ + endpoint: 'http://localhost/v1', + apiHeaders: resources.apiHeaders + }); + + sleep(3); // Await Attributes to be provisioned + + console.log(`----- Amount of documents: ${amount} -----`); + + return { + databaseId, + collectionId, + apiHeaders: resources.apiHeaders, + resources + }; +} + +export function teardown(data) { + cleanup(data.resources); +} + +let documents = Array(amount).fill({ + $id: "unique()", + name: "asd", +}); + +documents = documents.map((document) => { + return { + ...document, + age: Math.floor(Math.random() * 100), + email: `${unique()}@test.com`, + height: Math.random() * 100, + }; +}); + +export default function (data) { + const payload = JSON.stringify({ + documents, + }); + + const res = http.post( + `http://localhost/v1/databases/${data.databaseId}/collections/${data.collectionId}/documents`, + payload, + { + headers: data.apiHeaders + } + ); + + check(res, { + "status is 201": (r) => r.status === 201, + }); + + return { + resources: data.resources + }; +} + +export const options = { + scenarios: { + bulk_create: { + executor: 'per-vu-iterations', + vus: 1, + iterations: 20, + exec: 'default' + } + } +}; diff --git a/tests/benchmarks/bulk-operations/noisyNeighbor.js b/tests/benchmarks/bulk-operations/noisyNeighbor.js new file mode 100644 index 0000000000..4bfec4e564 --- /dev/null +++ b/tests/benchmarks/bulk-operations/noisyNeighbor.js @@ -0,0 +1,144 @@ +import { check, sleep } from "k6"; +import http from "k6/http"; +import { Trend } from "k6/metrics"; +import { provisionProject, provisionDatabase, cleanup, unique } from "./utils.js"; + +// Custom Trend metric for light response time tracking +export const lightResponseTime = new Trend("light_response_time", true); + +const BULK_AMOUNT = 100_000; // Heavy operation amount +const LIGHT_AMOUNT = 10; // Light operation amount + +export function setup() { + // Set up two separate projects - one for bulk operations (noisy neighbor) and one for light operations + const heavyResources = provisionProject({ + endpoint: 'http://localhost/v1', + email: 'heavy@test.com', + password: 'password123', + name: 'Heavy User', + projectName: 'Noisy Neighbor - Heavy' + }); + + const lightResources = provisionProject({ + endpoint: 'http://localhost/v1', + email: 'light@test.com', + password: 'password123', + name: 'Light User', + projectName: 'Noisy Neighbor - Light' + }); + + // Set up databases for both projects + const heavy = provisionDatabase({ + endpoint: 'http://localhost/v1', + apiHeaders: heavyResources.apiHeaders + }); + + const light = provisionDatabase({ + endpoint: 'http://localhost/v1', + apiHeaders: lightResources.apiHeaders + }); + + sleep(3); // Await Attributes to be provisioned + + console.log(`----- Heavy operations: ${BULK_AMOUNT} documents | Light operations: ${LIGHT_AMOUNT} document -----`); + + return { + heavy: { + databaseId: heavy.databaseId, + collectionId: heavy.collectionId, + apiHeaders: heavyResources.apiHeaders, + resources: heavyResources + }, + light: { + databaseId: light.databaseId, + collectionId: light.collectionId, + apiHeaders: lightResources.apiHeaders, + resources: lightResources + } + }; +} + +export function teardown(data) { + cleanup(data.heavy.resources); + cleanup(data.light.resources); +} + +// Create document payloads +function createDocuments(amount) { + let documents = Array(amount).fill({ + $id: "unique()", + name: "test", + }); + + return documents.map((document) => ({ + ...document, + age: Math.floor(Math.random() * 100), + email: `${unique()}@test.com`, + height: Math.random() * 100, + })); +} + +// Heavy operation function +export function heavy(data) { + const documents = createDocuments(BULK_AMOUNT); + const payload = JSON.stringify({ documents }); + + const res = http.post( + `http://localhost/v1/databases/${data.heavy.databaseId}/collections/${data.heavy.collectionId}/documents`, + payload, + { + headers: data.heavy.apiHeaders + } + ); + + check(res, { + "heavy operation status is 201": (r) => r.status === 201, + }); +} + +// Light operation function +export function light(data) { + const documents = createDocuments(LIGHT_AMOUNT); + const payload = JSON.stringify({ documents }); + + const startTime = new Date(); + const res = http.post( + `http://localhost/v1/databases/${data.light.databaseId}/collections/${data.light.collectionId}/documents`, + payload, + { + headers: data.light.apiHeaders + } + ); + const duration = new Date() - startTime; + + // Record the light operation response time using the custom Trend metric + lightResponseTime.add(duration); + + check(res, { + "light operation status is 201": (r) => r.status === 201, + }); +} + +export const options = { + scenarios: { + // Heavy bulk operations running continuously + heavy_load: { + executor: 'constant-vus', + vus: 5, + duration: '30s', + exec: 'heavy' + }, + // Light operations to measure impact + light_operations: { + executor: 'constant-arrival-rate', + rate: 5, + timeUnit: '1s', + duration: '30s', + preAllocatedVUs: 10, + exec: 'light' + } + }, + thresholds: { + http_req_duration: ['p(95)<2000'], // 95% of requests should complete within 2s + } +}; \ No newline at end of file diff --git a/tests/benchmarks/bulk-operations/utils.js b/tests/benchmarks/bulk-operations/utils.js new file mode 100644 index 0000000000..dc8dcac569 --- /dev/null +++ b/tests/benchmarks/bulk-operations/utils.js @@ -0,0 +1,336 @@ +import http from 'k6/http'; +import { check } from 'k6'; + +/** + * @typedef {Object} AuthHeaders + * @property {string} 'Content-Type' - Content type header + * @property {string} 'Cookie' - Session cookie + * @property {string} 'X-Appwrite-Project' - Project ID header + */ + +/** + * @typedef {Object} ApiHeaders + * @property {string} 'Content-Type' - Content type header + * @property {string} 'X-Appwrite-Project' - Project ID header + * @property {string} 'X-Appwrite-Key' - API key header + */ + +/** + * @typedef {Object} ProvisionedResources + * @property {string} userId - The ID of the created user + * @property {string} teamId - The ID of the created team + * @property {string} projectId - The ID of the created project + * @property {string} cookies - Session cookies for authentication + * @property {AuthHeaders} headers - Headers for cookie-based authentication + * @property {string} apiKey - The API key secret + * @property {ApiHeaders} apiHeaders - Headers for API key authentication + */ + +function assert(response, checkName, condition) { + const result = check(response, { + [checkName]: condition + }); + if (!result) { + console.error(`Assertion failed: ${checkName}`); + console.error(`Response status: ${response.status}`); + console.error(`Response body: ${response.body}`); + throw new Error(`Assertion failed: ${checkName}`); + } +} + +/** + * Provisions an Appwrite project setup including: + * - Account creation + * - Session creation + * - Team creation + * - Project creation + * - API Key creation + * + * @param {Object} config Configuration object + * @param {string} config.endpoint Base endpoint URL (e.g., 'http://localhost:80/v1') + * @param {string} config.email Email for account creation + * @param {string} config.password Password for account creation + * @param {string} config.name Name for account creation + * @param {string} config.projectName Name for the project + * @returns {ProvisionedResources} Object containing all created resource IDs and session information + */ +export function provisionProject(config) { + const { + endpoint, + email, + password, + name, + projectName, + } = config; + + // Step 1: Create Account + const accountResponse = http.post(`${endpoint}/account`, JSON.stringify({ + userId: 'unique()', + email, + password, + name + }), { + headers: { + 'Content-Type': 'application/json', + } + }); + + assert(accountResponse, 'account created successfully', (r) => r.status === 201 || r.status === 409); + + const userId = accountResponse.json('$id'); + + // Step 2: Create Session + const sessionResponse = http.post(`${endpoint}/account/sessions/email`, JSON.stringify({ + email, + password + }), { + headers: { + 'Content-Type': 'application/json', + } + }); + + assert(sessionResponse, 'session created successfully', (r) => r.status === 201); + + // Keep manual control of the cookies to allow for simultaneous requests + const jar = http.cookieJar(); + jar.clear(`${endpoint}`); + + // Extract cookies for subsequent requests + const cookies = sessionResponse.headers['Set-Cookie']; + + // Common headers for authenticated requests + const authHeaders = { + 'Content-Type': 'application/json', + 'Cookie': cookies + }; + + // Step 3: Create Team + const teamResponse = http.post(`${endpoint}/teams`, JSON.stringify({ + teamId: 'unique()', + name: `${projectName} Team` + }), { + headers: authHeaders + }); + + assert(teamResponse, 'team created successfully', (r) => r.status === 201); + + const teamId = teamResponse.json('$id'); + + // Step 4: Create Project + const projectResponse = http.post(`${endpoint}/projects`, JSON.stringify({ + projectId: 'unique()', + name: projectName, + teamId: teamId + }), { + headers: authHeaders + }); + + assert(projectResponse, 'project created successfully', (r) => r.status === 201); + + const projectId = projectResponse.json('$id'); + + // Step 5: Create API Key + const apiKeyResponse = http.post(`${endpoint}/projects/${projectId}/keys`, JSON.stringify({ + name: 'Test API Key', + scopes: SCOPES, // All permissions + }), { + headers: authHeaders + }); + + assert(apiKeyResponse, 'api key created successfully', (r) => r.status === 201); + + const apiKey = apiKeyResponse.json('secret'); + + // Create a new headers object for API key authentication + const apiHeaders = { + 'Content-Type': 'application/json', + 'X-Appwrite-Project': projectId, + 'X-Appwrite-Key': apiKey + }; + + // Return all created resources and session info + return { + endpoint, + userId, + teamId, + projectId, + cookies, + headers: authHeaders, + apiKey, + apiHeaders + }; +} + +/** + * Example usage: + * + * const config = { + * endpoint: 'http://localhost:80/v1', + * email: 'test@example.com', + * password: 'complex-password', + * name: 'Test User', + * projectName: 'Test Project' + * }; + * + * const resources = provisionProject(config); + */ + +const SCOPES = [ + "sessions.write", + "users.read", + "users.write", + "teams.read", + "teams.write", + "databases.read", + "databases.write", + "collections.read", + "collections.write", + "attributes.read", + "attributes.write", + "indexes.read", + "indexes.write", + "documents.read", + "documents.write", + "files.read", + "files.write", + "buckets.read", + "buckets.write", + "functions.read", + "functions.write", + "execution.read", + "execution.write", + "targets.read", + "targets.write", + "providers.read", + "providers.write", + "messages.read", + "messages.write", + "topics.read", + "topics.write", + "subscribers.read", + "subscribers.write", + "locale.read", + "avatars.read", + "health.read", + "migrations.read", + "migrations.write" +] + +export function provisionDatabase(config) { + const { + endpoint, + apiHeaders + } = config; + + // Create database + const databaseResponse = http.post( + `${endpoint}/databases`, + JSON.stringify({ + databaseId: 'unique()', + name: 'Bulk Test DB' + }), + { headers: apiHeaders } + ); + + assert(databaseResponse, 'database created successfully', (r) => r.status === 201); + + const databaseId = databaseResponse.json('$id'); + + // Create collection + const collectionResponse = http.post( + `${endpoint}/databases/${databaseId}/collections`, + JSON.stringify({ + collectionId: 'unique()', + name: 'Bulk Test Collection', + permissions: ['read("any")', 'write("any")'], + documentSecurity: false + }), + { headers: apiHeaders } + ); + + assert(collectionResponse, 'collection created successfully', (r) => r.status === 201); + + const collectionId = collectionResponse.json('$id'); + + // Create name attribute + const nameAttributeResponse = http.post( + `${endpoint}/databases/${databaseId}/collections/${collectionId}/attributes/string`, + JSON.stringify({ + key: 'name', + size: 100, + required: false, + default: null, + array: false, + encrypt: false + }), + { headers: apiHeaders } + ); + + assert(nameAttributeResponse, 'name attribute created successfully', (r) => r.status === 202); + + // Create age attribute + const ageAttributeResponse = http.post( + `${endpoint}/databases/${databaseId}/collections/${collectionId}/attributes/integer`, + JSON.stringify({ + key: 'age', + required: false, + }), + { headers: apiHeaders } + ); + + assert(ageAttributeResponse, 'age attribute created successfully', (r) => r.status === 202); + + // Create email attribute + const emailAttributeResponse = http.post( + `${endpoint}/databases/${databaseId}/collections/${collectionId}/attributes/email`, + JSON.stringify({ + key: 'email', + required: false, + }), + { headers: apiHeaders } + ); + + assert(emailAttributeResponse, 'email attribute created successfully', (r) => r.status === 202); + + // Create height attribute + const heightAttributeResponse = http.post( + `${endpoint}/databases/${databaseId}/collections/${collectionId}/attributes/float`, + JSON.stringify({ + key: 'height', + required: false, + }), + { headers: apiHeaders } + ); + + assert(heightAttributeResponse, 'height attribute created successfully', (r) => r.status === 202); + + return { + databaseId, + collectionId + }; +} + +export function cleanup(config) { + const { + endpoint, + teamId, + headers + } = config; + + // Delete Organization + const organizationResponse = http.del( + `${endpoint}/teams/${teamId}`, + null, + { + headers + } + ); + + assert(organizationResponse, 'organization deleted successfully', (r) => r.status === 204); +} + +export function unique() { + const timestamp = Date.now().toString(36); + const randomPart = Math.random().toString(36).substring(2, 15); + return `${timestamp}-${randomPart}`; +} \ No newline at end of file From c560524a42f08471940567505896c41f166fc3a0 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Thu, 13 Feb 2025 16:52:53 +0900 Subject: [PATCH 5/5] Add more benchmarks, update DB Lib --- composer.json | 6 +- composer.lock | 80 +++++------ .../bulk-operations/LargeDocumentCreation.js | 127 ++++++++++++++++++ 3 files changed, 170 insertions(+), 43 deletions(-) create mode 100644 tests/benchmarks/bulk-operations/LargeDocumentCreation.js diff --git a/composer.json b/composer.json index dc86ab2346..3e696334b0 100644 --- a/composer.json +++ b/composer.json @@ -45,13 +45,13 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.16.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "0.49.*", + "utopia-php/abuse": "0.50.*", "utopia-php/analytics": "0.10.*", - "utopia-php/audit": "0.49.*", + "utopia-php/audit": "0.51.*", "utopia-php/cache": "0.11.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.58.4", + "utopia-php/database": "0.59.0", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index df834c4c02..6365889eea 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "53423a0249dc52d6d27bc26dc9425206", + "content-hash": "b17c58729c4380afcba7714e9bced863", "packages": [ { "name": "adhocore/jwt", @@ -3377,16 +3377,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.49.0", + "version": "0.50.0", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "76612c274b895aa3d4d1fa27557a6402463eea99" + "reference": "3ff67819e9de61506c5ca070a70552f7ebe99f80" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/76612c274b895aa3d4d1fa27557a6402463eea99", - "reference": "76612c274b895aa3d4d1fa27557a6402463eea99", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/3ff67819e9de61506c5ca070a70552f7ebe99f80", + "reference": "3ff67819e9de61506c5ca070a70552f7ebe99f80", "shasum": "" }, "require": { @@ -3394,7 +3394,7 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/database": "0.58.*" + "utopia-php/database": "0.59.*" }, "require-dev": { "laravel/pint": "1.*", @@ -3422,9 +3422,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.49.0" + "source": "https://github.com/utopia-php/abuse/tree/0.50.0" }, - "time": "2025-02-04T07:33:59+00:00" + "time": "2025-02-12T09:13:59+00:00" }, { "name": "utopia-php/analytics", @@ -3474,21 +3474,21 @@ }, { "name": "utopia-php/audit", - "version": "0.49.0", + "version": "0.51.0", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "9d5c5e0cf0f6d9157b911fc3971da4331d71c96d" + "reference": "a5a4b73a57e27a0fac8025b1d6038e145a1ca04e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/9d5c5e0cf0f6d9157b911fc3971da4331d71c96d", - "reference": "9d5c5e0cf0f6d9157b911fc3971da4331d71c96d", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/a5a4b73a57e27a0fac8025b1d6038e145a1ca04e", + "reference": "a5a4b73a57e27a0fac8025b1d6038e145a1ca04e", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/database": "0.58.*" + "utopia-php/database": "0.59.*" }, "require-dev": { "laravel/pint": "1.*", @@ -3515,9 +3515,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.49.0" + "source": "https://github.com/utopia-php/audit/tree/0.51.0" }, - "time": "2025-02-04T07:27:18+00:00" + "time": "2025-02-12T09:12:44+00:00" }, { "name": "utopia-php/cache", @@ -3717,16 +3717,16 @@ }, { "name": "utopia-php/database", - "version": "0.58.4", + "version": "0.59.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "ff3fd22e4fe757cc2a78f17169f6dcc45c96d0fe" + "reference": "0eed7f1ad3eb66ff4a7d73b68dd9d3e05089eb18" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/ff3fd22e4fe757cc2a78f17169f6dcc45c96d0fe", - "reference": "ff3fd22e4fe757cc2a78f17169f6dcc45c96d0fe", + "url": "https://api.github.com/repos/utopia-php/database/zipball/0eed7f1ad3eb66ff4a7d73b68dd9d3e05089eb18", + "reference": "0eed7f1ad3eb66ff4a7d73b68dd9d3e05089eb18", "shasum": "" }, "require": { @@ -3767,9 +3767,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.58.4" + "source": "https://github.com/utopia-php/database/tree/0.59.0" }, - "time": "2025-02-05T02:51:02+00:00" + "time": "2025-02-12T08:08:29+00:00" }, { "name": "utopia-php/domains", @@ -4170,16 +4170,16 @@ }, { "name": "utopia-php/migration", - "version": "0.6.17", + "version": "0.6.18", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "677a5c4688d7f54d1631a91f76a35d51346cf96b" + "reference": "013fe03ff17fffd80dbd3564dc1a0cdd4f4a98ae" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/677a5c4688d7f54d1631a91f76a35d51346cf96b", - "reference": "677a5c4688d7f54d1631a91f76a35d51346cf96b", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/013fe03ff17fffd80dbd3564dc1a0cdd4f4a98ae", + "reference": "013fe03ff17fffd80dbd3564dc1a0cdd4f4a98ae", "shasum": "" }, "require": { @@ -4187,7 +4187,7 @@ "ext-curl": "*", "ext-openssl": "*", "php": ">=8.1", - "utopia-php/database": "0.58.*", + "utopia-php/database": "0.59.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*", "utopia-php/storage": "0.18.*" @@ -4220,9 +4220,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.17" + "source": "https://github.com/utopia-php/migration/tree/0.6.18" }, - "time": "2025-02-05T05:27:29+00:00" + "time": "2025-02-12T09:09:32+00:00" }, { "name": "utopia-php/mongo", @@ -4607,16 +4607,16 @@ }, { "name": "utopia-php/storage", - "version": "0.18.8", + "version": "0.18.9", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "84737afa634e6a833fc4f8b0c967553234d3f215" + "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/84737afa634e6a833fc4f8b0c967553234d3f215", - "reference": "84737afa634e6a833fc4f8b0c967553234d3f215", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/1cf455404e8700b3093fd73d74a38d41cdced90c", + "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c", "shasum": "" }, "require": { @@ -4656,9 +4656,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.18.8" + "source": "https://github.com/utopia-php/storage/tree/0.18.9" }, - "time": "2024-12-04T08:30:35+00:00" + "time": "2025-02-11T13:10:40+00:00" }, { "name": "utopia-php/swoole", @@ -5560,16 +5560,16 @@ }, { "name": "myclabs/deep-copy", - "version": "1.12.1", + "version": "1.13.0", "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "123267b2c49fbf30d78a7b2d333f6be754b94845" + "reference": "024473a478be9df5fdaca2c793f2232fe788e414" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/123267b2c49fbf30d78a7b2d333f6be754b94845", - "reference": "123267b2c49fbf30d78a7b2d333f6be754b94845", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/024473a478be9df5fdaca2c793f2232fe788e414", + "reference": "024473a478be9df5fdaca2c793f2232fe788e414", "shasum": "" }, "require": { @@ -5608,7 +5608,7 @@ ], "support": { "issues": "https://github.com/myclabs/DeepCopy/issues", - "source": "https://github.com/myclabs/DeepCopy/tree/1.12.1" + "source": "https://github.com/myclabs/DeepCopy/tree/1.13.0" }, "funding": [ { @@ -5616,7 +5616,7 @@ "type": "tidelift" } ], - "time": "2024-11-08T17:47:46+00:00" + "time": "2025-02-12T12:17:51+00:00" }, { "name": "nikic/php-parser", diff --git a/tests/benchmarks/bulk-operations/LargeDocumentCreation.js b/tests/benchmarks/bulk-operations/LargeDocumentCreation.js new file mode 100644 index 0000000000..c903ed2dba --- /dev/null +++ b/tests/benchmarks/bulk-operations/LargeDocumentCreation.js @@ -0,0 +1,127 @@ +import { check, sleep } from "k6"; +import http from "k6/http"; +import { provisionProject, provisionDatabase, cleanup, unique } from "./utils.js"; + +const millionRecords = 1_000_000; +const batchSize = 10_000; +const numBatches = millionRecords / batchSize; + +export function setup() { + const resources = provisionProject({ + endpoint: 'http://localhost/v1', + email: 'test@test.com', + password: 'password123', + name: 'Test User', + projectName: 'Large Document Creation Test' + }); + + const { databaseId, collectionId } = provisionDatabase({ + endpoint: 'http://localhost/v1', + apiHeaders: resources.apiHeaders + }); + + // Wait to ensure that provisioning is complete + sleep(5); + + // Create an index for the collection + const index = { + key: "name", + type: "fulltext", + orders: ["ASC"], + attributes: ["name", "email"] + }; + + const indexRes = http.post(`http://localhost/v1/databases/${databaseId}/collections/${collectionId}/indexes`, + JSON.stringify(index), { + headers: resources.apiHeaders + }); + + console.log(indexRes.status); + + check(indexRes, { + "status is 202": (r) => r.status === 202, + }); + + console.log(`----- Inserting ${millionRecords} documents in ${numBatches} batches of ${batchSize} -----`); + + const timeStart = new Date(); + + const requests = []; + for (let i = 0; i < numBatches; i++) { + const docs = Array.from({ length: batchSize }, () => ({ + $id: unique(), + name: "bulk_document", + age: Math.floor(Math.random() * 100), + email: `${unique()}@test.com`, + height: Math.random() * 100 + })); + requests.push({ + method: "POST", + url: `http://localhost/v1/databases/${databaseId}/collections/${collectionId}/documents`, + body: JSON.stringify({ documents: docs }), + params: { + headers: resources.apiHeaders, + timeout: '300s' + } + }); + } + + const responses = http.batch(requests); + responses.forEach((res, index) => { + if (res.status !== 201) { + throw new Error(`Batch ${index + 1} failed with status ${res.status}`); + } + }); + + const timeEnd = new Date(); + const timeTaken = timeEnd - timeStart; + console.log(`Created 1 million documents in ${timeTaken} milliseconds`); + + return { + databaseId, + collectionId, + apiHeaders: resources.apiHeaders, + resources + }; +} + +export default function (data) { + const docs = Array.from({ length: 10000 }, () => ({ + $id: unique(), + name: "performance_document", + age: Math.floor(Math.random() * 100), + email: `${unique()}@test.com`, + height: Math.random() * 100 + })); + + const payload = JSON.stringify({ documents: docs }); + const res = http.post( + `http://localhost/v1/databases/${data.databaseId}/collections/${data.collectionId}/documents`, + payload, + { + headers: data.apiHeaders, + timeout: '300s' + } + ); + + check(res, { + "status is 201": (r) => r.status === 201 + }); + + sleep(1); +} + +export function teardown(data) { + cleanup(data.resources); +} + +export const options = { + scenarios: { + large_document_creation: { + executor: 'per-vu-iterations', + vus: 1, + iterations: 20, + exec: 'default' + } + } +}; \ No newline at end of file