Merge branch 'feat-bulk-operations' of github.com:appwrite/appwrite into feat-bulk-operations

# Conflicts:
#	app/controllers/api/databases.php
#	app/controllers/shared/api.php
#	composer.json
#	composer.lock
This commit is contained in:
Jake Barnby
2025-03-05 12:47:47 +13:00
10 changed files with 703 additions and 71 deletions
+3 -3
View File
@@ -3192,7 +3192,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents')
}
if (!empty($documents) && !empty($documentId)) {
throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Param "documentId" is disallowed when creating multiple documents, use $id inside the documents');
throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Param "documentId" is disallowed when creating multiple documents, set $id in each document');
}
if (!empty($data)) {
@@ -3353,7 +3353,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents')
$document['$id'] = $documentId == 'unique()' ? ID::unique() : $documentId;
} else {
if (empty($document['$id'])) {
throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, '$id is required inside documents when creating bulk documents');
throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, '$id must be set in each document when creating bulk documents');
}
$document['$id'] = $document['$id'] == 'unique()' ? ID::unique() : $document['$id'];
@@ -3445,7 +3445,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents')
->dynamic($documents[0], Response::MODEL_DOCUMENT);
}
$queueForUsage
$queueForStatsUsage
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection
});
+4 -4
View File
@@ -513,10 +513,10 @@ App::init()
$queueForRealtime = new Realtime();
$dbForProject
->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForUsage))
->on(Database::EVENT_DOCUMENTS_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForUsage))
->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForUsage))
->on(Database::EVENT_DOCUMENTS_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForUsage))
->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage))
->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage))
->on(Database::EVENT_DOCUMENTS_CREATE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage))
->on(Database::EVENT_DOCUMENTS_DELETE, 'calculate-usage', fn ($event, $document) => $usageDatabaseListener($event, $document, $queueForStatsUsage))
->on(Database::EVENT_DOCUMENT_CREATE, 'create-trigger-events', fn ($event, $document) => $eventDatabaseListener(
$project,
$document,
+2 -12
View File
@@ -47,7 +47,7 @@
"appwrite/php-clamav": "2.0.*",
"utopia-php/abuse": "0.51.*",
"utopia-php/analytics": "0.10.*",
"utopia-php/audit": "0.54.0",
"utopia-php/audit": "0.54.*",
"utopia-php/cache": "0.11.*",
"utopia-php/cli": "0.15.*",
"utopia-php/config": "0.2.*",
@@ -97,16 +97,6 @@
"config": {
"platform": {
"php": "8.3"
},
"allow-plugins": {
"php-http/discovery": false,
"tbachert/spi": false
}
},
"repositories": [
{
"type": "vcs",
"url": "https://github.com/appwrite/sdk-generator"
}
]
}
}
Generated
+4 -19
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "c2209d50d5e1a4fc625c914f58160521",
"content-hash": "cfc5d9267ecdd5f5159addd4ebfffd2d",
"packages": [
{
"name": "adhocore/jwt",
@@ -5071,22 +5071,7 @@
"Appwrite\\Spec\\": "src/Spec"
}
},
"autoload-dev": {
"psr-4": {
"Tests\\": "tests"
}
},
"scripts": {
"test": [
"vendor/bin/phpunit"
],
"lint": [
"vendor/bin/phpcs"
],
"format": [
"vendor/bin/phpcbf"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
@@ -5098,8 +5083,8 @@
],
"description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms",
"support": {
"source": "https://github.com/appwrite/sdk-generator/tree/0.40.1",
"issues": "https://github.com/appwrite/sdk-generator/issues"
"issues": "https://github.com/appwrite/sdk-generator/issues",
"source": "https://github.com/appwrite/sdk-generator/tree/0.40.1"
},
"time": "2025-02-26T07:07:10+00:00"
},
@@ -1 +1 @@
Update all documents that match your queries, If none are submitted then all documents are updated. Using the patch method you can pass only specific fields that will get updated.
Update all documents that match your queries, if no queries are submitted then all documents are updated. You can pass only specific fields to be updated.
-32
View File
@@ -1,32 +0,0 @@
import { check } from "k6";
import http from "k6/http";
const amount = 100_000;
const databaseId = "674918b20017411b94b2";
const collectionId = "674918b4002b46c47d5d";
const documents = Array(amount).fill({
$id: "unique()",
name: "asd",
});
export default function () {
const payload = JSON.stringify({
documents,
});
const res = http.post(`http://localhost/v1/databases/${databaseId}/collections/${collectionId}/documents`,
payload,
{
headers: {
"X-Appwrite-Key": "standard_fa89c4834660f39e95ca2c2996fe7dd4ff498725e37c09323234c009570c1719f1c10610bf3541cf9ead120c107e41397a4eae1c787c83bdf577857bbc5963341641c77f582cc41e11a0d50eb4c2e4b1fda74418a8b9a253d6e63008e33560ba35310b9dc2fed5f09ca599e646f744cc6308b8ccd27ff04f9e498ec5a5f2c3db",
"X-Appwrite-Project": "674818bc0017934d58dd",
"Content-Type": "application/json",
},
}
);
check(res, {
"status is 200": (r) => r.status === 201,
});
}
@@ -0,0 +1,127 @@
import { check, sleep } from "k6";
import http from "k6/http";
import { provisionProject, provisionDatabase, cleanup, unique } from "./utils.js";
const millionRecords = 1_000_000;
const batchSize = 10_000;
const numBatches = millionRecords / batchSize;
export function setup() {
const resources = provisionProject({
endpoint: 'http://localhost/v1',
email: 'test@test.com',
password: 'password123',
name: 'Test User',
projectName: 'Large Document Creation Test'
});
const { databaseId, collectionId } = provisionDatabase({
endpoint: 'http://localhost/v1',
apiHeaders: resources.apiHeaders
});
// Wait to ensure that provisioning is complete
sleep(5);
// Create an index for the collection
const index = {
key: "name",
type: "fulltext",
orders: ["ASC"],
attributes: ["name", "email"]
};
const indexRes = http.post(`http://localhost/v1/databases/${databaseId}/collections/${collectionId}/indexes`,
JSON.stringify(index), {
headers: resources.apiHeaders
});
console.log(indexRes.status);
check(indexRes, {
"status is 202": (r) => r.status === 202,
});
console.log(`----- Inserting ${millionRecords} documents in ${numBatches} batches of ${batchSize} -----`);
const timeStart = new Date();
const requests = [];
for (let i = 0; i < numBatches; i++) {
const docs = Array.from({ length: batchSize }, () => ({
$id: unique(),
name: "bulk_document",
age: Math.floor(Math.random() * 100),
email: `${unique()}@test.com`,
height: Math.random() * 100
}));
requests.push({
method: "POST",
url: `http://localhost/v1/databases/${databaseId}/collections/${collectionId}/documents`,
body: JSON.stringify({ documents: docs }),
params: {
headers: resources.apiHeaders,
timeout: '300s'
}
});
}
const responses = http.batch(requests);
responses.forEach((res, index) => {
if (res.status !== 201) {
throw new Error(`Batch ${index + 1} failed with status ${res.status}`);
}
});
const timeEnd = new Date();
const timeTaken = timeEnd - timeStart;
console.log(`Created 1 million documents in ${timeTaken} milliseconds`);
return {
databaseId,
collectionId,
apiHeaders: resources.apiHeaders,
resources
};
}
export default function (data) {
const docs = Array.from({ length: 10000 }, () => ({
$id: unique(),
name: "performance_document",
age: Math.floor(Math.random() * 100),
email: `${unique()}@test.com`,
height: Math.random() * 100
}));
const payload = JSON.stringify({ documents: docs });
const res = http.post(
`http://localhost/v1/databases/${data.databaseId}/collections/${data.collectionId}/documents`,
payload,
{
headers: data.apiHeaders,
timeout: '300s'
}
);
check(res, {
"status is 201": (r) => r.status === 201
});
sleep(1);
}
export function teardown(data) {
cleanup(data.resources);
}
export const options = {
scenarios: {
large_document_creation: {
executor: 'per-vu-iterations',
vus: 1,
iterations: 20,
exec: 'default'
}
}
};
@@ -0,0 +1,82 @@
import { check, sleep } from "k6";
import http from "k6/http";
import { provisionProject, provisionDatabase, cleanup, unique } from "./utils.js";
const amount = 10_000;
export function setup() {
const resources = provisionProject({
endpoint: 'http://localhost/v1',
email: 'test@test.com',
password: 'password123',
name: 'Test User',
projectName: 'Bulk Operations Test'
});
const { databaseId, collectionId } = provisionDatabase({
endpoint: 'http://localhost/v1',
apiHeaders: resources.apiHeaders
});
sleep(3); // Await Attributes to be provisioned
console.log(`----- Amount of documents: ${amount} -----`);
return {
databaseId,
collectionId,
apiHeaders: resources.apiHeaders,
resources
};
}
export function teardown(data) {
cleanup(data.resources);
}
let documents = Array(amount).fill({
$id: "unique()",
name: "asd",
});
documents = documents.map((document) => {
return {
...document,
age: Math.floor(Math.random() * 100),
email: `${unique()}@test.com`,
height: Math.random() * 100,
};
});
export default function (data) {
const payload = JSON.stringify({
documents,
});
const res = http.post(
`http://localhost/v1/databases/${data.databaseId}/collections/${data.collectionId}/documents`,
payload,
{
headers: data.apiHeaders
}
);
check(res, {
"status is 201": (r) => r.status === 201,
});
return {
resources: data.resources
};
}
export const options = {
scenarios: {
bulk_create: {
executor: 'per-vu-iterations',
vus: 1,
iterations: 20,
exec: 'default'
}
}
};
@@ -0,0 +1,144 @@
import { check, sleep } from "k6";
import http from "k6/http";
import { Trend } from "k6/metrics";
import { provisionProject, provisionDatabase, cleanup, unique } from "./utils.js";
// Custom Trend metric for light response time tracking
export const lightResponseTime = new Trend("light_response_time", true);
const BULK_AMOUNT = 100_000; // Heavy operation amount
const LIGHT_AMOUNT = 10; // Light operation amount
export function setup() {
// Set up two separate projects - one for bulk operations (noisy neighbor) and one for light operations
const heavyResources = provisionProject({
endpoint: 'http://localhost/v1',
email: 'heavy@test.com',
password: 'password123',
name: 'Heavy User',
projectName: 'Noisy Neighbor - Heavy'
});
const lightResources = provisionProject({
endpoint: 'http://localhost/v1',
email: 'light@test.com',
password: 'password123',
name: 'Light User',
projectName: 'Noisy Neighbor - Light'
});
// Set up databases for both projects
const heavy = provisionDatabase({
endpoint: 'http://localhost/v1',
apiHeaders: heavyResources.apiHeaders
});
const light = provisionDatabase({
endpoint: 'http://localhost/v1',
apiHeaders: lightResources.apiHeaders
});
sleep(3); // Await Attributes to be provisioned
console.log(`----- Heavy operations: ${BULK_AMOUNT} documents | Light operations: ${LIGHT_AMOUNT} document -----`);
return {
heavy: {
databaseId: heavy.databaseId,
collectionId: heavy.collectionId,
apiHeaders: heavyResources.apiHeaders,
resources: heavyResources
},
light: {
databaseId: light.databaseId,
collectionId: light.collectionId,
apiHeaders: lightResources.apiHeaders,
resources: lightResources
}
};
}
export function teardown(data) {
cleanup(data.heavy.resources);
cleanup(data.light.resources);
}
// Create document payloads
function createDocuments(amount) {
let documents = Array(amount).fill({
$id: "unique()",
name: "test",
});
return documents.map((document) => ({
...document,
age: Math.floor(Math.random() * 100),
email: `${unique()}@test.com`,
height: Math.random() * 100,
}));
}
// Heavy operation function
export function heavy(data) {
const documents = createDocuments(BULK_AMOUNT);
const payload = JSON.stringify({ documents });
const res = http.post(
`http://localhost/v1/databases/${data.heavy.databaseId}/collections/${data.heavy.collectionId}/documents`,
payload,
{
headers: data.heavy.apiHeaders
}
);
check(res, {
"heavy operation status is 201": (r) => r.status === 201,
});
}
// Light operation function
export function light(data) {
const documents = createDocuments(LIGHT_AMOUNT);
const payload = JSON.stringify({ documents });
const startTime = new Date();
const res = http.post(
`http://localhost/v1/databases/${data.light.databaseId}/collections/${data.light.collectionId}/documents`,
payload,
{
headers: data.light.apiHeaders
}
);
const duration = new Date() - startTime;
// Record the light operation response time using the custom Trend metric
lightResponseTime.add(duration);
check(res, {
"light operation status is 201": (r) => r.status === 201,
});
}
export const options = {
scenarios: {
// Heavy bulk operations running continuously
heavy_load: {
executor: 'constant-vus',
vus: 5,
duration: '30s',
exec: 'heavy'
},
// Light operations to measure impact
light_operations: {
executor: 'constant-arrival-rate',
rate: 5,
timeUnit: '1s',
duration: '30s',
preAllocatedVUs: 10,
exec: 'light'
}
},
thresholds: {
http_req_duration: ['p(95)<2000'], // 95% of requests should complete within 2s
}
};
+336
View File
@@ -0,0 +1,336 @@
import http from 'k6/http';
import { check } from 'k6';
/**
* @typedef {Object} AuthHeaders
* @property {string} 'Content-Type' - Content type header
* @property {string} 'Cookie' - Session cookie
* @property {string} 'X-Appwrite-Project' - Project ID header
*/
/**
* @typedef {Object} ApiHeaders
* @property {string} 'Content-Type' - Content type header
* @property {string} 'X-Appwrite-Project' - Project ID header
* @property {string} 'X-Appwrite-Key' - API key header
*/
/**
* @typedef {Object} ProvisionedResources
* @property {string} userId - The ID of the created user
* @property {string} teamId - The ID of the created team
* @property {string} projectId - The ID of the created project
* @property {string} cookies - Session cookies for authentication
* @property {AuthHeaders} headers - Headers for cookie-based authentication
* @property {string} apiKey - The API key secret
* @property {ApiHeaders} apiHeaders - Headers for API key authentication
*/
function assert(response, checkName, condition) {
const result = check(response, {
[checkName]: condition
});
if (!result) {
console.error(`Assertion failed: ${checkName}`);
console.error(`Response status: ${response.status}`);
console.error(`Response body: ${response.body}`);
throw new Error(`Assertion failed: ${checkName}`);
}
}
/**
* Provisions an Appwrite project setup including:
* - Account creation
* - Session creation
* - Team creation
* - Project creation
* - API Key creation
*
* @param {Object} config Configuration object
* @param {string} config.endpoint Base endpoint URL (e.g., 'http://localhost:80/v1')
* @param {string} config.email Email for account creation
* @param {string} config.password Password for account creation
* @param {string} config.name Name for account creation
* @param {string} config.projectName Name for the project
* @returns {ProvisionedResources} Object containing all created resource IDs and session information
*/
export function provisionProject(config) {
const {
endpoint,
email,
password,
name,
projectName,
} = config;
// Step 1: Create Account
const accountResponse = http.post(`${endpoint}/account`, JSON.stringify({
userId: 'unique()',
email,
password,
name
}), {
headers: {
'Content-Type': 'application/json',
}
});
assert(accountResponse, 'account created successfully', (r) => r.status === 201 || r.status === 409);
const userId = accountResponse.json('$id');
// Step 2: Create Session
const sessionResponse = http.post(`${endpoint}/account/sessions/email`, JSON.stringify({
email,
password
}), {
headers: {
'Content-Type': 'application/json',
}
});
assert(sessionResponse, 'session created successfully', (r) => r.status === 201);
// Keep manual control of the cookies to allow for simultaneous requests
const jar = http.cookieJar();
jar.clear(`${endpoint}`);
// Extract cookies for subsequent requests
const cookies = sessionResponse.headers['Set-Cookie'];
// Common headers for authenticated requests
const authHeaders = {
'Content-Type': 'application/json',
'Cookie': cookies
};
// Step 3: Create Team
const teamResponse = http.post(`${endpoint}/teams`, JSON.stringify({
teamId: 'unique()',
name: `${projectName} Team`
}), {
headers: authHeaders
});
assert(teamResponse, 'team created successfully', (r) => r.status === 201);
const teamId = teamResponse.json('$id');
// Step 4: Create Project
const projectResponse = http.post(`${endpoint}/projects`, JSON.stringify({
projectId: 'unique()',
name: projectName,
teamId: teamId
}), {
headers: authHeaders
});
assert(projectResponse, 'project created successfully', (r) => r.status === 201);
const projectId = projectResponse.json('$id');
// Step 5: Create API Key
const apiKeyResponse = http.post(`${endpoint}/projects/${projectId}/keys`, JSON.stringify({
name: 'Test API Key',
scopes: SCOPES, // All permissions
}), {
headers: authHeaders
});
assert(apiKeyResponse, 'api key created successfully', (r) => r.status === 201);
const apiKey = apiKeyResponse.json('secret');
// Create a new headers object for API key authentication
const apiHeaders = {
'Content-Type': 'application/json',
'X-Appwrite-Project': projectId,
'X-Appwrite-Key': apiKey
};
// Return all created resources and session info
return {
endpoint,
userId,
teamId,
projectId,
cookies,
headers: authHeaders,
apiKey,
apiHeaders
};
}
/**
* Example usage:
*
* const config = {
* endpoint: 'http://localhost:80/v1',
* email: 'test@example.com',
* password: 'complex-password',
* name: 'Test User',
* projectName: 'Test Project'
* };
*
* const resources = provisionProject(config);
*/
const SCOPES = [
"sessions.write",
"users.read",
"users.write",
"teams.read",
"teams.write",
"databases.read",
"databases.write",
"collections.read",
"collections.write",
"attributes.read",
"attributes.write",
"indexes.read",
"indexes.write",
"documents.read",
"documents.write",
"files.read",
"files.write",
"buckets.read",
"buckets.write",
"functions.read",
"functions.write",
"execution.read",
"execution.write",
"targets.read",
"targets.write",
"providers.read",
"providers.write",
"messages.read",
"messages.write",
"topics.read",
"topics.write",
"subscribers.read",
"subscribers.write",
"locale.read",
"avatars.read",
"health.read",
"migrations.read",
"migrations.write"
]
export function provisionDatabase(config) {
const {
endpoint,
apiHeaders
} = config;
// Create database
const databaseResponse = http.post(
`${endpoint}/databases`,
JSON.stringify({
databaseId: 'unique()',
name: 'Bulk Test DB'
}),
{ headers: apiHeaders }
);
assert(databaseResponse, 'database created successfully', (r) => r.status === 201);
const databaseId = databaseResponse.json('$id');
// Create collection
const collectionResponse = http.post(
`${endpoint}/databases/${databaseId}/collections`,
JSON.stringify({
collectionId: 'unique()',
name: 'Bulk Test Collection',
permissions: ['read("any")', 'write("any")'],
documentSecurity: false
}),
{ headers: apiHeaders }
);
assert(collectionResponse, 'collection created successfully', (r) => r.status === 201);
const collectionId = collectionResponse.json('$id');
// Create name attribute
const nameAttributeResponse = http.post(
`${endpoint}/databases/${databaseId}/collections/${collectionId}/attributes/string`,
JSON.stringify({
key: 'name',
size: 100,
required: false,
default: null,
array: false,
encrypt: false
}),
{ headers: apiHeaders }
);
assert(nameAttributeResponse, 'name attribute created successfully', (r) => r.status === 202);
// Create age attribute
const ageAttributeResponse = http.post(
`${endpoint}/databases/${databaseId}/collections/${collectionId}/attributes/integer`,
JSON.stringify({
key: 'age',
required: false,
}),
{ headers: apiHeaders }
);
assert(ageAttributeResponse, 'age attribute created successfully', (r) => r.status === 202);
// Create email attribute
const emailAttributeResponse = http.post(
`${endpoint}/databases/${databaseId}/collections/${collectionId}/attributes/email`,
JSON.stringify({
key: 'email',
required: false,
}),
{ headers: apiHeaders }
);
assert(emailAttributeResponse, 'email attribute created successfully', (r) => r.status === 202);
// Create height attribute
const heightAttributeResponse = http.post(
`${endpoint}/databases/${databaseId}/collections/${collectionId}/attributes/float`,
JSON.stringify({
key: 'height',
required: false,
}),
{ headers: apiHeaders }
);
assert(heightAttributeResponse, 'height attribute created successfully', (r) => r.status === 202);
return {
databaseId,
collectionId
};
}
export function cleanup(config) {
const {
endpoint,
teamId,
headers
} = config;
// Delete Organization
const organizationResponse = http.del(
`${endpoint}/teams/${teamId}`,
null,
{
headers
}
);
assert(organizationResponse, 'organization deleted successfully', (r) => r.status === 204);
}
export function unique() {
const timestamp = Date.now().toString(36);
const randomPart = Math.random().toString(36).substring(2, 15);
return `${timestamp}-${randomPart}`;
}