diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 06755e1d24..a3b3b96ce1 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3235,7 +3235,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('user') ->inject('queueForEvents') ->inject('queueForStatsUsage') - ->action(function (string $databaseId, ?string $documentId, string $collectionId, string|array|null $data, ?array $documents, ?array $permissions, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, StatsUsage $queueForStatsUsage) { + ->action(function (string $databaseId, ?string $documentId, string $collectionId, string|array|null $data, ?array $permissions, ?array $documents, Response $response, Database $dbForProject, Document $user, Event $queueForEvents, StatsUsage $queueForStatsUsage) { $data = \is_string($data) ? \json_decode($data, true) : $data; @@ -4359,6 +4359,88 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents') ]), Response::MODEL_DOCUMENT_LIST); }); +App::put('/v1/databases/:databaseId/collections/:collectionId/documents') + ->desc('Create or update documents') + ->groups(['api', 'database']) + ->label('scope', 'documents.write') + ->label('resourceType', RESOURCE_TYPE_DATABASES) + ->label('audits.event', 'documents.upsert') + ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}') + ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}') + ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2) + ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT) + ->label('sdk', new Method( + namespace: 'databases', + group: 'documents', + name: 'upsertDocuments', + description: '/docs/references/databases/upsert-documents.md', + auth: [AuthType::KEY], + responses: [ + new SDKResponse( + code: Response::STATUS_CODE_OK, + model: Response::MODEL_DOCUMENT_LIST, + ) + ], + contentType: ContentType::JSON + )) + ->param('databaseId', '', new UID(), 'Database ID.') + ->param('collectionId', '', new UID(), 'Collection ID.') + ->param('documents', [], fn (array $plan) => new ArrayList(new JSON(), $plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH), 'Array of document data as JSON objects. May contain partial documents.', true, ['plan']) + ->inject('response') + ->inject('dbForProject') + ->inject('queueForStatsUsage') + ->inject('plan') + ->action(function (string $databaseId, string $collectionId, array $documents, Response $response, Database $dbForProject, StatsUsage $queueForStatsUsage, array $plan) { + $database = $dbForProject->getDocument('databases', $databaseId); + if ($database->isEmpty()) { + throw new Exception(Exception::DATABASE_NOT_FOUND); + } + + $collection = $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); + if ($collection->isEmpty()) { + throw new Exception(Exception::COLLECTION_NOT_FOUND); + } + + $hasRelationships = \array_filter( + $collection->getAttribute('attributes', []), + fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP + ); + + if ($hasRelationships) { + throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Bulk upsert is not supported for collections with relationship attributes'); + } + + foreach ($documents as $key => $document) { + $documents[$key] = new Document($document); + } + + $upserted = []; + + $modified = $dbForProject->createOrUpdateDocuments( + 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), + $documents, + onNext: function (Document $document) use ($plan, &$upserted) { + if (\count($upserted) < ($plan['databasesBatchSize'] ?? APP_LIMIT_DATABASE_BATCH)) { + $upserted[] = $document; + } + }, + ); + + foreach ($upserted as $document) { + $document->setAttribute('$databaseId', $database->getId()); + $document->setAttribute('$collectionId', $collection->getId()); + } + + $queueForStatsUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, \max(1, $modified)) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), \max(1, $modified)); + + $response->dynamic(new Document([ + 'total' => $modified, + 'documents' => $upserted + ]), Response::MODEL_DOCUMENT_LIST); + }); + App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:documentId') ->alias('/v1/database/collections/:collectionId/documents/:documentId', ['databaseId' => 'default']) ->desc('Delete document') @@ -4912,7 +4994,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage') $response->dynamic(new Document([ 'range' => $range, - 'documentsTotal' => $usage[$metrics[0]]['total'], - 'documents' => $usage[$metrics[0]]['data'], + 'documentsTotal' => $usage[$metrics[0]]['total'], + 'documents' => $usage[$metrics[0]]['data'], ]), Response::MODEL_USAGE_COLLECTION); }); diff --git a/composer.lock b/composer.lock index 560d90af8b..6e9475dc5c 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "49b9828ae72763d2a398358c28cebdd5", + "content-hash": "3c140ba66f30878720ebe009abc158a7", "packages": [ { "name": "adhocore/jwt", @@ -3499,16 +3499,16 @@ }, { "name": "utopia-php/database", - "version": "0.67.2", + "version": "0.67.4", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "cd55117beab0025fd0d3f945a5a004125600982c" + "reference": "f2d687abe6a94e742b070bfce0356ab33d62cff8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/cd55117beab0025fd0d3f945a5a004125600982c", - "reference": "cd55117beab0025fd0d3f945a5a004125600982c", + "url": "https://api.github.com/repos/utopia-php/database/zipball/f2d687abe6a94e742b070bfce0356ab33d62cff8", + "reference": "f2d687abe6a94e742b070bfce0356ab33d62cff8", "shasum": "" }, "require": { @@ -3549,9 +3549,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.67.2" + "source": "https://github.com/utopia-php/database/tree/0.67.4" }, - "time": "2025-05-06T12:11:58+00:00" + "time": "2025-05-07T14:37:07+00:00" }, { "name": "utopia-php/domains", diff --git a/docs/references/databases/upsert-documents.md b/docs/references/databases/upsert-documents.md new file mode 100644 index 0000000000..f46254bd7b --- /dev/null +++ b/docs/references/databases/upsert-documents.md @@ -0,0 +1 @@ +Create or update Documents. Before using this route, you should create a new collection resource using either a [server integration](https://appwrite.io/docs/server/databases#databasesCreateCollection) API or directly from your database console. diff --git a/tests/e2e/Services/Databases/DatabasesBase.php b/tests/e2e/Services/Databases/DatabasesBase.php index 30403b66d3..b5ddedbf90 100644 --- a/tests/e2e/Services/Databases/DatabasesBase.php +++ b/tests/e2e/Services/Databases/DatabasesBase.php @@ -5054,126 +5054,6 @@ trait DatabasesBase $this->assertEquals(400, $response['headers']['status-code']); } - public function testBulkCreateRelationships(): void - { - $database = $this->client->call(Client::METHOD_POST, '/databases', [ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ], [ - 'databaseId' => ID::unique(), - 'name' => 'Bulk Creates Relationships' - ]); - - $this->assertNotEmpty($database['body']['$id']); - - $databaseId = $database['body']['$id']; - - $collection1 = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]), [ - 'collectionId' => ID::unique(), - 'name' => 'Collection1', - 'documentSecurity' => false, - 'permissions' => [ - Permission::create(Role::any()), - Permission::read(Role::any()), - Permission::delete(Role::any()), - ], - ]); - - $collection2 = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]), [ - 'collectionId' => ID::unique(), - 'name' => 'Collection2', - 'documentSecurity' => false, - 'permissions' => [ - Permission::create(Role::any()), - Permission::read(Role::any()), - Permission::delete(Role::any()), - ], - ]); - - $collection1 = $collection1['body']['$id']; - $collection2 = $collection2['body']['$id']; - - $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collection1 . '/attributes/string', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]), [ - 'key' => 'name', - 'size' => 256, - 'required' => true, - ]); - - $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collection2 . '/attributes/string', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]), [ - 'key' => 'name', - 'size' => 256, - 'required' => true, - ]); - - $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collection1 . '/attributes/relationship', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'x-appwrite-key' => $this->getProject()['apiKey'] - ]), [ - 'relatedCollectionId' => $collection2, - 'type' => Database::RELATION_ONE_TO_MANY, - 'key' => 'collection2', - 'onDelete' => Database::RELATION_MUTATE_RESTRICT, - ]); - - sleep(2); - - $response = $this->client->call(Client::METHOD_POST, "/databases/{$databaseId}/collections/{$collection1}/documents", array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), [ - 'documents' => [ - [ - '$id' => ID::unique(), - 'name' => 'Document 1', - 'collection2' => [ - [ - '$id' => ID::unique(), - 'name' => 'Document 2', - ], - [ - '$id' => ID::unique(), - 'name' => 'Document 3', - ], - ], - ], - [ - '$id' => ID::unique(), - 'name' => 'Document 2', - 'collection2' => [ - [ - '$id' => ID::unique(), - 'name' => 'Document 4', - ], - [ - '$id' => ID::unique(), - 'name' => 'Document 5', - ], - ], - ], - ], - ]); - - $this->assertEquals(400, $response['headers']['status-code']); - } - public function testBulkUpdates(): void { // Create database @@ -5405,6 +5285,156 @@ trait DatabasesBase $this->assertEquals(10, $documents['body']['total']); } + public function testBulkUpserts(): void + { + // Create database + $database = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Bulk Upserts' + ]); + + $this->assertNotEmpty($database['body']['$id']); + + $databaseId = $database['body']['$id']; + + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => ID::unique(), + 'name' => 'Bulk Upserts', + 'documentSecurity' => true, + 'permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::delete(Role::any()), + Permission::update(Role::any()), + ], + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + + $data = [ + '$id' => $collection['body']['$id'], + 'databaseId' => $collection['body']['databaseId'] + ]; + + // Await attribute + $numberAttribute = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $data['$id'] . '/attributes/integer', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'key' => 'number', + 'required' => true, + ]); + + $this->assertEquals(202, $numberAttribute['headers']['status-code']); + + // Wait for database worker to create attributes + sleep(2); + + // Create documents + $createBulkDocuments = function ($amount = 10) use ($data) { + $documents = []; + + for ($x = 1; $x <= $amount; $x++) { + $documents[] = [ + '$id' => "$x", + 'number' => $x, + ]; + } + + $response = $this->client->call(Client::METHOD_POST, '/databases/' . $data['databaseId'] . '/collections/' . $data['$id'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documents' => $documents, + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + return $documents; + }; + + $documents = $createBulkDocuments(); + + // Update a document + $documents[\array_key_last($documents)]['number'] = 1000; + + // Add a new document + $documents[] = ['number' => 11]; + + // TEST: Upsert all documents + $response = $this->client->call(Client::METHOD_PUT, '/databases/' . $data['databaseId'] . '/collections/' . $data['$id'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documents' => $documents, + ]); + + // Unchanged docs are skipped. 2 documents should be returned, 1 updated and 1 inserted. + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertCount(2, $response['body']['documents']); + $this->assertEquals(1000, $response['body']['documents'][0]['number']); + $this->assertEquals(11, $response['body']['documents'][1]['number']); + + $documents = $this->client->call(Client::METHOD_GET, '/databases/' . $data['databaseId'] . '/collections/' . $data['$id'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ])); + + $this->assertEquals(200, $documents['headers']['status-code']); + $this->assertEquals(11, $documents['body']['total']); + + foreach ($documents['body']['documents'] as $index => $document) { + $this->assertEquals($collection['body']['$id'], $document['$collectionId']); + $this->assertEquals($data['databaseId'], $document['$databaseId']); + switch ($index) { + case 9: + $this->assertEquals($document['number'], 1000); + break; + default: + $this->assertEquals($document['number'], $index + 1); + } + } + + // TEST: Upsert permissions: + $response = $this->client->call(Client::METHOD_PUT, '/databases/' . $data['databaseId'] . '/collections/' . $data['$id'] . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documents' => [ + [ + '$id' => '1', + 'number' => 1000, + ], + [ + '$id' => '10', + '$permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::update(Role::user($this->getUser()['$id'])), + Permission::delete(Role::user($this->getUser()['$id'])), + ], + 'number' => 10, + ], + ], + ]); + + $this->assertEquals(1000, $response['body']['documents'][0]['number']); + $this->assertEquals([], $response['body']['documents'][0]['$permissions']); + $this->assertEquals([ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::update(Role::user($this->getUser()['$id'])), + Permission::delete(Role::user($this->getUser()['$id'])), + ], $response['body']['documents'][1]['$permissions']); + } + public function testBulkDeletes(): void { // Create database