From b36472f0daef3f86d9e44610092c8b5dc8769760 Mon Sep 17 00:00:00 2001 From: Prem Palanisamy Date: Thu, 26 Mar 2026 11:24:14 +0000 Subject: [PATCH] add E2E tests for vectorsdb and documentsdb JSON import/export --- .../Services/Migrations/MigrationsBase.php | 318 ++++++++++++++++++ .../resources/json/documentsdb-documents.json | 162 +++++++++ tests/resources/json/vectorsdb-documents.json | 262 +++++++++++++++ 3 files changed, 742 insertions(+) create mode 100644 tests/resources/json/documentsdb-documents.json create mode 100644 tests/resources/json/vectorsdb-documents.json diff --git a/tests/e2e/Services/Migrations/MigrationsBase.php b/tests/e2e/Services/Migrations/MigrationsBase.php index 84b02643f2..2dd9b9f69a 100644 --- a/tests/e2e/Services/Migrations/MigrationsBase.php +++ b/tests/e2e/Services/Migrations/MigrationsBase.php @@ -1780,4 +1780,322 @@ trait MigrationsBase 'x-appwrite-key' => $this->getProject()['apiKey'] ]); } + + public function testCreateVectorsDBJSONExport(): void + { + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]; + + // Create vectorsdb database + $database = $this->client->call(Client::METHOD_POST, '/vectorsdb', $headers, [ + 'databaseId' => ID::unique(), + 'name' => 'VectorsDB Export Test' + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection with dimension 16 + $collection = $this->client->call(Client::METHOD_POST, '/vectorsdb/' . $databaseId . '/collections', $headers, [ + 'collectionId' => ID::unique(), + 'name' => 'VecExportCol', + 'dimension' => 16, + ]); + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Seed 5 documents + for ($i = 1; $i <= 5; $i++) { + $embeddings = array_map(fn () => round((mt_rand() / mt_getrandmax()) * 2 - 1, 6), range(1, 16)); + $doc = $this->client->call(Client::METHOD_POST, '/vectorsdb/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers, [ + 'documentId' => ID::unique(), + 'data' => [ + 'embeddings' => $embeddings, + 'metadata' => ['title' => 'Doc ' . $i, 'score' => round($i * 0.2, 1)] + ] + ]); + $this->assertEquals(201, $doc['headers']['status-code'], 'Failed to create vector document ' . $i); + } + + // Trigger JSON export + $migration = $this->client->call(Client::METHOD_POST, '/migrations/json/exports', $headers, [ + 'resourceId' => $databaseId . ':' . $collectionId, + 'filename' => 'vectorsdb-export-test', + 'columns' => [], + 'queries' => [], + 'notify' => true, + ]); + $this->assertEquals(202, $migration['headers']['status-code']); + $migrationId = $migration['body']['$id']; + + // Poll until completed + $this->assertEventually(function () use ($migrationId, $headers) { + $migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, $headers); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('completed', $migration['body']['status']); + $this->assertEquals('Appwrite', $migration['body']['source']); + $this->assertEquals('JSON', $migration['body']['destination']); + }, 30_000, 500); + + // Verify email notification + $lastEmail = $this->getLastEmail(); + $this->assertNotEmpty($lastEmail); + $this->assertEquals('Your JSON export is ready', $lastEmail['subject']); + + // Download and verify JSON content + $body = $lastEmail['html']; + preg_match('/href="([^"]*)"/', $body, $matches); + $this->assertNotEmpty($matches[1], 'Download link should be present in email'); + + $downloadUrl = html_entity_decode($matches[1]); + $parsedUrl = parse_url($downloadUrl); + parse_str($parsedUrl['query'] ?? '', $queryParams); + + $jsonFile = $this->client->call(Client::METHOD_GET, $parsedUrl['path'], [ + 'x-appwrite-project' => $this->getProject()['$id'], + 'origin' => 'http://localhost', + ], $queryParams); + + $this->assertNotEmpty($jsonFile['body']); + $jsonData = $jsonFile['body']; + $this->assertNotEmpty($jsonData, 'JSON export should not be empty'); + + // Verify content has embeddings + $decoded = json_decode($jsonData, true); + $this->assertNotNull($decoded); + $this->assertGreaterThanOrEqual(5, count($decoded)); + $this->assertArrayHasKey('embeddings', $decoded[0]); + $this->assertCount(16, $decoded[0]['embeddings'], 'Embeddings should have 16 dimensions'); + $this->assertArrayHasKey('metadata', $decoded[0]); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/vectorsdb/' . $databaseId, $headers); + } + + public function testCreateVectorsDBJSONImport(): void + { + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]; + + // Create vectorsdb database + $database = $this->client->call(Client::METHOD_POST, '/vectorsdb', $headers, [ + 'databaseId' => ID::unique(), + 'name' => 'VectorsDB Import Test' + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection with dimension 16 + $collection = $this->client->call(Client::METHOD_POST, '/vectorsdb/' . $databaseId . '/collections', $headers, [ + 'collectionId' => ID::unique(), + 'name' => 'VecImportCol', + 'dimension' => 16, + ]); + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Create bucket and upload test file + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', $headers, [ + 'bucketId' => ID::unique(), + 'name' => 'VectorsDB Import Bucket', + 'maximumFileSize' => 2000000, + 'allowedFileExtensions' => ['json'], + ]); + $this->assertEquals(201, $bucket['headers']['status-code']); + $bucketId = $bucket['body']['$id']; + + $file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'fileId' => ID::unique(), + 'file' => new \CURLFile(realpath(__DIR__ . '/../../../resources/json/vectorsdb-documents.json'), 'application/json', 'vectorsdb-documents.json'), + ]); + $this->assertEquals(201, $file['headers']['status-code']); + $fileId = $file['body']['$id']; + + // Trigger import + $migration = $this->performJsonMigration([ + 'fileId' => $fileId, + 'bucketId' => $bucketId, + 'resourceId' => $databaseId . ':' . $collectionId, + ]); + $this->assertEquals(202, $migration['headers']['status-code']); + + // Poll until completed + $this->assertEventually(function () use ($migration, $headers) { + $migrationId = $migration['body']['$id']; + $result = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, $headers); + + $this->assertEquals(200, $result['headers']['status-code']); + $this->assertEquals('finished', $result['body']['stage']); + $this->assertEquals('completed', $result['body']['status']); + $this->assertEquals('JSON', $result['body']['source']); + $this->assertEquals('Appwrite', $result['body']['destination']); + }, 30_000, 500); + + // Verify documents were imported + $docs = $this->client->call(Client::METHOD_GET, '/vectorsdb/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers); + $this->assertEquals(200, $docs['headers']['status-code']); + $this->assertEquals(10, $docs['body']['total'], 'Should have imported 10 vectorsdb documents'); + + // Verify first document structure + $firstDoc = $docs['body']['documents'][0]; + $this->assertArrayHasKey('embeddings', $firstDoc); + $this->assertCount(16, $firstDoc['embeddings'], 'Imported embeddings should have 16 dimensions'); + $this->assertArrayHasKey('metadata', $firstDoc); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/vectorsdb/' . $databaseId, $headers); + } + + public function testCreateDocumentsDBJSONExport(): void + { + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]; + + // Create documentsdb database + $database = $this->client->call(Client::METHOD_POST, '/documentsdb', $headers, [ + 'databaseId' => ID::unique(), + 'name' => 'DocumentsDB Export Test' + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection (schemaless — no attributes needed) + $collection = $this->client->call(Client::METHOD_POST, '/documentsdb/' . $databaseId . '/collections', $headers, [ + 'collectionId' => ID::unique(), + 'name' => 'DocExportCol', + ]); + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Seed 5 documents + for ($i = 1; $i <= 5; $i++) { + $doc = $this->client->call(Client::METHOD_POST, '/documentsdb/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers, [ + 'documentId' => ID::unique(), + 'data' => [ + 'name' => 'User ' . $i, + 'email' => 'user' . $i . '@test.com', + 'age' => 20 + $i, + 'address' => ['city' => 'City ' . $i, 'zip' => '1000' . $i] + ] + ]); + $this->assertEquals(201, $doc['headers']['status-code'], 'Failed to create document ' . $i); + } + + // Trigger JSON export + $migration = $this->client->call(Client::METHOD_POST, '/migrations/json/exports', $headers, [ + 'resourceId' => $databaseId . ':' . $collectionId, + 'filename' => 'documentsdb-export-test', + 'columns' => [], + 'queries' => [], + 'notify' => false, + ]); + $this->assertEquals(202, $migration['headers']['status-code']); + $migrationId = $migration['body']['$id']; + + // Poll until completed + $this->assertEventually(function () use ($migrationId, $headers) { + $migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, $headers); + + $this->assertEquals(200, $migration['headers']['status-code']); + $this->assertEquals('finished', $migration['body']['stage']); + $this->assertEquals('completed', $migration['body']['status']); + $this->assertEquals('Appwrite', $migration['body']['source']); + $this->assertEquals('JSON', $migration['body']['destination']); + }, 30_000, 500); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/documentsdb/' . $databaseId, $headers); + } + + public function testCreateDocumentsDBJSONImport(): void + { + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]; + + // Create documentsdb database + $database = $this->client->call(Client::METHOD_POST, '/documentsdb', $headers, [ + 'databaseId' => ID::unique(), + 'name' => 'DocumentsDB Import Test' + ]); + $this->assertEquals(201, $database['headers']['status-code']); + $databaseId = $database['body']['$id']; + + // Create collection (schemaless) + $collection = $this->client->call(Client::METHOD_POST, '/documentsdb/' . $databaseId . '/collections', $headers, [ + 'collectionId' => ID::unique(), + 'name' => 'DocImportCol', + ]); + $this->assertEquals(201, $collection['headers']['status-code']); + $collectionId = $collection['body']['$id']; + + // Create bucket and upload test file + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', $headers, [ + 'bucketId' => ID::unique(), + 'name' => 'DocumentsDB Import Bucket', + 'maximumFileSize' => 2000000, + 'allowedFileExtensions' => ['json'], + ]); + $this->assertEquals(201, $bucket['headers']['status-code']); + $bucketId = $bucket['body']['$id']; + + $file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'fileId' => ID::unique(), + 'file' => new \CURLFile(realpath(__DIR__ . '/../../../resources/json/documentsdb-documents.json'), 'application/json', 'documentsdb-documents.json'), + ]); + $this->assertEquals(201, $file['headers']['status-code']); + $fileId = $file['body']['$id']; + + // Trigger import + $migration = $this->performJsonMigration([ + 'fileId' => $fileId, + 'bucketId' => $bucketId, + 'resourceId' => $databaseId . ':' . $collectionId, + ]); + $this->assertEquals(202, $migration['headers']['status-code']); + + // Poll until completed + $this->assertEventually(function () use ($migration, $headers) { + $migrationId = $migration['body']['$id']; + $result = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, $headers); + + $this->assertEquals(200, $result['headers']['status-code']); + $this->assertEquals('finished', $result['body']['stage']); + $this->assertEquals('completed', $result['body']['status']); + $this->assertEquals('JSON', $result['body']['source']); + $this->assertEquals('Appwrite', $result['body']['destination']); + }, 30_000, 500); + + // Verify documents were imported + $docs = $this->client->call(Client::METHOD_GET, '/documentsdb/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers); + $this->assertEquals(200, $docs['headers']['status-code']); + $this->assertEquals(10, $docs['body']['total'], 'Should have imported 10 documentsdb documents'); + + // Verify first document has nested data + $firstDoc = $docs['body']['documents'][0]; + $this->assertArrayHasKey('name', $firstDoc); + $this->assertArrayHasKey('address', $firstDoc); + $this->assertIsArray($firstDoc['address']); + + // Cleanup + $this->client->call(Client::METHOD_DELETE, '/documentsdb/' . $databaseId, $headers); + } } diff --git a/tests/resources/json/documentsdb-documents.json b/tests/resources/json/documentsdb-documents.json new file mode 100644 index 0000000000..908c123e81 --- /dev/null +++ b/tests/resources/json/documentsdb-documents.json @@ -0,0 +1,162 @@ +[ + { + "$id": "doc01", + "name": "Alice", + "email": "alice@test.com", + "age": 35, + "active": false, + "tags": [ + "analyst", + "engineer" + ], + "address": { + "city": "New York", + "zip": "27436", + "country": "DE" + } + }, + { + "$id": "doc02", + "name": "Bob", + "email": "bob@test.com", + "age": 58, + "active": false, + "tags": [ + "engineer", + "manager" + ], + "address": { + "city": "London", + "zip": "49320", + "country": "JP" + } + }, + { + "$id": "doc03", + "name": "Charlie", + "email": "charlie@test.com", + "age": 23, + "active": true, + "tags": [ + "analyst", + "manager" + ], + "address": { + "city": "Tokyo", + "zip": "74758", + "country": "JP" + } + }, + { + "$id": "doc04", + "name": "Diana", + "email": "diana@test.com", + "age": 60, + "active": true, + "tags": [ + "analyst", + "manager" + ], + "address": { + "city": "Paris", + "zip": "50703", + "country": "JP" + } + }, + { + "$id": "doc05", + "name": "Eve", + "email": "eve@test.com", + "age": 59, + "active": false, + "tags": [ + "engineer", + "designer" + ], + "address": { + "city": "Berlin", + "zip": "98929", + "country": "UK" + } + }, + { + "$id": "doc06", + "name": "Frank", + "email": "frank@test.com", + "age": 23, + "active": true, + "tags": [ + "engineer", + "manager" + ], + "address": { + "city": "Sydney", + "zip": "82907", + "country": "US" + } + }, + { + "$id": "doc07", + "name": "Grace", + "email": "grace@test.com", + "age": 36, + "active": true, + "tags": [ + "analyst", + "developer" + ], + "address": { + "city": "Toronto", + "zip": "68710", + "country": "US" + } + }, + { + "$id": "doc08", + "name": "Hank", + "email": "hank@test.com", + "age": 28, + "active": false, + "tags": [ + "engineer", + "manager" + ], + "address": { + "city": "Mumbai", + "zip": "61026", + "country": "FR" + } + }, + { + "$id": "doc09", + "name": "Iris", + "email": "iris@test.com", + "age": 40, + "active": true, + "tags": [ + "developer", + "designer" + ], + "address": { + "city": "Seoul", + "zip": "48039", + "country": "FR" + } + }, + { + "$id": "doc10", + "name": "Jack", + "email": "jack@test.com", + "age": 45, + "active": false, + "tags": [ + "manager", + "developer" + ], + "address": { + "city": "Dubai", + "zip": "22733", + "country": "JP" + } + } +] \ No newline at end of file diff --git a/tests/resources/json/vectorsdb-documents.json b/tests/resources/json/vectorsdb-documents.json new file mode 100644 index 0000000000..f1bada8a7e --- /dev/null +++ b/tests/resources/json/vectorsdb-documents.json @@ -0,0 +1,262 @@ +[ + { + "$id": "vec01", + "metadata": { + "title": "Vector Document 1", + "score": 0.643, + "category": "art" + }, + "embeddings": [ + -0.516361, + 0.261456, + -0.77323, + 0.909775, + -0.003065, + -0.626888, + 0.704171, + -0.366385, + 0.198681, + 0.854914, + -0.227548, + -0.14278, + -0.548405, + -0.822939, + 0.407267, + 0.570791 + ] + }, + { + "$id": "vec02", + "metadata": { + "title": "Vector Document 2", + "score": 0.322, + "category": "art" + }, + "embeddings": [ + 0.103211, + 0.028378, + 0.806001, + -0.412638, + -0.200009, + 0.448239, + -0.781144, + 0.621957, + -0.868352, + 0.217013, + 0.308217, + -0.851528, + -0.340766, + 0.805996, + -0.065638, + -0.363951 + ] + }, + { + "$id": "vec03", + "metadata": { + "title": "Vector Document 3", + "score": 0.503, + "category": "science" + }, + "embeddings": [ + 0.971901, + -0.475154, + 0.60526, + -0.151618, + -0.938199, + -0.139977, + 0.154581, + -0.868021, + 0.069567, + 0.233545, + -0.164826, + 0.239036, + 0.8875, + 0.488075, + 0.787231, + 0.090293 + ] + }, + { + "$id": "vec04", + "metadata": { + "title": "Vector Document 4", + "score": 0.943, + "category": "science" + }, + "embeddings": [ + 0.372035, + -0.868405, + -0.974987, + -0.836673, + 0.030057, + -0.667698, + 0.095189, + 0.518373, + -0.173732, + 0.966379, + 0.509861, + -0.172607, + 0.420855, + -0.534562, + 0.600872, + 0.194391 + ] + }, + { + "$id": "vec05", + "metadata": { + "title": "Vector Document 5", + "score": 0.924, + "category": "music" + }, + "embeddings": [ + -0.318783, + -0.876486, + 0.843971, + -0.472869, + -0.350164, + 0.936237, + -0.741591, + 0.298213, + -0.075429, + 0.243415, + 0.929625, + 0.96649, + 0.251843, + -0.371953, + 0.348079, + 0.090382 + ] + }, + { + "$id": "vec06", + "metadata": { + "title": "Vector Document 6", + "score": 0.385, + "category": "science" + }, + "embeddings": [ + -0.608111, + 0.549885, + 0.720346, + 0.260442, + -0.023344, + 0.800346, + -0.956586, + -0.407161, + 0.516883, + 0.230456, + 0.376704, + -0.347203, + 0.925657, + -0.486608, + 0.330032, + 0.323414 + ] + }, + { + "$id": "vec07", + "metadata": { + "title": "Vector Document 7", + "score": 0.61, + "category": "science" + }, + "embeddings": [ + 0.45451, + -0.251783, + -0.479459, + 0.167412, + 0.890343, + 0.415136, + -0.506263, + 0.318171, + -0.729294, + -0.907919, + 0.607033, + 0.652333, + 0.97129, + -0.118671, + 0.110045, + -0.514242 + ] + }, + { + "$id": "vec08", + "metadata": { + "title": "Vector Document 8", + "score": 0.107, + "category": "history" + }, + "embeddings": [ + -0.883255, + -0.294593, + -0.153187, + 0.595936, + 0.341818, + -0.410719, + 0.017348, + -0.350047, + -0.888906, + 0.344646, + 0.184335, + -0.429053, + 0.540542, + -0.408493, + -0.982748, + -0.255031 + ] + }, + { + "$id": "vec09", + "metadata": { + "title": "Vector Document 9", + "score": 0.914, + "category": "art" + }, + "embeddings": [ + 0.31891, + 0.502493, + -0.245712, + -0.325282, + -0.514772, + 0.097323, + 0.812412, + -0.762069, + 0.846688, + -0.391482, + 0.879953, + 0.986352, + -0.562588, + -0.566111, + 0.163109, + 0.119346 + ] + }, + { + "$id": "vec10", + "metadata": { + "title": "Vector Document 10", + "score": 0.169, + "category": "science" + }, + "embeddings": [ + -0.563645, + -0.19119, + -0.853168, + -0.526952, + -0.459635, + 0.850766, + -0.345261, + 0.547467, + 0.455409, + -0.507607, + -0.228436, + -0.587395, + 0.109146, + 0.190046, + 0.861559, + -0.724741 + ] + } +] \ No newline at end of file