Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

add E2E tests for vectorsdb and documentsdb JSON import/export

Browse Source
This commit is contained in:
Prem Palanisamy
2026-03-26 11:24:14 +00:00
parent 52ae8b3880
commit b36472f0da
3 changed files with 742 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/e2e/Services/Migrations/MigrationsBase.php
+318
View File
@@ -1780,4 +1780,322 @@ trait MigrationsBase
'x-appwrite-key' => $this->getProject()['apiKey']
]);
}
public function testCreateVectorsDBJSONExport(): void
{
$headers = [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
];
// Create vectorsdb database
$database = $this->client->call(Client::METHOD_POST, '/vectorsdb', $headers, [
'databaseId' => ID::unique(),
'name' => 'VectorsDB Export Test'
]);
$this->assertEquals(201, $database['headers']['status-code']);
$databaseId = $database['body']['$id'];
// Create collection with dimension 16
$collection = $this->client->call(Client::METHOD_POST, '/vectorsdb/' . $databaseId . '/collections', $headers, [
'collectionId' => ID::unique(),
'name' => 'VecExportCol',
'dimension' => 16,
]);
$this->assertEquals(201, $collection['headers']['status-code']);
$collectionId = $collection['body']['$id'];
// Seed 5 documents
for ($i = 1; $i <= 5; $i++) {
$embeddings = array_map(fn () => round((mt_rand() / mt_getrandmax()) * 2 - 1, 6), range(1, 16));
$doc = $this->client->call(Client::METHOD_POST, '/vectorsdb/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers, [
'documentId' => ID::unique(),
'data' => [
'embeddings' => $embeddings,
'metadata' => ['title' => 'Doc ' . $i, 'score' => round($i * 0.2, 1)]
]
]);
$this->assertEquals(201, $doc['headers']['status-code'], 'Failed to create vector document ' . $i);
}
// Trigger JSON export
$migration = $this->client->call(Client::METHOD_POST, '/migrations/json/exports', $headers, [
'resourceId' => $databaseId . ':' . $collectionId,
'filename' => 'vectorsdb-export-test',
'columns' => [],
'queries' => [],
'notify' => true,
]);
$this->assertEquals(202, $migration['headers']['status-code']);
$migrationId = $migration['body']['$id'];
// Poll until completed
$this->assertEventually(function () use ($migrationId, $headers) {
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, $headers);
$this->assertEquals(200, $migration['headers']['status-code']);
$this->assertEquals('finished', $migration['body']['stage']);
$this->assertEquals('completed', $migration['body']['status']);
$this->assertEquals('Appwrite', $migration['body']['source']);
$this->assertEquals('JSON', $migration['body']['destination']);
}, 30_000, 500);
// Verify email notification
$lastEmail = $this->getLastEmail();
$this->assertNotEmpty($lastEmail);
$this->assertEquals('Your JSON export is ready', $lastEmail['subject']);
// Download and verify JSON content
$body = $lastEmail['html'];
preg_match('/href="([^"]*)"/', $body, $matches);
$this->assertNotEmpty($matches[1], 'Download link should be present in email');
$downloadUrl = html_entity_decode($matches[1]);
$parsedUrl = parse_url($downloadUrl);
parse_str($parsedUrl['query'] ?? '', $queryParams);
$jsonFile = $this->client->call(Client::METHOD_GET, $parsedUrl['path'], [
'x-appwrite-project' => $this->getProject()['$id'],
'origin' => 'http://localhost',
], $queryParams);
$this->assertNotEmpty($jsonFile['body']);
$jsonData = $jsonFile['body'];
$this->assertNotEmpty($jsonData, 'JSON export should not be empty');
// Verify content has embeddings
$decoded = json_decode($jsonData, true);
$this->assertNotNull($decoded);
$this->assertGreaterThanOrEqual(5, count($decoded));
$this->assertArrayHasKey('embeddings', $decoded[0]);
$this->assertCount(16, $decoded[0]['embeddings'], 'Embeddings should have 16 dimensions');
$this->assertArrayHasKey('metadata', $decoded[0]);
// Cleanup
$this->client->call(Client::METHOD_DELETE, '/vectorsdb/' . $databaseId, $headers);
}
public function testCreateVectorsDBJSONImport(): void
{
$headers = [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
];
// Create vectorsdb database
$database = $this->client->call(Client::METHOD_POST, '/vectorsdb', $headers, [
'databaseId' => ID::unique(),
'name' => 'VectorsDB Import Test'
]);
$this->assertEquals(201, $database['headers']['status-code']);
$databaseId = $database['body']['$id'];
// Create collection with dimension 16
$collection = $this->client->call(Client::METHOD_POST, '/vectorsdb/' . $databaseId . '/collections', $headers, [
'collectionId' => ID::unique(),
'name' => 'VecImportCol',
'dimension' => 16,
]);
$this->assertEquals(201, $collection['headers']['status-code']);
$collectionId = $collection['body']['$id'];
// Create bucket and upload test file
$bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', $headers, [
'bucketId' => ID::unique(),
'name' => 'VectorsDB Import Bucket',
'maximumFileSize' => 2000000,
'allowedFileExtensions' => ['json'],
]);
$this->assertEquals(201, $bucket['headers']['status-code']);
$bucketId = $bucket['body']['$id'];
$file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'fileId' => ID::unique(),
'file' => new \CURLFile(realpath(__DIR__ . '/../../../resources/json/vectorsdb-documents.json'), 'application/json', 'vectorsdb-documents.json'),
]);
$this->assertEquals(201, $file['headers']['status-code']);
$fileId = $file['body']['$id'];
// Trigger import
$migration = $this->performJsonMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $collectionId,
]);
$this->assertEquals(202, $migration['headers']['status-code']);
// Poll until completed
$this->assertEventually(function () use ($migration, $headers) {
$migrationId = $migration['body']['$id'];
$result = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, $headers);
$this->assertEquals(200, $result['headers']['status-code']);
$this->assertEquals('finished', $result['body']['stage']);
$this->assertEquals('completed', $result['body']['status']);
$this->assertEquals('JSON', $result['body']['source']);
$this->assertEquals('Appwrite', $result['body']['destination']);
}, 30_000, 500);
// Verify documents were imported
$docs = $this->client->call(Client::METHOD_GET, '/vectorsdb/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers);
$this->assertEquals(200, $docs['headers']['status-code']);
$this->assertEquals(10, $docs['body']['total'], 'Should have imported 10 vectorsdb documents');
// Verify first document structure
$firstDoc = $docs['body']['documents'][0];
$this->assertArrayHasKey('embeddings', $firstDoc);
$this->assertCount(16, $firstDoc['embeddings'], 'Imported embeddings should have 16 dimensions');
$this->assertArrayHasKey('metadata', $firstDoc);
// Cleanup
$this->client->call(Client::METHOD_DELETE, '/vectorsdb/' . $databaseId, $headers);
}
public function testCreateDocumentsDBJSONExport(): void
{
$headers = [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
];
// Create documentsdb database
$database = $this->client->call(Client::METHOD_POST, '/documentsdb', $headers, [
'databaseId' => ID::unique(),
'name' => 'DocumentsDB Export Test'
]);
$this->assertEquals(201, $database['headers']['status-code']);
$databaseId = $database['body']['$id'];
// Create collection (schemaless — no attributes needed)
$collection = $this->client->call(Client::METHOD_POST, '/documentsdb/' . $databaseId . '/collections', $headers, [
'collectionId' => ID::unique(),
'name' => 'DocExportCol',
]);
$this->assertEquals(201, $collection['headers']['status-code']);
$collectionId = $collection['body']['$id'];
// Seed 5 documents
for ($i = 1; $i <= 5; $i++) {
$doc = $this->client->call(Client::METHOD_POST, '/documentsdb/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers, [
'documentId' => ID::unique(),
'data' => [
'name' => 'User ' . $i,
'email' => 'user' . $i . '@test.com',
'age' => 20 + $i,
'address' => ['city' => 'City ' . $i, 'zip' => '1000' . $i]
]
]);
$this->assertEquals(201, $doc['headers']['status-code'], 'Failed to create document ' . $i);
}
// Trigger JSON export
$migration = $this->client->call(Client::METHOD_POST, '/migrations/json/exports', $headers, [
'resourceId' => $databaseId . ':' . $collectionId,
'filename' => 'documentsdb-export-test',
'columns' => [],
'queries' => [],
'notify' => false,
]);
$this->assertEquals(202, $migration['headers']['status-code']);
$migrationId = $migration['body']['$id'];
// Poll until completed
$this->assertEventually(function () use ($migrationId, $headers) {
$migration = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, $headers);
$this->assertEquals(200, $migration['headers']['status-code']);
$this->assertEquals('finished', $migration['body']['stage']);
$this->assertEquals('completed', $migration['body']['status']);
$this->assertEquals('Appwrite', $migration['body']['source']);
$this->assertEquals('JSON', $migration['body']['destination']);
}, 30_000, 500);
// Cleanup
$this->client->call(Client::METHOD_DELETE, '/documentsdb/' . $databaseId, $headers);
}
public function testCreateDocumentsDBJSONImport(): void
{
$headers = [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
];
// Create documentsdb database
$database = $this->client->call(Client::METHOD_POST, '/documentsdb', $headers, [
'databaseId' => ID::unique(),
'name' => 'DocumentsDB Import Test'
]);
$this->assertEquals(201, $database['headers']['status-code']);
$databaseId = $database['body']['$id'];
// Create collection (schemaless)
$collection = $this->client->call(Client::METHOD_POST, '/documentsdb/' . $databaseId . '/collections', $headers, [
'collectionId' => ID::unique(),
'name' => 'DocImportCol',
]);
$this->assertEquals(201, $collection['headers']['status-code']);
$collectionId = $collection['body']['$id'];
// Create bucket and upload test file
$bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', $headers, [
'bucketId' => ID::unique(),
'name' => 'DocumentsDB Import Bucket',
'maximumFileSize' => 2000000,
'allowedFileExtensions' => ['json'],
]);
$this->assertEquals(201, $bucket['headers']['status-code']);
$bucketId = $bucket['body']['$id'];
$file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', array_merge([
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'fileId' => ID::unique(),
'file' => new \CURLFile(realpath(__DIR__ . '/../../../resources/json/documentsdb-documents.json'), 'application/json', 'documentsdb-documents.json'),
]);
$this->assertEquals(201, $file['headers']['status-code']);
$fileId = $file['body']['$id'];
// Trigger import
$migration = $this->performJsonMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $collectionId,
]);
$this->assertEquals(202, $migration['headers']['status-code']);
// Poll until completed
$this->assertEventually(function () use ($migration, $headers) {
$migrationId = $migration['body']['$id'];
$result = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, $headers);
$this->assertEquals(200, $result['headers']['status-code']);
$this->assertEquals('finished', $result['body']['stage']);
$this->assertEquals('completed', $result['body']['status']);
$this->assertEquals('JSON', $result['body']['source']);
$this->assertEquals('Appwrite', $result['body']['destination']);
}, 30_000, 500);
// Verify documents were imported
$docs = $this->client->call(Client::METHOD_GET, '/documentsdb/' . $databaseId . '/collections/' . $collectionId . '/documents', $headers);
$this->assertEquals(200, $docs['headers']['status-code']);
$this->assertEquals(10, $docs['body']['total'], 'Should have imported 10 documentsdb documents');
// Verify first document has nested data
$firstDoc = $docs['body']['documents'][0];
$this->assertArrayHasKey('name', $firstDoc);
$this->assertArrayHasKey('address', $firstDoc);
$this->assertIsArray($firstDoc['address']);
// Cleanup
$this->client->call(Client::METHOD_DELETE, '/documentsdb/' . $databaseId, $headers);
}
}
tests/resources/json/documentsdb-documents.json
+162
View File
@@ -0,0 +1,162 @@
[
{
"$id": "doc01",
"name": "Alice",
"email": "alice@test.com",
"age": 35,
"active": false,
"tags": [
"analyst",
"engineer"
],
"address": {
"city": "New York",
"zip": "27436",
"country": "DE"
}
},
{
"$id": "doc02",
"name": "Bob",
"email": "bob@test.com",
"age": 58,
"active": false,
"tags": [
"engineer",
"manager"
],
"address": {
"city": "London",
"zip": "49320",
"country": "JP"
}
},
{
"$id": "doc03",
"name": "Charlie",
"email": "charlie@test.com",
"age": 23,
"active": true,
"tags": [
"analyst",
"manager"
],
"address": {
"city": "Tokyo",
"zip": "74758",
"country": "JP"
}
},
{
"$id": "doc04",
"name": "Diana",
"email": "diana@test.com",
"age": 60,
"active": true,
"tags": [
"analyst",
"manager"
],
"address": {
"city": "Paris",
"zip": "50703",
"country": "JP"
}
},
{
"$id": "doc05",
"name": "Eve",
"email": "eve@test.com",
"age": 59,
"active": false,
"tags": [
"engineer",
"designer"
],
"address": {
"city": "Berlin",
"zip": "98929",
"country": "UK"
}
},
{
"$id": "doc06",
"name": "Frank",
"email": "frank@test.com",
"age": 23,
"active": true,
"tags": [
"engineer",
"manager"
],
"address": {
"city": "Sydney",
"zip": "82907",
"country": "US"
}
},
{
"$id": "doc07",
"name": "Grace",
"email": "grace@test.com",
"age": 36,
"active": true,
"tags": [
"analyst",
"developer"
],
"address": {
"city": "Toronto",
"zip": "68710",
"country": "US"
}
},
{
"$id": "doc08",
"name": "Hank",
"email": "hank@test.com",
"age": 28,
"active": false,
"tags": [
"engineer",
"manager"
],
"address": {
"city": "Mumbai",
"zip": "61026",
"country": "FR"
}
},
{
"$id": "doc09",
"name": "Iris",
"email": "iris@test.com",
"age": 40,
"active": true,
"tags": [
"developer",
"designer"
],
"address": {
"city": "Seoul",
"zip": "48039",
"country": "FR"
}
},
{
"$id": "doc10",
"name": "Jack",
"email": "jack@test.com",
"age": 45,
"active": false,
"tags": [
"manager",
"developer"
],
"address": {
"city": "Dubai",
"zip": "22733",
"country": "JP"
}
}
]
tests/resources/json/vectorsdb-documents.json
+262
View File
@@ -0,0 +1,262 @@
[
{
"$id": "vec01",
"metadata": {
"title": "Vector Document 1",
"score": 0.643,
"category": "art"
},
"embeddings": [
-0.516361,
0.261456,
-0.77323,
0.909775,
-0.003065,
-0.626888,
0.704171,
-0.366385,
0.198681,
0.854914,
-0.227548,
-0.14278,
-0.548405,
-0.822939,
0.407267,
0.570791
]
},
{
"$id": "vec02",
"metadata": {
"title": "Vector Document 2",
"score": 0.322,
"category": "art"
},
"embeddings": [
0.103211,
0.028378,
0.806001,
-0.412638,
-0.200009,
0.448239,
-0.781144,
0.621957,
-0.868352,
0.217013,
0.308217,
-0.851528,
-0.340766,
0.805996,
-0.065638,
-0.363951
]
},
{
"$id": "vec03",
"metadata": {
"title": "Vector Document 3",
"score": 0.503,
"category": "science"
},
"embeddings": [
0.971901,
-0.475154,
0.60526,
-0.151618,
-0.938199,
-0.139977,
0.154581,
-0.868021,
0.069567,
0.233545,
-0.164826,
0.239036,
0.8875,
0.488075,
0.787231,
0.090293
]
},
{
"$id": "vec04",
"metadata": {
"title": "Vector Document 4",
"score": 0.943,
"category": "science"
},
"embeddings": [
0.372035,
-0.868405,
-0.974987,
-0.836673,
0.030057,
-0.667698,
0.095189,
0.518373,
-0.173732,
0.966379,
0.509861,
-0.172607,
0.420855,
-0.534562,
0.600872,
0.194391
]
},
{
"$id": "vec05",
"metadata": {
"title": "Vector Document 5",
"score": 0.924,
"category": "music"
},
"embeddings": [
-0.318783,
-0.876486,
0.843971,
-0.472869,
-0.350164,
0.936237,
-0.741591,
0.298213,
-0.075429,
0.243415,
0.929625,
0.96649,
0.251843,
-0.371953,
0.348079,
0.090382
]
},
{
"$id": "vec06",
"metadata": {
"title": "Vector Document 6",
"score": 0.385,
"category": "science"
},
"embeddings": [
-0.608111,
0.549885,
0.720346,
0.260442,
-0.023344,
0.800346,
-0.956586,
-0.407161,
0.516883,
0.230456,
0.376704,
-0.347203,
0.925657,
-0.486608,
0.330032,
0.323414
]
},
{
"$id": "vec07",
"metadata": {
"title": "Vector Document 7",
"score": 0.61,
"category": "science"
},
"embeddings": [
0.45451,
-0.251783,
-0.479459,
0.167412,
0.890343,
0.415136,
-0.506263,
0.318171,
-0.729294,
-0.907919,
0.607033,
0.652333,
0.97129,
-0.118671,
0.110045,
-0.514242
]
},
{
"$id": "vec08",
"metadata": {
"title": "Vector Document 8",
"score": 0.107,
"category": "history"
},
"embeddings": [
-0.883255,
-0.294593,
-0.153187,
0.595936,
0.341818,
-0.410719,
0.017348,
-0.350047,
-0.888906,
0.344646,
0.184335,
-0.429053,
0.540542,
-0.408493,
-0.982748,
-0.255031
]
},
{
"$id": "vec09",
"metadata": {
"title": "Vector Document 9",
"score": 0.914,
"category": "art"
},
"embeddings": [
0.31891,
0.502493,
-0.245712,
-0.325282,
-0.514772,
0.097323,
0.812412,
-0.762069,
0.846688,
-0.391482,
0.879953,
0.986352,
-0.562588,
-0.566111,
0.163109,
0.119346
]
},
{
"$id": "vec10",
"metadata": {
"title": "Vector Document 10",
"score": 0.169,
"category": "science"
},
"embeddings": [
-0.563645,
-0.19119,
-0.853168,
-0.526952,
-0.459635,
0.850766,
-0.345261,
0.547467,
0.455409,
-0.507607,
-0.228436,
-0.587395,
0.109146,
0.190046,
0.861559,
-0.724741
]
}
]