Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

updated migrations for vectordb importing/exporting in csv endpoints

Browse Source
This commit is contained in:
ArnabChatterjee20k
2025-11-20 18:54:41 +05:30
parent cc759868b8
commit c3ad337398
6 changed files with 263 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/controllers/api/migrations.php
+2 -1
View File
@@ -47,7 +47,8 @@ function getDatabaseTransferResourceServices(string $databaseType)
{
return match($databaseType) {
DATABASE_LEGACY_TYPE,
DATABASE_TABLESDB_TYPE => Transfer::GROUP_DATABASES_TABLES_DB
DATABASE_TABLESDB_TYPE => Transfer::GROUP_DATABASES_TABLES_DB,
DATABASE_VECTORDB_TYPE => Transfer::GROUP_DATABASES_VECTOR_DB
};
}
app/init/constants.php
+2 -1
View File
@@ -302,6 +302,7 @@ const SCHEDULE_RESOURCE_TYPE_MESSAGE = 'message';
const DATABASE_LEGACY_TYPE = 'legacy';
const DATABASE_TABLESDB_TYPE = 'tablesdb';
const DATABASE_DOCUMENTSDB_TYPE = 'documentsdb';
const DATABASE_VECTORDB_TYPE = 'vectordb';
// CSV import/export allowed database types
const CSV_ALLOWED_DATABASE_TYPES = [DATABASE_LEGACY_TYPE, DATABASE_LEGACY_TYPE];
const CSV_ALLOWED_DATABASE_TYPES = [DATABASE_LEGACY_TYPE, DATABASE_LEGACY_TYPE, DATABASE_VECTORDB_TYPE];
composer.lock
Generated
+4 -4
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "33efc5db38812c8c9257d0e398948685",
"content-hash": "bdd11652978ebab653ca17a00074343d",
"packages": [
{
"name": "adhocore/jwt",
@@ -4509,12 +4509,12 @@
"source": {
"type": "git",
"url": "https://github.com/utopia-php/migration.git",
"reference": "105b48dcf2bc8a1a969497813f3d7d06231770e6"
"reference": "bcd7bcb634dea60485c16648e68ed7a255315560"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/105b48dcf2bc8a1a969497813f3d7d06231770e6",
"reference": "105b48dcf2bc8a1a969497813f3d7d06231770e6",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/bcd7bcb634dea60485c16648e68ed7a255315560",
"reference": "bcd7bcb634dea60485c16648e68ed7a255315560",
"shasum": ""
},
"require": {
src/Appwrite/Platform/Workers/Migrations.php
+2 -1
View File
@@ -205,7 +205,8 @@ class Migrations extends Action
$resourceId,
$migrationOptions['path'],
$this->deviceForMigrations,
$this->dbForProject
$this->dbForProject,
$getDatabasesDB
),
default => throw new \Exception('Invalid source type'),
};
tests/e2e/Services/Migrations/MigrationsBase.php
+250
View File
@@ -1377,6 +1377,256 @@ trait MigrationsBase
]);
}
/**
* Import VectorDB documents from CSV
*/
public function testImportVectordbCSV(): void
{
$databaseId = null;
$collectionId = null;
$bucketId = null;
try {
$database = $this->client->call(Client::METHOD_POST, '/vectordb', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
], [
'databaseId' => ID::unique(),
'name' => 'Vector CSV Import DB'
]);
$this->assertEquals(201, $database['headers']['status-code']);
$databaseId = $database['body']['$id'];
$collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey']
], [
'collectionId' => ID::unique(),
'name' => 'Vector CSV Import Collection',
'dimensions' => 3,
'documentSecurity' => true,
]);
$this->assertEquals(201, $collection['headers']['status-code']);
$collectionId = $collection['body']['$id'];
$bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'bucketId' => ID::unique(),
'name' => 'Vector CSV Bucket',
'maximumFileSize' => 2000000,
'allowedFileExtensions' => ['csv'],
]);
$this->assertEquals(201, $bucket['headers']['status-code']);
$bucketId = $bucket['body']['$id'];
$file = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . $bucketId . '/files', [
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'fileId' => ID::unique(),
'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/csv/vectordb-documents.csv'), 'text/csv', 'vectordb-documents.csv'),
]);
$this->assertEquals(201, $file['headers']['status-code']);
$fileId = $file['body']['$id'];
$migration = $this->performCsvMigration([
'fileId' => $fileId,
'bucketId' => $bucketId,
'resourceId' => $databaseId . ':' . $collectionId,
]);
$this->assertEquals(202, $migration['headers']['status-code']);
$this->assertEventually(function () use ($migration) {
$migrationId = $migration['body']['$id'];
$status = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]);
$this->assertEquals(200, $status['headers']['status-code']);
$this->assertEquals('finished', $status['body']['stage']);
$this->assertEquals('completed', $status['body']['status']);
$this->assertContains(Resource::TYPE_DOCUMENT, $status['body']['resources']);
$this->assertArrayHasKey(Resource::TYPE_DOCUMENT, $status['body']['statusCounters']);
$this->assertEquals(2, $status['body']['statusCounters'][Resource::TYPE_DOCUMENT]['success']);
return true;
}, 60_000, 500);
$documents = $this->client->call(Client::METHOD_GET, '/vectordb/' . $databaseId . '/collections/' . $collectionId . '/documents', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'queries' => [
Query::limit(10)->toString(),
],
]);
$this->assertEquals(200, $documents['headers']['status-code']);
$this->assertEquals(2, $documents['body']['total']);
$titles = array_map(fn ($doc) => $doc['metadata']['title'] ?? null, $documents['body']['documents']);
$this->assertContains('Vector Alpha', $titles);
$this->assertContains('Vector Beta', $titles);
} finally {
if ($bucketId) {
$this->client->call(Client::METHOD_DELETE, '/storage/buckets/' . $bucketId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]);
}
if ($databaseId) {
$this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]);
}
}
}
/**
* Export VectorDB documents to CSV
*/
public function testExportVectordbCSV(): void
{
$databaseId = null;
try {
$database = $this->client->call(Client::METHOD_POST, '/vectordb', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'databaseId' => ID::unique(),
'name' => 'Vector CSV Export DB',
]);
$this->assertEquals(201, $database['headers']['status-code']);
$databaseId = $database['body']['$id'];
$collection = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'collectionId' => ID::unique(),
'name' => 'Vector CSV Export Collection',
'dimensions' => 3,
'documentSecurity' => true,
]);
$this->assertEquals(201, $collection['headers']['status-code']);
$collectionId = $collection['body']['$id'];
$documentsPayload = [
[
'documentId' => ID::unique(),
'data' => [
'embeddings' => [0.11, 0.22, 0.33],
'metadata' => ['title' => 'Vector Sample One', 'category' => 'alpha'],
],
],
[
'documentId' => ID::unique(),
'data' => [
'embeddings' => [0.44, 0.55, 0.66],
'metadata' => ['title' => 'Vector Sample Two', 'category' => 'beta'],
],
],
];
foreach ($documentsPayload as $payload) {
$response = $this->client->call(Client::METHOD_POST, '/vectordb/' . $databaseId . '/collections/' . $collectionId . '/documents', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], $payload);
$this->assertEquals(201, $response['headers']['status-code']);
}
$filename = 'vectordb-export-' . ID::unique();
$migration = $this->client->call(Client::METHOD_POST, '/migrations/csv/exports', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'resourceId' => $databaseId . ':' . $collectionId,
'filename' => $filename,
'columns' => [],
'queries' => [],
'delimiter' => ',',
'enclosure' => '"',
'escape' => '\\',
'header' => true,
'notify' => true,
]);
$this->assertEquals(202, $migration['headers']['status-code']);
$migrationId = $migration['body']['$id'];
$this->assertEventually(function () use ($migrationId) {
$response = $this->client->call(Client::METHOD_GET, '/migrations/' . $migrationId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals('finished', $response['body']['stage']);
$this->assertEquals('completed', $response['body']['status']);
return true;
}, 30_000, 500);
$lastEmail = $this->getLastEmail();
$this->assertNotEmpty($lastEmail);
$this->assertEquals('Your CSV export is ready', $lastEmail['subject']);
\preg_match('/href="([^"]*\/storage\/buckets\/[^"]*\/push[^"]*)"/', $lastEmail['html'], $matches);
$this->assertNotEmpty($matches[1], 'Download URL not found in email');
$downloadUrl = html_entity_decode($matches[1]);
$components = \parse_url($downloadUrl);
$this->assertNotEmpty($components);
\parse_str($components['query'] ?? '', $queryParams);
$this->assertArrayHasKey('jwt', $queryParams);
$this->assertArrayHasKey('project', $queryParams);
$path = \str_replace('/v1', '', $components['path']);
$downloadResponse = $this->client->call(Client::METHOD_GET, $path . '?project=' . $queryParams['project'] . '&jwt=' . $queryParams['jwt']);
$this->assertEquals(200, $downloadResponse['headers']['status-code']);
$csvData = $downloadResponse['body'];
$this->assertStringContainsString('Vector Sample One', $csvData);
$this->assertStringContainsString('Vector Sample Two', $csvData);
$this->assertStringContainsString('[0.11,0.22,0.33]', $csvData);
} finally {
if ($databaseId) {
$this->client->call(Client::METHOD_DELETE, '/vectordb/' . $databaseId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
]);
}
}
}
/**
* DocumentsDB (schemaless)
*/
tests/resources/csv/vectordb-documents.csv
+3
View File
@@ -0,0 +1,3 @@
$id,embeddings,metadata
vector-doc-1,"[0.15,0.25,0.35]","{""title"":""Vector Alpha"",""category"":""science""}"
vector-doc-2,"[0.55,0.65,0.75]","{""title"":""Vector Beta"",""category"":""history""}"
1 $id embeddings metadata
2 vector-doc-1 [0.15,0.25,0.35] {"title":"Vector Alpha","category":"science"}
3 vector-doc-2 [0.55,0.65,0.75] {"title":"Vector Beta","category":"history"}